Add integration test assertion to ensure cortex_distributor_replicati…

…on_factor is not exported when ingest storage is enabled (#9385) * Add integration test to ensure cortex_distributor_replication_factor is not exported when ingest storage is enabled Signed-off-by: Marco Pracucci <[email protected]> * Added counterproof that cortex_distributor_replication_factor is exported when ingest storage is disabled Signed-off-by: Marco Pracucci <[email protected]> --------- Signed-off-by: Marco Pracucci <[email protected]>
grafana · Sep 24, 2024 · 41a6c6d · 41a6c6d
1 parent 9698ad8
commit 41a6c6d
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 0 deletions.
diff --git a/integration/asserts.go b/integration/asserts.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/grafana/regexp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -96,3 +97,56 @@ func getBlacklistedMetricsPrefixesByService(serviceType ServiceType) []string {
 
 	return blacklist
 }
+
+func assertServiceMetricsNotMatching(t *testing.T, metricName string, services ...*e2emimir.MimirService) {
+	for _, service := range services {
+		if service == nil {
+			continue
+		}
+
+		metrics, err := service.Metrics()
+		require.NoError(t, err)
+
+		if isRawMetricsContainingMetricName(metricName, metrics) {
+			assert.Failf(t, "the service %s exported metrics include the metric name %s but it should not export it", service.Name(), metricName)
+		}
+	}
+}
+
+func assertServiceMetricsMatching(t *testing.T, metricName string, services ...*e2emimir.MimirService) {
+	for _, service := range services {
+		if service == nil {
+			continue
+		}
+
+		metrics, err := service.Metrics()
+		require.NoError(t, err)
+
+		if !isRawMetricsContainingMetricName(metricName, metrics) {
+			assert.Failf(t, "the service %s exported metrics don't include the metric name %s but it should export it", service.Name(), metricName)
+		}
+	}
+}
+
+func isRawMetricsContainingMetricName(metricName string, metrics string) bool {
+	metricNameRegex := regexp.MustCompile("^[^ \\{]+")
+
+	// Ensure no metric name matches the input one.
+	for _, metricLine := range strings.Split(metrics, "\n") {
+		metricLine = strings.TrimSpace(metricLine)
+		if metricLine == "" || strings.HasPrefix(metricLine, "#") {
+			continue
+		}
+
+		actualMetricName := metricNameRegex.FindStringSubmatch(metricLine)
+		if len(actualMetricName) != 1 {
+			continue
+		}
+
+		if actualMetricName[0] == metricName {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/integration/asserts_test.go b/integration/asserts_test.go
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Provenance-includes-location: https://github.com/cortexproject/cortex/blob/master/integration/asserts.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: The Cortex Authors.
+//go:build requires_docker
+
+package integration
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsRawMetricsContainingMetricName(t *testing.T) {
+	rawMetrics := `
+# HELP metric_1 Test
+# TYPE metric_1 counter
+metric_1{a="b"} 0
+metric_1{c="d"} 0
+
+# HELP metric_10 Test
+# TYPE metric_10 counter
+metric_10 0
+
+# HELP metric_20 Test
+# TYPE metric_20 counter
+metric_20 0
+`
+
+	assert.True(t, isRawMetricsContainingMetricName("metric_1", rawMetrics))
+	assert.True(t, isRawMetricsContainingMetricName("metric_10", rawMetrics))
+	assert.True(t, isRawMetricsContainingMetricName("metric_20", rawMetrics))
+	assert.False(t, isRawMetricsContainingMetricName("metric_2", rawMetrics))
+	assert.False(t, isRawMetricsContainingMetricName("metric_200", rawMetrics))
+}
diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go
@@ -455,6 +455,14 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 	assertServiceMetricsPrefixes(t, Querier, querier)
 	assertServiceMetricsPrefixes(t, QueryFrontend, queryFrontend)
 	assertServiceMetricsPrefixes(t, QueryScheduler, queryScheduler)
+
+	if flags["-ingest-storage.enabled"] == "true" {
+		// Ensure cortex_distributor_replication_factor is not exported when ingest storage is enabled
+		// because it's how we detect whether a Mimir cluster is running with ingest storage.
+		assertServiceMetricsNotMatching(t, "cortex_distributor_replication_factor", queryFrontend, queryScheduler, distributor, ingester, querier)
+	} else {
+		assertServiceMetricsMatching(t, "cortex_distributor_replication_factor", distributor)
+	}
 }
 
 // This spins up a minimal query-frontend setup and compares if errors returned

diff --git a/integration/ruler_test.go b/integration/ruler_test.go
@@ -1208,6 +1208,10 @@ func TestRulerRemoteEvaluation_ShouldEnforceStrongReadConsistencyForDependentRul
 		// have run with eventual consistency because they are independent.
 		require.NoError(t, queryFrontend.WaitSumMetrics(e2e.Equals(0), "cortex_ingest_storage_strong_consistency_requests_total"))
 		require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(0), "cortex_ingest_storage_strong_consistency_requests_total"))
+
+		// Ensure cortex_distributor_replication_factor is not exported when ingest storage is enabled
+		// because it's how we detect whether a Mimir cluster is running with ingest storage.
+		assertServiceMetricsNotMatching(t, "cortex_distributor_replication_factor", queryFrontend, distributor, ingester, querier, ruler)
 	})
 
 	t.Run("evaluation of dependent rules should require strong consistency", func(t *testing.T) {
@@ -1242,6 +1246,10 @@ func TestRulerRemoteEvaluation_ShouldEnforceStrongReadConsistencyForDependentRul
 		// We expect the offsets to be fetched by query-frontend and then propagated to ingesters.
 		require.NoError(t, ingester.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"cortex_ingest_storage_strong_consistency_requests_total"}, e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "with_offset", "true"))))
 		require.NoError(t, ingester.WaitSumMetricsWithOptions(e2e.Equals(0), []string{"cortex_ingest_storage_strong_consistency_requests_total"}, e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "with_offset", "false"))))
+
+		// Ensure cortex_distributor_replication_factor is not exported when ingest storage is enabled
+		// because it's how we detect whether a Mimir cluster is running with ingest storage.
+		assertServiceMetricsNotMatching(t, "cortex_distributor_replication_factor", queryFrontend, distributor, ingester, querier, ruler)
 	})
 }