Allow additional queries for prometheus external metrics

allegro · Oct 10, 2023 · 7598d81 · 7598d81
1 parent f4c58bb
commit 7598d81
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 10 deletions.
diff --git a/...c/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java b/...c/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java
@@ -55,7 +55,8 @@ public GraphiteClient graphiteClient(@Qualifier("monitoringRestTemplate") RestTe
     public VictoriaMetricsMetricsProvider prometheusMetricsProvider(PrometheusClient prometheusClient,
                                                                     PrometheusMonitoringClientProperties properties) {
         return new VictoriaMetricsMetricsProvider(prometheusClient,
-                properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix());
+                properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix(),
+                properties.getAdditionalFilters());
     }
 
     @Bean

diff --git a/...n/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java b/...n/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java
@@ -3,6 +3,7 @@
 public class PrometheusMonitoringClientProperties extends ExternalMonitoringClientProperties {
     private String consumersMetricsPrefix = "hermes_consumers";
     private String frontendMetricsPrefix = "hermes_frontend";
+    private String additionalFilters = "";
 
     public String getConsumersMetricsPrefix() {
         return consumersMetricsPrefix;
@@ -19,4 +20,12 @@ public String getFrontendMetricsPrefix() {
     public void setFrontendMetricsPrefix(String frontendMetricsPrefix) {
         this.frontendMetricsPrefix = frontendMetricsPrefix;
     }
+
+    public String getAdditionalFilters() {
+        return additionalFilters;
+    }
+
+    public void setAdditionalFilters(String additionalFilters) {
+        this.additionalFilters = additionalFilters;
+    }
 }
diff --git a/...egro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java b/...egro/tech/hermes/management/infrastructure/prometheus/VictoriaMetricsMetricsProvider.java
@@ -28,15 +28,17 @@ public class VictoriaMetricsMetricsProvider implements MonitoringSubscriptionMet
 
     private final String consumersMetricsPrefix;
     private final String frontendMetricsPrefix;
+    private final String additionalFilters;
     private final String subscriptionMetricsToQuery;
     private final String topicMetricsToQuery;
     private final PrometheusClient prometheusClient;
 
     public VictoriaMetricsMetricsProvider(PrometheusClient prometheusClient, String consumersMetricsPrefix,
-                                          String frontendMetricsPrefix) {
+                                          String frontendMetricsPrefix, String additionalFilters) {
         this.prometheusClient = prometheusClient;
         this.consumersMetricsPrefix = consumersMetricsPrefix.isEmpty() ? "" : consumersMetricsPrefix + "_";
         this.frontendMetricsPrefix = frontendMetricsPrefix.isEmpty() ? "" : frontendMetricsPrefix + "_";
+        this.additionalFilters = additionalFilters;
         this.subscriptionMetricsToQuery = Stream.of(SUBSCRIPTION_DELIVERED, SUBSCRIPTION_TIMEOUTS,
                         SUBSCRIPTION_THROUGHPUT, SUBSCRIPTION_OTHER_ERRORS, SUBSCRIPTION_BATCHES,
                         SUBSCRIPTION_STATUS_CODES)
@@ -56,9 +58,9 @@ public MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscr
         https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param.
          */
         String queryFormat = "sum by (__name__,group,topic,subscription,status_code)"
-                + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s'}[1m]) keep_metric_names)";
+                + "(irate({__name__=~'%s',group='%s',topic='%s',subscription='%s', %s}[1m]) keep_metric_names)";
         String query = String.format(queryFormat, subscriptionMetricsToQuery, subscriptionName.getTopicName().getGroupName(),
-                subscriptionName.getTopicName().getName(), subscriptionName.getName());
+                subscriptionName.getTopicName().getName(), subscriptionName.getName(), additionalFilters);
         MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query);
         return MonitoringSubscriptionMetricsProvider
                 .metricsBuilder()
@@ -80,8 +82,9 @@ public MonitoringTopicMetrics topicMetrics(TopicName topicName) {
         https://docs.victoriametrics.com/MetricsQL.html. Basic PromQL does not support `keep_metric_names` param.
          */
         String queryFormat = "sum by (__name__, group, topic) (irate({__name__=~'%s', group='%s', "
-                + "topic='%s'}[1m]) keep_metric_names)";
-        String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName());
+                + "topic='%s', %s}[1m]) keep_metric_names)";
+        String query = String.format(queryFormat, topicMetricsToQuery, topicName.getGroupName(), topicName.getName(),
+                additionalFilters);
         MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics(query);
         return MonitoringTopicMetricsProvider
                 .metricsBuilder()

diff --git a/...ment/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy b/...ment/infrastructure/metrics/HybridPrometheusBasedSubscriptionMetricsRepositoryTest.groovy
@@ -22,7 +22,8 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati
 
     private SubscriptionLagSource lagSource = new NoOpSubscriptionLagSource()
 
-    private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client, "hermes_consumers", "hermes_frontend");
+    private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(
+            client, "hermes_consumers", "hermes_frontend", "service=~'hermes'");
 
     private HybridSubscriptionMetricsRepository repository = new HybridSubscriptionMetricsRepository(prometheusMetricsProvider,
             summedSharedCounter, zookeeperPaths, lagSource)
@@ -34,7 +35,7 @@ class HybridPrometheusBasedSubscriptionMetricsRepositoryTest extends Specificati
             "|hermes_consumers_subscription_other_errors_total" +
             "|hermes_consumers_subscription_batches_total" +
             "|hermes_consumers_subscription_http_status_codes_total'," +
-            "group='group',topic='topic',subscription='subscription'}[1m]) keep_metric_names)"
+            "group='group',topic='topic',subscription='subscription', service=~'hermes'}[1m]) keep_metric_names)"
 
     def "should read subscription metrics from multiple places"() {
         given:

diff --git a/.../management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy b/.../management/infrastructure/metrics/HybridPrometheusBasedTopicMetricsRepositoryTest.groovy
@@ -21,7 +21,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification {
     private SubscriptionRepository subscriptionRepository = Mock(SubscriptionRepository)
 
     private VictoriaMetricsMetricsProvider prometheusMetricsProvider = new VictoriaMetricsMetricsProvider(client,
-            "hermes_consumers", "hermes_frontend")
+            "hermes_consumers", "hermes_frontend", "service='hermes'")
 
     private HybridTopicMetricsRepository repository = new HybridTopicMetricsRepository(prometheusMetricsProvider,
             summedSharedCounter, zookeeperPaths, subscriptionRepository)
@@ -31,7 +31,7 @@ class HybridPrometheusBasedTopicMetricsRepositoryTest extends Specification {
         String query = "sum by (__name__, group, topic) (irate({__name__=~'hermes_frontend_topic_requests_total" +
                 "|hermes_consumers_subscription_delivered_total" +
                 "|hermes_frontend_topic_throughput_bytes_total', group='group', " +
-                "topic='topic'}[1m]) keep_metric_names)"
+                "topic='topic', service='hermes'}[1m]) keep_metric_names)"
         TopicName topic = new TopicName('group', 'topic')
 
         client.readMetrics(query) >> MonitoringMetricsContainer.createEmpty()