Fetching external metrics from Prometheus (#1711)

* Fetching external metrics from Prometheus

* Rewriting metrics tests for prometheus

* Metrics tests based on prometheus

* Unhelathy topic/subscriptions tests rewritten to prometheus

* Rewrite tests to stop using Graphite metrics storage

* Fix not working tests

* Fix checkstyle errors

* Add some fixes

* Add tests for graphite external monitoring

* Add code style fixes

* Add code refactor

* Fix error after refactor

* Fix error after refactor

* Fix unworking test

* Fix unworking tests

* Adjust prometheus config for management

* Adjust prometheus config for management

* Adjust prometheus config for management

* Fix prometheus url encoding

* Fix prometheus url encoding

* Fix prometheus config

* Test prometheus uri

* Test prometheus uri

* Cr fixes + metrics documentation.

* Cr fixes

docker/latest/management/management.yaml

@@ -17,10 +17,10 @@ kafka:
       connectionTimeout: 3000
       bootstrapKafkaServer: kafka:29092
 
-metrics:
-  graphiteHttpUri: graphite:8082
-  graphite:
+graphite:
+  client:
     enabled: true
+    externalMonitoringUrl: graphite:8082
 
 server:
   port: 8090

docs/docs/configuration/metrics.md

@@ -1,44 +1,20 @@
 # Metrics
 
-Hermes Frontend and Consumers use [Dropwizard Metrics](https://dropwizard.github.io/metrics/3.1.0/) library to gather
-and publish metrics to Metric Store.
+Hermes Frontend, Consumers and Management use [Micrometer Metrics](https://github.com/micrometer-metrics/micrometer) library to gather
+and expose metrics.
 
-If you would like to preview or debug metrics, set `{modulePrefix}.metrics.consoleReporterEnabled` to `true`, so they will be printed
-to stdout.
+## Prometheus
+By default, Hermes includes Prometheus reporter. It exposes metrics on `/status/prometheus` endpoint. 
+Reporter configuration can be configured using following options:
 
-## Graphite
+Option                                         | Description                                           | Default value
+---------------------------------------------- |-------------------------------------------------------| -------------
+{modulePrefix}.metrics.prometheus.step         | The step size to use in computing windowed statistics | 60s
+{modulePrefix}.metrics.prometheus.descriptions | If meter descriptions should be sent to Prometheus    | true
 
-By default, Hermes includes Graphite reporter, which can be configured using following options:
+In order to be able to access basic metrics via Management API, it needs to be configured to reach VictoriaMetrics API:
 
-Option                                         | Description                                  | Default value
----------------------------------------------- | -------------------------------------- | -------------
-{modulePrefix}.metrics.graphiteReporterEnabled | enable Graphite reporter               | false
-{modulePrefix}.graphite.host                   | Graphite host                          | localhost
-{modulePrefix}.graphite.port                   | Graphite port                          | 2003
-{modulePrefix}.graphite.prefix                 | prefix for all metrics                 | stats.tech.hermes
-{modulePrefix}.metrics.reportPeriod            | how often to report metrics            | 20s
-
-In order to be able to access basic metrics via Management API, it needs to be configured to reach Graphite API:
-
-Option                  | Description              | Default value
------------------------ | ------------------------ | -------------
-metrics.graphiteHttpUri | URI to Graphite HTTP API | http://localhost:80
-metrics.prefix          | prefix for all metrics   | stats.tech.hermes
-
-## Custom
-
-You can register any custom reporter that is compatible with Dropwizard `MetricRegistry`.
-
-For the Consumers and Frontend modules register the reporter as a bean, for example:
-
-```java
-@Configuration
-public class CustomHermesConsumersConfiguration {
-
-    @Bean
-    @Primary
-    public MetricRegistry myMetricRegistry(MetricRegistry metricRegistry) {
-        return new MyMetricsReporter(metricRegistry);
-    }
-}
-```
+Option                                    | Description                                   | Default value
+------------------------------------------|-----------------------------------------------| -------------
+prometheus.client.enabled                 | Should fetch external metrics from Prometheus | true
+prometheus.client.externalMonitoringUrl   | URI to VictoriaMetrics HTTP API               | http://localhost:18090

hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UnhealthyEndpoint.java

@@ -51,7 +51,7 @@ public Response listUnhealthy(
                 ))
                 .orElseThrow(() -> new OwnerSource.OwnerNotFound(ownerSourceName, id));
         return Response.ok()
-                .entity(new GenericEntity<List<UnhealthySubscription>>(unhealthySubscriptions) {
+                .entity(new GenericEntity<>(unhealthySubscriptions) {
                 })
                 .build();
     }

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/GraphiteClientProperties.java → hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringClientProperties.java

@@ -1,9 +1,6 @@
 package pl.allegro.tech.hermes.management.config;
 
-import org.springframework.boot.context.properties.ConfigurationProperties;
-
-@ConfigurationProperties("graphite.client")
-public class GraphiteClientProperties {
+public class ExternalMonitoringClientProperties {
 
     private int connectionTimeoutMillis = 1000;
 
@@ -17,6 +14,8 @@ public class GraphiteClientProperties {
 
     private int cacheSize = 100_000;
 
+    private String externalMonitoringUrl = "http://localhost:18090";
+
     public int getConnectionTimeoutMillis() {
         return connectionTimeoutMillis;
     }
@@ -64,4 +63,12 @@ public int getMaxConnectionsPerRoute() {
     public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) {
         this.maxConnectionsPerRoute = maxConnectionsPerRoute;
     }
+
+    public String getExternalMonitoringUrl() {
+        return externalMonitoringUrl;
+    }
+
+    public void setExternalMonitoringUrl(String externalMonitoringUrl) {
+        this.externalMonitoringUrl = externalMonitoringUrl;
+    }
 }

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java

@@ -0,0 +1,96 @@
+package pl.allegro.tech.hermes.management.config;
+
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
+import org.apache.hc.core5.util.Timeout;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.ClientHttpRequestFactory;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.web.client.RestTemplate;
+import pl.allegro.tech.hermes.management.infrastructure.graphite.CachingGraphiteClient;
+import pl.allegro.tech.hermes.management.infrastructure.graphite.GraphiteClient;
+import pl.allegro.tech.hermes.management.infrastructure.graphite.GraphiteMetricsProvider;
+import pl.allegro.tech.hermes.management.infrastructure.graphite.RestTemplateGraphiteClient;
+import pl.allegro.tech.hermes.management.infrastructure.prometheus.CachingPrometheusClient;
+import pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient;
+import pl.allegro.tech.hermes.management.infrastructure.prometheus.RestTemplatePrometheusClient;
+import pl.allegro.tech.hermes.management.infrastructure.prometheus.VictoriaMetricsMetricsProvider;
+
+import java.net.URI;
+
+import static com.google.common.base.Ticker.systemTicker;
+
+@Configuration
+public class ExternalMonitoringConfiguration {
+
+    @Bean
+    @ConditionalOnProperty(value = "graphite.client.enabled", havingValue = "true")
+    public GraphiteMetricsProvider graphiteMetricsProvider(GraphiteClient graphiteClient,
+                                                           GraphiteMonitoringMetricsProperties properties) {
+        return new GraphiteMetricsProvider(graphiteClient, properties.getPrefix());
+    }
+
+    @Bean
+    @ConditionalOnProperty(value = "graphite.client.enabled", havingValue = "true")
+    public GraphiteClient graphiteClient(@Qualifier("monitoringRestTemplate") RestTemplate graphiteRestTemplate,
+                                         GraphiteMonitoringMetricsProperties graphiteClientProperties) {
+        RestTemplateGraphiteClient underlyingGraphiteClient =
+                new RestTemplateGraphiteClient(graphiteRestTemplate, URI.create(graphiteClientProperties.getExternalMonitoringUrl()));
+        return new CachingGraphiteClient(
+                underlyingGraphiteClient,
+                systemTicker(),
+                graphiteClientProperties.getCacheTtlSeconds(),
+                graphiteClientProperties.getCacheSize()
+        );
+    }
+
+    @Bean
+    @ConditionalOnProperty(value = "prometheus.client.enabled", havingValue = "true")
+    public VictoriaMetricsMetricsProvider prometheusMetricsProvider(PrometheusClient prometheusClient,
+                                                                    PrometheusMonitoringClientProperties properties) {
+        return new VictoriaMetricsMetricsProvider(prometheusClient,
+                properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix());
+    }
+
+    @Bean
+    @ConditionalOnProperty(value = "prometheus.client.enabled", havingValue = "true")
+    public PrometheusClient prometheusClient(@Qualifier("monitoringRestTemplate") RestTemplate graphiteRestTemplate,
+                                             PrometheusMonitoringClientProperties clientProperties) {
+        RestTemplatePrometheusClient underlyingPrometheusClient =
+                new RestTemplatePrometheusClient(graphiteRestTemplate, URI.create(clientProperties.getExternalMonitoringUrl()));
+        return new CachingPrometheusClient(
+                underlyingPrometheusClient,
+                systemTicker(),
+                clientProperties.getCacheTtlSeconds(),
+                clientProperties.getCacheSize()
+        );
+    }
+
+    @Bean("monitoringRestTemplate")
+    public RestTemplate restTemplate(ExternalMonitoringClientProperties clientProperties) {
+        PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
+                .setMaxConnTotal(clientProperties.getMaxConnections())
+                .setMaxConnPerRoute(clientProperties.getMaxConnectionsPerRoute())
+                .build();
+
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(Timeout.ofMilliseconds(clientProperties.getConnectionTimeoutMillis()))
+                .setResponseTimeout(Timeout.ofMilliseconds(clientProperties.getSocketTimeoutMillis()))
+                .build();
+
+        HttpClient client = HttpClientBuilder.create()
+                .setDefaultRequestConfig(requestConfig)
+                .setConnectionManager(connectionManager)
+                .build();
+
+        ClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(client);
+
+        return new RestTemplate(clientHttpRequestFactory);
+    }
+}

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/GraphiteMonitoringMetricsProperties.java

@@ -0,0 +1,14 @@
+package pl.allegro.tech.hermes.management.config;
+
+public class GraphiteMonitoringMetricsProperties extends ExternalMonitoringClientProperties {
+
+    private String prefix = "stats.tech.hermes";
+
+    public String getPrefix() {
+        return prefix;
+    }
+
+    public void setPrefix(String prefix) {
+        this.prefix = prefix;
+    }
+}

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ManagementConfiguration.java

@@ -28,7 +28,6 @@
 @Configuration
 @EnableConfigurationProperties({
         TopicProperties.class,
-        MetricsProperties.class,
         HttpClientProperties.class,
         ConsistencyCheckerProperties.class,
         PrometheusProperties.class,

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MonitoringClientPropertiesConfiguration.java

@@ -0,0 +1,27 @@
+package pl.allegro.tech.hermes.management.config;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/*
+These properties beans must be in different configuration class than ExternalMonitoringConfiguration.java. It allows
+avoiding circular dependencies between beans.
+ */
+@Configuration
+public class MonitoringClientPropertiesConfiguration {
+    @Bean
+    @ConfigurationProperties("graphite.client")
+    @ConditionalOnProperty(value = "graphite.client.enabled", havingValue = "true")
+    public GraphiteMonitoringMetricsProperties graphiteMonitoringClientProperties() {
+        return new GraphiteMonitoringMetricsProperties();
+    }
+
+    @Bean
+    @ConfigurationProperties("prometheus.client")
+    @ConditionalOnProperty(value = "prometheus.client.enabled", havingValue = "true")
+    public PrometheusMonitoringClientProperties prometheusMonitoringClientProperties() {
+        return new PrometheusMonitoringClientProperties();
+    }
+}

hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfigAdapter.java

@@ -14,7 +14,7 @@ public PrometheusConfigAdapter(PrometheusProperties prometheusReporterProperties
 
     @Override
     public boolean descriptions() {
-        return prometheusReporterProperties.isDescriptions();
+        return prometheusReporterProperties.getDescriptions();
     }
 
     @Override