Merge branch 'master' into bigquery-sender

allegro · Aug 19, 2024 · ff76011 · ff76011
2 parents 6d19115 + af320d6
commit ff76011
Show file tree

Hide file tree

Showing 727 changed files with 22,346 additions and 21,927 deletions.
diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml
@@ -25,7 +25,7 @@ jobs:
         uses: gradle/gradle-build-action@v2
       - name: Run check style
         # ignore lengthy console setup tasks
-        run: ./gradlew --continue clean checkstyleMain checkstyleTest checkstyleIntegration checkstyleJmh -PmaxCheckstyleWarnings=0 -x attachHermesConsole -x prepareIndexTemplate
+        run: ./gradlew --continue clean checkstyleMain checkstyleTest checkstyleIntegrationTest checkstyleSlowIntegrationTest checkstyleJmh -PmaxCheckstyleWarnings=0 -x attachHermesConsole -x prepareIndexTemplate
       - name: Run reviewdog
         if: ${{ success() || failure() }}
         env:

diff --git a/.github/workflows/ci-console.yml b/.github/workflows/ci-console.yml
@@ -16,7 +16,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
       - name: Run linter
         run: yarn && yarn lint
       - name: Run frontend tests

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,6 +14,7 @@ jobs:
         tasks: [
           {alias: "unitTests", name: "check"},
           {alias: "integrationTests", name: "integrationTest"},
+          {alias: "slowIntegrationTests", name: "slowIntegrationTest"},
           {alias: "benchmark", name: "jmh -Pjmh.iterations=1 -Pjmh.timeOnIteration=5s -Pjmh.warmupIterations=0"}
         ]
       fail-fast: false

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -27,5 +27,5 @@ You can also use other *magic words* from [GitHub handbook](https://help.github.
 
 * use `spock` when writing new unit tests in all modules
 * when changing old tests use your best judgement as to when rewrite them to `spock`
-* use `TestNG` with defined environment in `integration` module
+* use `JUnit5` with defined environment in `integration-tests` module
 * prepend configuration options with module name, i.e. `frontend.` or `consumer.` when it applies to single module
diff --git a/build.gradle b/build.gradle
@@ -49,26 +49,27 @@ allprojects {
 
     project.ext.versions = [
             kafka             : '2.8.2',
-            guava             : '23.0',
-            jackson           : '2.15.2',
-            jersey            : '3.1.2',
-            jetty             : '11.0.15',
+            guava             : '33.1.0-jre',
+            jackson           : '2.17.0',
+            jersey            : '3.1.6',
+            jetty             : '12.0.8',
             curator           : '5.4.0',
-            dropwizard_metrics: '4.1.0',
-            micrometer_metrics: '1.11.1',
-            wiremock          : '3.0.1',
-            spock             : '2.4-M1-groovy-4.0',
-            groovy            : '4.0.12',
-            avro              : '1.9.1',
+            dropwizard_metrics: '4.2.25',
+            micrometer_metrics: '1.12.5',
+            wiremock          : '3.9.0',
+            spock             : '2.4-M4-groovy-4.0',
+            groovy            : '4.0.21',
+            avro              : '1.11.3',
             json2avro         : '0.2.14',
+            // TODO: newest version requires subject alternative name in a certificate during host verification, current test cert does not have a one
             okhttp            : '3.9.1',
-            undertow          : '2.0.29.Final',
-            spring_web        : '6.0.8',
-            failsafe          : '2.3.1',
-            junit_jupiter     : '5.8.2',
-            testcontainers    : '1.18.1',
-            spring            : '3.0.6',
-            assertj           : '3.22.0'
+            undertow          : '2.3.12.Final',
+            spring_web        : '6.1.6',
+            failsafe          : '2.4.4',
+            junit_jupiter     : '5.10.2',
+            testcontainers    : '1.19.8',
+            spring            : '3.2.4',
+            assertj           : '3.25.3'
     ]
 
     repositories {
@@ -77,15 +78,14 @@ allprojects {
 
     dependencies {
         implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.4'
-        implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
+        implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
 
         testImplementation group: 'junit', name: 'junit', version: '4.11'
         testImplementation group: 'com.tngtech.java', name: 'junit-dataprovider', version: '1.10.0'
         testImplementation group: 'pl.pragmatists', name: 'JUnitParams', version: '1.0.2'
-        testImplementation group: 'org.mockito', name: 'mockito-all', version: '1.9.5'
+        testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.11.0'
         testImplementation group: 'org.assertj', name: 'assertj-core', version: versions.assertj
-        testImplementation group: 'com.jayway.awaitility', name: 'awaitility', version: '1.6.1'
-        testImplementation group: 'com.googlecode.catch-exception', name: 'catch-exception', version: '1.2.0'
+        testImplementation group: 'org.awaitility', name: 'awaitility', version: '4.2.1'
 
         annotationProcessor group: 'org.springframework.boot', name: 'spring-boot-configuration-processor', version: versions.spring
     }
@@ -113,7 +113,7 @@ allprojects {
 }
 
 
-configure(subprojects - project(':integration')) {
+configure(subprojects - project(':integration-tests')) {
     apply plugin: 'jacoco'
     apply plugin: 'maven-publish'
     apply plugin: 'signing'
@@ -123,6 +123,8 @@ configure(subprojects - project(':integration')) {
         withSourcesJar()
     }
 
+    javadoc.options.addStringOption('Xdoclint:none', '-quiet')
+
     publishing {
         publications {
             mavenJava(MavenPublication) {

diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml
@@ -46,7 +46,7 @@
     -->
     <module name="LineLength">
         <property name="fileExtensions" value="java"/>
-        <property name="max" value="140"/>
+        <property name="max" value="400"/>
         <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
     </module>
 

diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml
@@ -36,6 +36,9 @@
     <suppress checks="VisibilityModifier"
               files="[/\\]src[/\\](test|integration)[/\\]java[/\\]"
             />
+    <suppress checks="VariableDeclarationUsageDistance"
+              files="[/\\]src[/\\](test|slowIntegrationTest)[/\\]java[/\\]"
+    />
 
     <!-- Build dirs -->
     <suppress checks="[a-zA-Z0-9]*"
@@ -44,4 +47,10 @@
     <!-- Classes repackaged from external libraries -->
     <suppress checks="[a-zA-Z0-9]*"
               files="(DirectBufferPool|LinkedHashSetBlockingQueue)\.java" />
+
+    <suppress checks="MethodName"
+              files="ChronicleMapMessageRepositoryTest\.java" />
+
+    <suppress checks="LocalVariableName"
+              files="ChronicleMapMessageRepositoryTest\.java" />
 </suppressions>
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -30,19 +30,6 @@ services:
       - kafka_data:/var/lib/kafka/data
       - kafka_secrets:/etc/kafka/secrets
 
-  graphite:
-    image: graphiteapp/graphite-statsd:1.1.3
-    ports:
-      - '2003-2004:2003-2004'
-      - '2023-2024:2023-2024'
-      - '8125:8125/udp'
-      - '8126:8126'
-      - '8082:80'
-    volumes:
-      - graphite_conf:/opt/graphite/conf
-      - graphite_data:/opt/graphite/storage
-      - statsd_data:/opt/statsd
-
   frontend:
     build:
       context: ../
@@ -52,7 +39,6 @@ services:
     depends_on:
       - zk
       - kafka
-      - graphite
 
   consumers:
     build:
@@ -61,7 +47,6 @@ services:
     depends_on:
       - zk
       - kafka
-      - graphite
 
   management:
     build:
@@ -72,7 +57,6 @@ services:
     depends_on:
       - zk
       - kafka
-      - graphite
 
   schema-registry:
     image: "confluentinc/cp-schema-registry:${CONFLUENT_IMAGES_TAG}"
@@ -87,9 +71,6 @@ services:
       - "8081:8081"
 
 volumes:
-  graphite_conf:
-  graphite_data:
-  statsd_data:
   zk_secrets:
   zk_data:
   zk_log:

diff --git a/docker/latest/consumers/consumers.yaml b/docker/latest/consumers/consumers.yaml
@@ -8,11 +8,6 @@ consumer:
     clusters:
       - datacenter: "dc"
         brokerList: "kafka:29092"
-  graphite:
-    host: "graphite"
-  metrics:
-    metric-registry:
-      graphiteReporterEnabled: true
   workload:
     consumerPerSubscription: 1
   schema:

diff --git a/docker/latest/frontend/frontend.yaml b/docker/latest/frontend/frontend.yaml
@@ -8,11 +8,6 @@ frontend:
     clusters:
       - datacenter: "dc"
         brokerList: "kafka:29092"
-  graphite:
-    host: "graphite"
-  metrics:
-    metric-registry:
-      graphiteReporterEnabled: true
   schema:
     cache:
       refreshAfterWrite: 1m

diff --git a/docker/latest/management/management.yaml b/docker/latest/management/management.yaml
@@ -17,11 +17,6 @@ kafka:
       connectionTimeout: 3000
       bootstrapKafkaServer: kafka:29092
 
-graphite:
-  client:
-    enabled: true
-    externalMonitoringUrl: graphite:8082
-
 server:
   port: 8090
 

diff --git a/docs/docs/configuration/buffer-persistence.md b/docs/docs/configuration/buffer-persistence.md
@@ -1,4 +1,4 @@
-# Publishing buffer persistence
+# Publishing buffer persistence [deprecated]
 
 Hermes Frontend API has option to register callbacks triggered during different phases of message lifetime:
 
@@ -15,7 +15,7 @@ to disk. Map structure is continuously persisted to disk, as it is stored in off
 
 When Hermes Frontend starts up it scans filesystem in search of existing persisted map. If found, it is read and any
 persisted events are sent to Message Store. This way recovering after crash is fully automatic. If Hermes process or
-server crashes, nothing is lost.
+server crashes, events that were flushed to disk are recovered. 
 
 There is additional protection against flooding subscribers with outdated events. When reading events from persisted
 storage, Hermes filters out messages older than N hours, where N is a system parameter and is set to 3 days by default.

diff --git a/docs/docs/configuration/consumers-tuning.md b/docs/docs/configuration/consumers-tuning.md
@@ -2,18 +2,18 @@
 
 ## HTTP Sender
 
-Option                                               | Description                                                 | Default value
----------------------------------------------------- | ----------------------------------------------------------- | -------------
-consumer.http-client.serial.http1.threadPoolSize                  | size of thread pool for sender threads (global)             | 30
-consumer.http-client.serial.http1.maxConnectionsPerDestination    | max connections per remote host                             | 100
+| Option                                                         | Description                                     | Default value |
+|----------------------------------------------------------------|-------------------------------------------------|---------------|
+| consumer.http-client.serial.http1.threadPoolSize               | size of thread pool for sender threads (global) | 30            |
+| consumer.http-client.serial.http1.maxConnectionsPerDestination | max connections per remote host                 | 100           |
 
 ## Consumers core
 
-Option                        | Description                                                              | Default value
------------------------------ | ------------------------------------------------------------------------ | -------------
-consumer.commit.offset.period | interval between committing offsets to Kafka                             | 60s
-consumer.threadPoolSize       | thread pool for threads involved in consuming, 1 thread per subscription | 500
-consumer.serialConsumer.inflightSize         | how many messages can be kept in send queue, per subscription            | 100
+| Option                               | Description                                                              | Default value |
+|--------------------------------------|--------------------------------------------------------------------------|---------------|
+| consumer.commit.offset.period        | interval between committing offsets to Kafka                             | 60s           |
+| consumer.threadPoolSize              | thread pool for threads involved in consuming, 1 thread per subscription | 500           |
+| consumer.serialConsumer.inflightSize | how many messages can be kept in send queue, per subscription            | 100           |
 
 ## Workload constraints management
 
@@ -26,10 +26,10 @@ subscriptions assigned to itself.
 
 These numbers can be configured:
 
-Option                                              | Description                               | Default value
---------------------------------------------------- | ----------------------------------------- | ---------------------
-consumer.workload.consumersPerSubscription          | Number of consumers to which the subscription will be assigned. If this value is greater than the number of available consumers, Hermes will assign the subscription to all available consumers. | 2
-consumer.workload.maxSubscriptionsPerConsumer       | The maximum number of subscriptions assigned to a single consumer. If all consumers have the maximum number of subscriptions assigned, a new subscription will not be activated until a new consumer is added or another subscription is unassigned. | 200
+| Option                                        | Description                                                                                                                                                                                                                                          | Default value |
+|-----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
+| consumer.workload.consumersPerSubscription    | Number of consumers to which the subscription will be assigned. If this value is greater than the number of available consumers, Hermes will assign the subscription to all available consumers.                                                     | 2             |
+| consumer.workload.maxSubscriptionsPerConsumer | The maximum number of subscriptions assigned to a single consumer. If all consumers have the maximum number of subscriptions assigned, a new subscription will not be activated until a new consumer is added or another subscription is unassigned. | 200           |
 
 Additionally, Hermes allows to configure the property `consumer.workload.consumersPerSubscription` for specific
 topics or subscriptions in the runtime via REST API.

diff --git a/docs/docs/configuration/metrics.md b/docs/docs/configuration/metrics.md
@@ -12,9 +12,9 @@ Option                                         | Description
 {modulePrefix}.metrics.prometheus.step         | The step size to use in computing windowed statistics | 60s
 {modulePrefix}.metrics.prometheus.descriptions | If meter descriptions should be sent to Prometheus    | true
 
-In order to be able to access basic metrics via Management API, it needs to be configured to reach VictoriaMetrics API:
+In order to be able to access basic metrics via Management API, it needs to be configured to reach Prometheus API:
 
 Option                                    | Description                                   | Default value
 ------------------------------------------|-----------------------------------------------| -------------
 prometheus.client.enabled                 | Should fetch external metrics from Prometheus | true
-prometheus.client.externalMonitoringUrl   | URI to VictoriaMetrics HTTP API               | http://localhost:18090
+prometheus.client.externalMonitoringUrl   | URI to Prometheus HTTP API                    | http://localhost:18090
diff --git a/docs/docs/overview/architecture.md b/docs/docs/overview/architecture.md
@@ -16,7 +16,7 @@ Hermes integrates with multiple systems, each having different role.
 
 * **Message Store** - stores and routes messages, current implementation: Kafka
 * **Metadata Store** - shared metadata storage for all Hermes modules, current implementation: Zookeeper
-* **Metrics Store** *[optional]* - stores metrics gathered by Hermes, current implementation: Graphite
+* **Metrics Store** *[optional]* - stores metrics gathered by Hermes, currently Hermes exposes metrics in Prometheus format
 * **Tracking Store** *[optional]* - stores tracking (message trace) information, current implementation: ElasticSearch
 
 ## Message flow

diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md
@@ -56,7 +56,7 @@ image: allegro/hermes-management:hermes-[specific version tag]
 ## Development
 
 The default `docker-compose` setup will start all hermes modules (consumers, frontend, management), together
-with its dependencies (Kafka, ZooKeeper, Graphite, Schema Registry). To run a specific module with gradle/IntelliJ,
+with its dependencies (Kafka, ZooKeeper, Schema Registry). To run a specific module with gradle/IntelliJ,
 just comment out the module in `services` section of the `docker-compose.yml` file, and start the java process locally:
 
 `./gradlew -p hermes-frontend run`
@@ -175,7 +175,6 @@ management:
     depends_on:
       - zk
       - kafka
-      - graphite
 [...]
 ```
 

diff --git a/docs/docs/user/java-client.md b/docs/docs/user/java-client.md
@@ -19,7 +19,7 @@ At the moment there are four implementations of `HermesSender`:
   for asynchronous transmission
 * **WebClientHermesSender** - for services using [Spring WebFlux](https://docs.spring.io/spring-framework/docs/current/reference/html/web-reactive.html); 
   uses [WebClient](https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/web/reactive/function/client/WebClient.html)
-* **JerseyHermesSender** - recommended for services using  [Jersey](<https://jersey.java.net/>)
+* **JerseyHermesSender** - recommended for services using  [Jersey](<https://eclipse-ee4j.github.io/jersey/>)
 * **OkHttpHermesSender** - supports both HTTP/1.1 and HTTP/2 protocols, uses [OkHttp3 client](http://square.github.io/okhttp/)