From 9b8756d7643224614b9f78d2b49e54de6faabd22 Mon Sep 17 00:00:00 2001 From: Jakub Scholz Date: Tue, 6 Feb 2024 14:10:36 +0100 Subject: [PATCH] Add basic Strimzi pom.xml file with the usual plugins, checkstyle, etc. (#10) Signed-off-by: Jakub Scholz --- .azure/build-pipeline.yaml | 63 ++++ .azure/scripts/build.sh | 31 ++ .azure/scripts/push-to-nexus.sh | 12 + .azure/scripts/settings.xml | 9 + .azure/scripts/setup-kubernetes.sh | 104 ++++++ .azure/templates/setup_java.yaml | 11 + .azure/templates/setup_minikube.yaml | 15 + .checkstyle/checkstyle.xml | 144 +++++++++ .checkstyle/suppressions.xml | 9 + .spotbugs/spotbugs-exclude.xml | 4 + pom.xml | 302 +++++++++++++++--- .../kafka/metrics/KafkaMetricsCollector.java | 11 +- .../kafka/metrics/YammerMetricsCollector.java | 3 +- .../KafkaPrometheusMetricsReporterTest.java | 2 - .../PrometheusMetricsReporterConfigTest.java | 7 +- 15 files changed, 674 insertions(+), 53 deletions(-) create mode 100644 .azure/build-pipeline.yaml create mode 100755 .azure/scripts/build.sh create mode 100755 .azure/scripts/push-to-nexus.sh create mode 100644 .azure/scripts/settings.xml create mode 100755 .azure/scripts/setup-kubernetes.sh create mode 100644 .azure/templates/setup_java.yaml create mode 100644 .azure/templates/setup_minikube.yaml create mode 100644 .checkstyle/checkstyle.xml create mode 100644 .checkstyle/suppressions.xml create mode 100644 .spotbugs/spotbugs-exclude.xml diff --git a/.azure/build-pipeline.yaml b/.azure/build-pipeline.yaml new file mode 100644 index 0000000..ffdcff9 --- /dev/null +++ b/.azure/build-pipeline.yaml @@ -0,0 +1,63 @@ +# Triggers +trigger: + branches: + include: + - 'main' + - 'release-*' + tags: + include: + - '*' +pr: + autoCancel: true + branches: + include: + - '*' +# Jobs +jobs: + - job: 'build_and_test' + displayName: 'Build & Test' + # Strategy for the job + strategy: + matrix: + 'java-11': + jdk_version: '11' + 'java-17': + jdk_version: '17' + # Set timeout for jobs + timeoutInMinutes: 60 + # Base system + pool: + vmImage: Ubuntu-22.04 + # Variables + variables: + MVN_CACHE_FOLDER: $(HOME)/.m2/repository + MVN_ARGS: '-e -V -B' + # Pipeline steps + steps: + - task: Cache@2 + inputs: + key: 'maven-cache | $(System.JobName) | **/pom.xml' + restoreKeys: | + maven-cache | $(System.JobName) + maven-cache + path: $(HOME)/.m2/repository + displayName: Maven cache + - template: 'templates/setup_java.yaml' + parameters: + JDK_VERSION: $(jdk_version) + - template: 'templates/setup_minikube.yaml' + - bash: ".azure/scripts/build.sh" + env: + BUILD_REASON: $(Build.Reason) + BRANCH: $(Build.SourceBranch) + GPG_PASSPHRASE: $(GPG_PASSPHRASE) + GPG_SIGNING_KEY: $(GPG_SIGNING_KEY) + NEXUS_USERNAME: $(NEXUS_USERNAME) + NEXUS_PASSWORD: $(NEXUS_PASSWORD) + displayName: "Build and test" + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + testResultsFiles: '**/TEST-*.xml' + testRunTitle: "Unit & Integration tests" + condition: always() diff --git a/.azure/scripts/build.sh b/.azure/scripts/build.sh new file mode 100755 index 0000000..7f9b9ba --- /dev/null +++ b/.azure/scripts/build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -e + +echo "Build reason: ${BUILD_REASON}" +echo "Source branch: ${BRANCH}" + +# The first segment of the version number is '1' for releases < 9; then '9', '10', '11', ... +JAVA_MAJOR_VERSION=$(java -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p') +if [ "${JAVA_MAJOR_VERSION}" -eq "11" ] ; then + # some parts of the workflow should be done only one on the main build which is currently Java 11 + export MAIN_BUILD="TRUE" + echo "Running main build" +fi + +# Build with Maven +# shellcheck disable=SC2086 +mvn $MVN_ARGS install +# shellcheck disable=SC2086 +mvn $MVN_ARGS spotbugs:check + +# Push to Nexus +if [ "$BUILD_REASON" == "PullRequest" ] ; then + echo "Building Pull Request - nothing to push" +elif [[ "$BRANCH" != "refs/tags/"* ]] && [ "$BRANCH" != "refs/heads/main" ]; then + echo "Not in main branch or in release tag - nothing to push" +else + if [ "${MAIN_BUILD}" = "TRUE" ] ; then + echo "In main branch or in release tag - pushing to nexus" + ./.azure/scripts/push-to-nexus.sh + fi +fi diff --git a/.azure/scripts/push-to-nexus.sh b/.azure/scripts/push-to-nexus.sh new file mode 100755 index 0000000..ce5edc7 --- /dev/null +++ b/.azure/scripts/push-to-nexus.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +export GPG_TTY=$(tty) + +echo $GPG_SIGNING_KEY | base64 -d > signing.gpg +gpg --batch --import signing.gpg + +GPG_EXECUTABLE=gpg mvn $MVN_ARGS -DskipTests -s ./.azure/scripts/settings.xml -P ossrh verify deploy + +rm -rf signing.gpg +gpg --delete-keys +gpg --delete-secret-keys \ No newline at end of file diff --git a/.azure/scripts/settings.xml b/.azure/scripts/settings.xml new file mode 100644 index 0000000..f32f174 --- /dev/null +++ b/.azure/scripts/settings.xml @@ -0,0 +1,9 @@ + + + + ossrh + ${env.NEXUS_USERNAME} + ${env.NEXUS_PASSWORD} + + + \ No newline at end of file diff --git a/.azure/scripts/setup-kubernetes.sh b/.azure/scripts/setup-kubernetes.sh new file mode 100755 index 0000000..2a9598b --- /dev/null +++ b/.azure/scripts/setup-kubernetes.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -xe + +rm -rf ~/.kube + +KUBE_VERSION=${KUBE_VERSION:-1.16.0} +COPY_DOCKER_LOGIN=${COPY_DOCKER_LOGIN:-"false"} + +DEFAULT_MINIKUBE_MEMORY=$(free -m | grep "Mem" | awk '{print $2}') +DEFAULT_MINIKUBE_CPU=$(awk '$1~/cpu[0-9]/{usage=($2+$4)*100/($2+$4+$5); print $1": "usage"%"}' /proc/stat | wc -l) + +MINIKUBE_MEMORY=${MINIKUBE_MEMORY:-$DEFAULT_MINIKUBE_MEMORY} +MINIKUBE_CPU=${MINIKUBE_CPU:-$DEFAULT_MINIKUBE_CPU} + +echo "[INFO] MINIKUBE_MEMORY: ${MINIKUBE_MEMORY}" +echo "[INFO] MINIKUBE_CPU: ${MINIKUBE_CPU}" + +function install_kubectl { + if [ "${KUBECTL_VERSION:-latest}" = "latest" ]; then + KUBECTL_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) + fi + curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl + sudo cp kubectl /usr/local/bin +} + +function install_nsenter { + # Pre-req for helm + curl https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v${NSENTER_VERSION}/util-linux-${NSENTER_VERSION}.tar.gz -k | tar -zxf- + cd util-linux-${NSENTER_VERSION} + ./configure --without-ncurses + make nsenter + sudo cp nsenter /usr/bin +} + +function label_node { + # It should work for all clusters + for nodeName in $(kubectl get nodes -o custom-columns=:.metadata.name --no-headers); + do + echo ${nodeName}; + kubectl label node ${nodeName} rack-key=zone; + done +} + +if [ "$KUBE_CLUSTER" = "minikube" ]; then + install_kubectl + if [ "${MINIKUBE_VERSION:-latest}" = "latest" ]; then + MINIKUBE_URL=https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + else + MINIKUBE_URL=https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 + fi + + if [ "$KUBE_VERSION" != "latest" ] && [ "$KUBE_VERSION" != "stable" ]; then + KUBE_VERSION="v${KUBE_VERSION}" + fi + + curl -Lo minikube ${MINIKUBE_URL} && chmod +x minikube + sudo cp minikube /usr/bin + + export MINIKUBE_WANTUPDATENOTIFICATION=false + export MINIKUBE_WANTREPORTERRORPROMPT=false + export MINIKUBE_HOME=$HOME + export CHANGE_MINIKUBE_NONE_USER=true + + mkdir $HOME/.kube || true + touch $HOME/.kube/config + + docker run -d -p 5000:5000 registry + + export KUBECONFIG=$HOME/.kube/config + # We can turn on network polices support by adding the following options --network-plugin=cni --cni=calico + # We have to allow trafic for ITS when NPs are turned on + # We can allow NP after Strimzi#4092 which should fix some issues on STs side + minikube start --vm-driver=docker --kubernetes-version=${KUBE_VERSION} \ + --insecure-registry=localhost:5000 --extra-config=apiserver.authorization-mode=Node,RBAC \ + --cpus=${MINIKUBE_CPU} --memory=${MINIKUBE_MEMORY} + + if [ $? -ne 0 ] + then + echo "Minikube failed to start or RBAC could not be properly set up" + exit 1 + fi + + minikube addons enable default-storageclass + + # Add Docker hub credentials to Minikube + if [ "$COPY_DOCKER_LOGIN" = "true" ] + then + set +ex + + docker exec "minikube" bash -c "echo '$(cat $HOME/.docker/config.json)'| sudo tee -a /var/lib/kubelet/config.json > /dev/null && sudo systemctl restart kubelet" + + set -ex + fi + + minikube addons enable registry + minikube addons enable registry-aliases + + kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default +else + echo "Unsupported KUBE_CLUSTER '$KUBE_CLUSTER'" + exit 1 +fi + +label_node diff --git a/.azure/templates/setup_java.yaml b/.azure/templates/setup_java.yaml new file mode 100644 index 0000000..ca43101 --- /dev/null +++ b/.azure/templates/setup_java.yaml @@ -0,0 +1,11 @@ +# Step to configure JAVA on the agent +parameters: + - name: JDK_VERSION + default: '17' +steps: + - task: JavaToolInstaller@0 + inputs: + versionSpec: $(JDK_VERSION) + jdkArchitectureOption: 'x64' + jdkSourceOption: 'PreInstalled' + displayName: 'Configure Java' diff --git a/.azure/templates/setup_minikube.yaml b/.azure/templates/setup_minikube.yaml new file mode 100644 index 0000000..4f25f33 --- /dev/null +++ b/.azure/templates/setup_minikube.yaml @@ -0,0 +1,15 @@ +# Steps needed for local Minikube instance +steps: +- task: DockerInstaller@0 + displayName: Docker Installer + inputs: + dockerVersion: 19.03.9 + releaseType: stable +- bash: ".azure/scripts/setup-kubernetes.sh" + displayName: "Setup Minikube cluster" + env: + KUBE_CLUSTER: 'minikube' + KUBE_VERSION: 'latest' + KUBECTL_VERSION: 'latest' + MINIKUBE_VERSION: 'latest' + NSENTER_VERSION: '2.32' \ No newline at end of file diff --git a/.checkstyle/checkstyle.xml b/.checkstyle/checkstyle.xml new file mode 100644 index 0000000..492023a --- /dev/null +++ b/.checkstyle/checkstyle.xml @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.checkstyle/suppressions.xml b/.checkstyle/suppressions.xml new file mode 100644 index 0000000..a66f167 --- /dev/null +++ b/.checkstyle/suppressions.xml @@ -0,0 +1,9 @@ + + + + + + + \ No newline at end of file diff --git a/.spotbugs/spotbugs-exclude.xml b/.spotbugs/spotbugs-exclude.xml new file mode 100644 index 0000000..4c1cc70 --- /dev/null +++ b/.spotbugs/spotbugs-exclude.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 51eba8c..7ae22d1 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,69 @@ https://github.com/strimzi/metrics-reporter/issues + + + Tom Bentley + tbentley@redhat.com + Red Hat + https://www.redhat.com + + + Paolo Patierno + ppatierno@live.com + Red Hat + https://www.redhat.com + + + Jakub Scholz + github@scholzj.com + Red Hat + https://www.redhat.com + + + Sam Hawker + sam.b.hawker@gmail.com + IBM + https://www.ibm.com + + + Jakub Stejskal + xstejs24@gmail.com + Red Hat + https://www.redhat.com + + + Stanislav Knot + knot@cngroup.dk + CN Group + https://www.cngroup.dk + + + Paul Mellor + pmellor@redhat.com + Red Hat + https://www.redhat.com + + + Lukáš Král + l.kral@outlook.com + Red Hat + https://www.redhat.com + + + Maroš Orsák + maros.orsak159@gmail.com + Red Hat + https://www.redhat.com + + + Mickael Maison + mickael.maison@gmail.com + Red Hat + https://www.redhat.com + + + The Apache License, Version 2.0 @@ -34,6 +97,20 @@ UTF-8 11 11 + + 3.3.1 + 4.8.3.0 + 4.8.3 + 3.2.5 + 3.2.5 + 3.6.3 + 3.6.1 + 3.2.1 + 3.3.0 + + 3.1.0 + 1.6.13 + 3.6.1 0.16.0 2.2.0 @@ -41,6 +118,17 @@ 5.10.1 + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + org.apache.kafka @@ -98,46 +186,176 @@ - - - - - maven-clean-plugin - 3.1.0 - - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - 11 - 11 - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.1.1 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven.checkstyle.version} + + + validate + validate + + .checkstyle/checkstyle.xml + .checkstyle/java.header + .checkstyle/suppressions.xml + true + true + true + + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven.surefire.version} + + + org.apache.maven.plugins + maven-failsafe-plugin + ${maven.failsafe.version} + + + + integration-test + verify + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${maven.spotbugs.version} + + Max + + Low + + true + + ${project.build.directory}/spotbugs + + ${project.basedir}/.spotbugs/spotbugs-exclude.xml + + + + org.apache.maven.plugins + maven-dependency-plugin + ${maven.dependency.version} + + + analyze + + analyze-only + + + true + + + + + + org.apache.maven.plugins + maven-source-plugin + ${maven.source.version} + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven.javadoc.version} + + + attach-javadocs + + jar + + + public + true + false + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.version} + + + + true + true + + + + + - + + + + ossrh + + false + + + + ${env.GPG_EXECUTABLE} + + ${env.GPG_PASSPHRASE} + + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven.gpg.version} + + + sign-artifacts + verify + + sign + + + + --batch + --pinentry-mode + loopback + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${sonatype.nexus.staging} + true + + ossrh + https://oss.sonatype.org/ + false + + + + + + + \ No newline at end of file diff --git a/src/main/java/io/strimzi/kafka/metrics/KafkaMetricsCollector.java b/src/main/java/io/strimzi/kafka/metrics/KafkaMetricsCollector.java index eabfd30..faad5e3 100644 --- a/src/main/java/io/strimzi/kafka/metrics/KafkaMetricsCollector.java +++ b/src/main/java/io/strimzi/kafka/metrics/KafkaMetricsCollector.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -74,15 +75,15 @@ String metricName(MetricName metricName) { String prefix = this.prefix .replace('.', '_') .replace('-', '_') - .toLowerCase(); + .toLowerCase(Locale.ROOT); String group = metricName.group() .replace('.', '_') .replace('-', '_') - .toLowerCase(); + .toLowerCase(Locale.ROOT); String name = metricName.name() .replace('.', '_') .replace('-', '_') - .toLowerCase(); + .toLowerCase(Locale.ROOT); return prefix + '_' + group + '_' + name; } @@ -98,7 +99,9 @@ static MetricFamilySamples convert(String name, String help, KafkaMetric metric, .collect(Collectors.toMap( e -> Collector.sanitizeMetricName(e.getKey()), Map.Entry::getValue, - (v1, v2) -> { throw new IllegalStateException("Unexpected duplicate key " + v1); }, + (v1, v2) -> { + throw new IllegalStateException("Unexpected duplicate key " + v1); + }, LinkedHashMap::new)); return new MetricFamilySamplesBuilder(Type.GAUGE, help) .addSample(name, ((Number) value).doubleValue(), sanitizedLabels) diff --git a/src/main/java/io/strimzi/kafka/metrics/YammerMetricsCollector.java b/src/main/java/io/strimzi/kafka/metrics/YammerMetricsCollector.java index 464f5cb..fd573ad 100644 --- a/src/main/java/io/strimzi/kafka/metrics/YammerMetricsCollector.java +++ b/src/main/java/io/strimzi/kafka/metrics/YammerMetricsCollector.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; /** @@ -89,7 +90,7 @@ static String metricName(MetricName metricName) { "kafka_server_" + metricName.getGroup() + '_' + metricName.getType() + '_' + - metricName.getName()).toLowerCase(); + metricName.getName()).toLowerCase(Locale.ROOT); LOG.info("metricName group {}, type {}, name {} converted into {}", metricName.getGroup(), metricName.getType(), metricName.getName(), metricNameStr); return metricNameStr; } diff --git a/src/test/java/io/strimzi/kafka/metrics/KafkaPrometheusMetricsReporterTest.java b/src/test/java/io/strimzi/kafka/metrics/KafkaPrometheusMetricsReporterTest.java index 0f1a266..268e613 100644 --- a/src/test/java/io/strimzi/kafka/metrics/KafkaPrometheusMetricsReporterTest.java +++ b/src/test/java/io/strimzi/kafka/metrics/KafkaPrometheusMetricsReporterTest.java @@ -28,9 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; - public class KafkaPrometheusMetricsReporterTest { - private final MetricConfig metricConfig = new MetricConfig(); private final Time time = Time.SYSTEM; private final Map labels = Collections.singletonMap("key", "value"); diff --git a/src/test/java/io/strimzi/kafka/metrics/PrometheusMetricsReporterConfigTest.java b/src/test/java/io/strimzi/kafka/metrics/PrometheusMetricsReporterConfigTest.java index 0eba5e9..86a34fd 100644 --- a/src/test/java/io/strimzi/kafka/metrics/PrometheusMetricsReporterConfigTest.java +++ b/src/test/java/io/strimzi/kafka/metrics/PrometheusMetricsReporterConfigTest.java @@ -4,18 +4,17 @@ */ package io.strimzi.kafka.metrics; - import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import static org.junit.jupiter.api.Assertions.*; - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class PrometheusMetricsReporterConfigTest { - @Test public void testDefaults() { PrometheusMetricsReporterConfig config = new PrometheusMetricsReporterConfig(Collections.emptyMap());