From e0d3fe859fa0b3dcb3589aadfe352a808f005cc5 Mon Sep 17 00:00:00 2001 From: Ishwar Kanse Date: Wed, 5 Jul 2023 16:45:35 +0530 Subject: [PATCH] Add functional test for OTLP metrics collection and forward to prometheus (#1850) Signed-off-by: Ishwar Kanse Co-authored-by: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> --- CONTRIBUTING.md | 9 ++++ Makefile | 5 ++ kuttl-test-openshift.yaml | 6 +++ .../otlp-metrics-traces/00-assert.yaml | 10 ++++ .../00-install-jaeger.yaml | 18 +++++++ .../otlp-metrics-traces/01-assert.yaml | 22 ++++++++ .../01-workload-monitoring.yaml | 11 ++++ .../otlp-metrics-traces/02-assert.yaml | 9 ++++ .../02-otel-metrics-collector.yaml | 39 ++++++++++++++ .../otlp-metrics-traces/03-assert.yaml | 25 +++++++++ .../otlp-metrics-traces/03-podmonitor.yaml | 26 ++++++++++ .../otlp-metrics-traces/04-assert.yaml | 22 ++++++++ .../04-metrics-traces-gen.yaml | 52 +++++++++++++++++++ .../otlp-metrics-traces/05-assert.yaml | 6 +++ .../otlp-metrics-traces/06-assert.yaml | 6 +++ .../otlp-metrics-traces/check_metrics.sh | 15 ++++++ .../otlp-metrics-traces/check_traces.sh | 12 +++++ 17 files changed, 293 insertions(+) create mode 100644 kuttl-test-openshift.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/00-assert.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/01-assert.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/01-workload-monitoring.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/02-assert.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/02-otel-metrics-collector.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/03-assert.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/03-podmonitor.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/04-assert.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/04-metrics-traces-gen.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/05-assert.yaml create mode 100644 tests/e2e-openshift/otlp-metrics-traces/06-assert.yaml create mode 100755 tests/e2e-openshift/otlp-metrics-traces/check_metrics.sh create mode 100755 tests/e2e-openshift/otlp-metrics-traces/check_traces.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7cc2550dac..ed869bb3f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,6 +107,15 @@ Once they are installed, the tests can be executed with `make prepare-e2e`, whic The tests are located under `tests/e2e` and are written to be used with `kuttl`. Refer to their documentation to understand how tests are written. +### OpenShift End to End tests +To run the end-to-end tests written for OpenShift, you'll need a OpenShift cluster. + +To install the OpenTelemetry operator, please follow the instructions in [Operator Lifecycle Manager (OLM)](https://github.com/open-telemetry/opentelemetry-operator/blob/main/CONTRIBUTING.md#operator-lifecycle-manager-olm) + +Once the operator is installed, the tests can be executed using `make e2e-openshift`, which will call to the `e2e-openshift` target. Note that `kind` is disabled for the TestSuite as the requirement is to use an OpenShift cluster for these test cases. + +The tests are located under `tests/e2e-openshift` and are written to be used with `kuttl`. + ### Undeploying the operator from the local cluster ```bash diff --git a/Makefile b/Makefile index a6657d4064..7afc850010 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,11 @@ e2e-upgrade: undeploy e2e-autoscale: $(KUTTL) test --config kuttl-test-autoscale.yaml +# end-to-end-test for testing OpenShift cases +.PHONY: e2e-openshift +e2e-openshift: + $(KUTTL) test --config kuttl-test-openshift.yaml + .PHONY: e2e-log-operator e2e-log-operator: kubectl get pod -n opentelemetry-operator-system | grep "opentelemetry-operator" | awk '{print $$1}' | xargs -I {} kubectl logs -n opentelemetry-operator-system {} manager diff --git a/kuttl-test-openshift.yaml b/kuttl-test-openshift.yaml new file mode 100644 index 0000000000..4185980835 --- /dev/null +++ b/kuttl-test-openshift.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +startKIND: false +testDirs: + - ./tests/e2e-openshift/ +timeout: 150 diff --git a/tests/e2e-openshift/otlp-metrics-traces/00-assert.yaml b/tests/e2e-openshift/otlp-metrics-traces/00-assert.yaml new file mode 100644 index 0000000000..ef885ad07c --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/00-assert.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: kuttl-otlp-metrics + name: jaeger-allinone +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 diff --git a/tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml b/tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml new file mode 100644 index 0000000000..a941089d4b --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml @@ -0,0 +1,18 @@ +#Create the "kuttl-otlp-metrics" project to fulfill the requirement of specifying the Jaeger and OTEL collector endpoints. This project is essential due to KUTTL's current lack of support for templating. +#For this test case you'll need to install the Jaeger operator (OpenShift Distributed Tracing Platform in OpenShift) + +apiVersion: v1 +kind: Namespace +metadata: + name: kuttl-otlp-metrics + +--- +apiVersion: jaegertracing.io/v1 +kind: Jaeger +metadata: + name: jaeger-allinone + namespace: kuttl-otlp-metrics +spec: + strategy: allinone + ingress: + security: none diff --git a/tests/e2e-openshift/otlp-metrics-traces/01-assert.yaml b/tests/e2e-openshift/otlp-metrics-traces/01-assert.yaml new file mode 100644 index 0000000000..268aa391d1 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/01-assert.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: openshift-user-workload-monitoring +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + namespace: openshift-user-workload-monitoring +status: + availableReplicas: 2 + currentReplicas: 2 + readyReplicas: 2 + replicas: 2 + updatedReplicas: 2 + diff --git a/tests/e2e-openshift/otlp-metrics-traces/01-workload-monitoring.yaml b/tests/e2e-openshift/otlp-metrics-traces/01-workload-monitoring.yaml new file mode 100644 index 0000000000..1d5c5e4b14 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/01-workload-monitoring.yaml @@ -0,0 +1,11 @@ +# oc -n openshift-user-workload-monitoring get pod +# https://docs.openshift.com/container-platform/4.13/monitoring/enabling-monitoring-for-user-defined-projects.html#accessing-metrics-from-outside-cluster_enabling-monitoring-for-user-defined-projects + +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true diff --git a/tests/e2e-openshift/otlp-metrics-traces/02-assert.yaml b/tests/e2e-openshift/otlp-metrics-traces/02-assert.yaml new file mode 100644 index 0000000000..2413e22839 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/02-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: kuttl-otlp-metrics +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 diff --git a/tests/e2e-openshift/otlp-metrics-traces/02-otel-metrics-collector.yaml b/tests/e2e-openshift/otlp-metrics-traces/02-otel-metrics-collector.yaml new file mode 100644 index 0000000000..83257f98a6 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/02-otel-metrics-collector.yaml @@ -0,0 +1,39 @@ +#https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/prometheusexporter/README.md + +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: cluster-collector + namespace: kuttl-otlp-metrics +spec: + mode: deployment + ports: + - name: promexporter + port: 8889 + protocol: TCP + config: | + receivers: + otlp: + protocols: + grpc: + http: + processors: + exporters: + jaeger: + endpoint: jaeger-allinone-collector-headless.kuttl-otlp-metrics.svc:14250 + tls: + ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true # by default resource attributes are dropped + service: + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [jaeger] + metrics: + receivers: [otlp] + processors: [] + exporters: [prometheus] diff --git a/tests/e2e-openshift/otlp-metrics-traces/03-assert.yaml b/tests/e2e-openshift/otlp-metrics-traces/03-assert.yaml new file mode 100644 index 0000000000..faf3c91c8e --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/03-assert.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: otel-collector + namespace: kuttl-otlp-metrics +spec: + podMetricsEndpoints: + - port: metrics + - metricRelabelings: + - action: labeldrop + regex: instance + - action: labeldrop + regex: job + port: promexporter + relabelings: + - action: labeldrop + regex: pod + - action: labeldrop + regex: container + - action: labeldrop + regex: endpoint + selector: + matchLabels: + app.kubernetes.io/name: cluster-collector-collector diff --git a/tests/e2e-openshift/otlp-metrics-traces/03-podmonitor.yaml b/tests/e2e-openshift/otlp-metrics-traces/03-podmonitor.yaml new file mode 100644 index 0000000000..b0eaf43637 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/03-podmonitor.yaml @@ -0,0 +1,26 @@ +#https://docs.openshift.com/container-platform/4.13/monitoring/managing-metrics.html +#https://docs.openshift.com/container-platform/4.13/rest_api/monitoring_apis/podmonitor-monitoring-coreos-com-v1.html +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: otel-collector + namespace: kuttl-otlp-metrics +spec: + selector: + matchLabels: + app.kubernetes.io/name: cluster-collector-collector + podMetricsEndpoints: + - port: metrics + - port: promexporter + relabelings: + - action: labeldrop + regex: pod + - action: labeldrop + regex: container + - action: labeldrop + regex: endpoint + metricRelabelings: + - action: labeldrop + regex: instance + - action: labeldrop + regex: job diff --git a/tests/e2e-openshift/otlp-metrics-traces/04-assert.yaml b/tests/e2e-openshift/otlp-metrics-traces/04-assert.yaml new file mode 100644 index 0000000000..73bbb738a9 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/04-assert.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: telemetrygen-traces +status: + active: 1 + ready: 1 + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: telemetrygen-metrics +status: + active: 1 + ready: 1 + +--- +apiVersion: v1 +kind: Pod +status: + phase: Running diff --git a/tests/e2e-openshift/otlp-metrics-traces/04-metrics-traces-gen.yaml b/tests/e2e-openshift/otlp-metrics-traces/04-metrics-traces-gen.yaml new file mode 100644 index 0000000000..441b8877aa --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/04-metrics-traces-gen.yaml @@ -0,0 +1,52 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: telemetrygen-traces +spec: + completions: 1 + parallelism: 1 + template: + metadata: + labels: + app: telemetrygen-traces + spec: + containers: + - name: telemetrygen-traces + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest + command: ["./telemetrygen"] + args: + - "--otlp-endpoint=cluster-collector-collector-headless.kuttl-otlp-metrics.svc:4317" + - "--otlp-insecure=true" + - "--rate=1" + - "--duration=30s" + - "--otlp-attributes=telemetrygen=\"traces\"" + - "--otlp-header=telemetrygen=\"traces\"" + - "traces" + restartPolicy: Never + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: telemetrygen-metrics +spec: + completions: 1 + parallelism: 1 + template: + metadata: + labels: + app: telemetrygen-metrics + spec: + containers: + - name: telemetrygen-metrics + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest + command: ["./telemetrygen"] + args: + - "--otlp-endpoint=cluster-collector-collector-headless.kuttl-otlp-metrics.svc:4317" + - "--otlp-insecure=true" + - "--duration=30s" + - "--rate=1" + - "--otlp-attributes=telemetrygen=\"metrics\"" + - "--otlp-header=telemetrygen=\"traces\"" + - "metrics" + restartPolicy: Never diff --git a/tests/e2e-openshift/otlp-metrics-traces/05-assert.yaml b/tests/e2e-openshift/otlp-metrics-traces/05-assert.yaml new file mode 100644 index 0000000000..8dcf1e6431 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/05-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +commands: +- script: ./tests/e2e-openshift/otlp-metrics/check_traces.sh diff --git a/tests/e2e-openshift/otlp-metrics-traces/06-assert.yaml b/tests/e2e-openshift/otlp-metrics-traces/06-assert.yaml new file mode 100644 index 0000000000..63d8d22c40 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/06-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +commands: +- script: ./tests/e2e-openshift/otlp-metrics/check_metrics.sh diff --git a/tests/e2e-openshift/otlp-metrics-traces/check_metrics.sh b/tests/e2e-openshift/otlp-metrics-traces/check_metrics.sh new file mode 100755 index 0000000000..dd44adc871 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/check_metrics.sh @@ -0,0 +1,15 @@ +#!/bin/bash +SECRET=$(oc get secret -n openshift-user-workload-monitoring | grep prometheus-user-workload-token | head -n 1 | awk '{print $1}') +TOKEN=$(echo $(oc get secret $SECRET -n openshift-user-workload-monitoring -o json | jq -r '.data.token') | base64 -d) +THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host') + +response=$(curl -k -H "Authorization: Bearer $TOKEN" -H "Content-type: application/json" "https://$THANOS_QUERIER_HOST/api/v1/query?query=gen") + +count=$(echo "$response" | jq -r '.data.result | length') + +if [[ $count -eq 0 ]]; then + echo "No telemetrygen metrics count with value present. Exiting with status 1." + exit 1 +else + echo "telemetrygen metrics with value is present." +fi diff --git a/tests/e2e-openshift/otlp-metrics-traces/check_traces.sh b/tests/e2e-openshift/otlp-metrics-traces/check_traces.sh new file mode 100755 index 0000000000..9761284aee --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/check_traces.sh @@ -0,0 +1,12 @@ +#!/bin/bash +JAEGER_URL=$(oc -n kuttl-otlp-metrics get route jaeger-allinone -o json | jq '.spec.host' -r) +SERVICE_NAME="telemetrygen" + +trace_exists=$(curl -ksSL "https://$JAEGER_URL/api/traces?service=$SERVICE_NAME&limit=1" | jq -r '.data | length') + +if [[ $trace_exists -gt 0 ]]; then + echo "Traces for $SERVICE_NAME exist in Jaeger." +else + echo "Trace for $SERVICE_NAME does not exist in Jaeger." + exit 1 # Fail the test step if the trace doesn't exist +fi