Add functional test for OTLP metrics collection and forward to promet…

…heus (#1850)

Signed-off-by: Ishwar Kanse <[email protected]>
Co-authored-by: Tyler Helmuth <[email protected]>

CONTRIBUTING.md

@@ -107,6 +107,15 @@ Once they are installed, the tests can be executed with `make prepare-e2e`, whic
 
 The tests are located under `tests/e2e` and are written to be used with `kuttl`. Refer to their documentation to understand how tests are written.
 
+### OpenShift End to End tests
+To run the end-to-end tests written for OpenShift, you'll need a OpenShift cluster. 
+
+To install the OpenTelemetry operator, please follow the instructions in  [Operator Lifecycle Manager (OLM)](https://github.com/open-telemetry/opentelemetry-operator/blob/main/CONTRIBUTING.md#operator-lifecycle-manager-olm)
+
+Once the operator is installed, the tests can be executed using `make e2e-openshift`, which will call to the `e2e-openshift` target. Note that `kind` is disabled for the TestSuite as the requirement is to use an OpenShift cluster for these test cases. 
+
+The tests are located under `tests/e2e-openshift` and are written to be used with `kuttl`.
+
 ### Undeploying the operator from the local cluster
 
 ```bash

Makefile

@@ -186,6 +186,11 @@ e2e-upgrade: undeploy
 e2e-autoscale:
 	$(KUTTL) test --config kuttl-test-autoscale.yaml
 
+# end-to-end-test for testing OpenShift cases
+.PHONY: e2e-openshift
+e2e-openshift:
+	$(KUTTL) test --config kuttl-test-openshift.yaml
+
 .PHONY: e2e-log-operator
 e2e-log-operator:
 	kubectl get pod -n opentelemetry-operator-system | grep "opentelemetry-operator" | awk '{print $$1}' | xargs -I {} kubectl logs -n opentelemetry-operator-system {} manager

kuttl-test-openshift.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+startKIND: false
+testDirs:
+  - ./tests/e2e-openshift/
+timeout: 150

tests/e2e-openshift/otlp-metrics-traces/00-assert.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: kuttl-otlp-metrics
+  name: jaeger-allinone
+status:
+  availableReplicas: 1
+  readyReplicas: 1
+  replicas: 1

tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml

@@ -0,0 +1,18 @@
+#Create the "kuttl-otlp-metrics" project to fulfill the requirement of specifying the Jaeger and OTEL collector endpoints. This project is essential due to KUTTL's current lack of support for templating.
+#For this test case you'll need to install the Jaeger operator (OpenShift Distributed Tracing Platform in OpenShift)
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kuttl-otlp-metrics
+
+---
+apiVersion: jaegertracing.io/v1
+kind: Jaeger
+metadata:
+  name: jaeger-allinone
+  namespace: kuttl-otlp-metrics
+spec:
+  strategy: allinone
+  ingress:
+    security: none

tests/e2e-openshift/otlp-metrics-traces/01-assert.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: openshift-user-workload-monitoring
+status:
+  availableReplicas: 1
+  readyReplicas: 1
+  replicas: 1
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  namespace: openshift-user-workload-monitoring
+status:
+  availableReplicas: 2
+  currentReplicas: 2
+  readyReplicas: 2
+  replicas: 2
+  updatedReplicas: 2
+

tests/e2e-openshift/otlp-metrics-traces/01-workload-monitoring.yaml

@@ -0,0 +1,11 @@
+# oc -n openshift-user-workload-monitoring get pod
+# https://docs.openshift.com/container-platform/4.13/monitoring/enabling-monitoring-for-user-defined-projects.html#accessing-metrics-from-outside-cluster_enabling-monitoring-for-user-defined-projects
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-monitoring-config
+  namespace: openshift-monitoring
+data:
+  config.yaml: |
+    enableUserWorkload: true 

tests/e2e-openshift/otlp-metrics-traces/02-assert.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: kuttl-otlp-metrics
+status:
+  availableReplicas: 1
+  readyReplicas: 1
+  replicas: 1

tests/e2e-openshift/otlp-metrics-traces/02-otel-metrics-collector.yaml

@@ -0,0 +1,39 @@
+#https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/prometheusexporter/README.md
+
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: cluster-collector
+  namespace: kuttl-otlp-metrics
+spec:
+  mode: deployment
+  ports:
+  - name: promexporter
+    port: 8889
+    protocol: TCP
+  config: |
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+          http:
+    processors:
+    exporters:
+      jaeger:
+        endpoint: jaeger-allinone-collector-headless.kuttl-otlp-metrics.svc:14250
+        tls:
+          ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
+      prometheus:
+        endpoint: 0.0.0.0:8889
+        resource_to_telemetry_conversion:
+          enabled: true # by default resource attributes are dropped
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: []
+          exporters: [jaeger]
+        metrics:
+          receivers: [otlp]
+          processors: []
+          exporters: [prometheus]

tests/e2e-openshift/otlp-metrics-traces/03-assert.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: otel-collector
+  namespace: kuttl-otlp-metrics
+spec:
+  podMetricsEndpoints:
+  - port: metrics
+  - metricRelabelings:
+    - action: labeldrop
+      regex: instance
+    - action: labeldrop
+      regex: job
+    port: promexporter
+    relabelings:
+    - action: labeldrop
+      regex: pod
+    - action: labeldrop
+      regex: container
+    - action: labeldrop
+      regex: endpoint
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cluster-collector-collector

tests/e2e-openshift/otlp-metrics-traces/03-podmonitor.yaml

@@ -0,0 +1,26 @@
+#https://docs.openshift.com/container-platform/4.13/monitoring/managing-metrics.html
+#https://docs.openshift.com/container-platform/4.13/rest_api/monitoring_apis/podmonitor-monitoring-coreos-com-v1.html
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: otel-collector
+  namespace: kuttl-otlp-metrics
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cluster-collector-collector
+  podMetricsEndpoints:
+  - port: metrics
+  - port: promexporter
+    relabelings:
+    - action: labeldrop
+      regex: pod
+    - action: labeldrop
+      regex: container
+    - action: labeldrop
+      regex: endpoint
+    metricRelabelings:
+    - action: labeldrop
+      regex: instance
+    - action: labeldrop
+      regex: job

tests/e2e-openshift/otlp-metrics-traces/04-assert.yaml

@@ -0,0 +1,22 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: telemetrygen-traces
+status:
+  active: 1
+  ready: 1
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: telemetrygen-metrics
+status:
+  active: 1
+  ready: 1
+
+---
+apiVersion: v1
+kind: Pod
+status:
+  phase: Running

tests/e2e-openshift/otlp-metrics-traces/04-metrics-traces-gen.yaml

@@ -0,0 +1,52 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: telemetrygen-traces
+spec:
+  completions: 1
+  parallelism: 1
+  template:
+    metadata:
+      labels:
+        app: telemetrygen-traces
+    spec:
+      containers:
+        - name: telemetrygen-traces
+          image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest
+          command: ["./telemetrygen"]
+          args:
+            - "--otlp-endpoint=cluster-collector-collector-headless.kuttl-otlp-metrics.svc:4317"
+            - "--otlp-insecure=true"
+            - "--rate=1"
+            - "--duration=30s"
+            - "--otlp-attributes=telemetrygen=\"traces\""
+            - "--otlp-header=telemetrygen=\"traces\""
+            - "traces"
+      restartPolicy: Never
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: telemetrygen-metrics
+spec:
+  completions: 1
+  parallelism: 1
+  template:
+    metadata:
+      labels:
+        app: telemetrygen-metrics
+    spec:
+      containers:
+        - name: telemetrygen-metrics
+          image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest
+          command: ["./telemetrygen"]
+          args:
+            - "--otlp-endpoint=cluster-collector-collector-headless.kuttl-otlp-metrics.svc:4317"
+            - "--otlp-insecure=true"
+            - "--duration=30s"
+            - "--rate=1"
+            - "--otlp-attributes=telemetrygen=\"metrics\""
+            - "--otlp-header=telemetrygen=\"traces\""
+            - "metrics"
+      restartPolicy: Never

tests/e2e-openshift/otlp-metrics-traces/05-assert.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+commands:
+- script: ./tests/e2e-openshift/otlp-metrics/check_traces.sh

tests/e2e-openshift/otlp-metrics-traces/06-assert.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+commands:
+- script: ./tests/e2e-openshift/otlp-metrics/check_metrics.sh

tests/e2e-openshift/otlp-metrics-traces/check_metrics.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+SECRET=$(oc get secret -n openshift-user-workload-monitoring | grep prometheus-user-workload-token | head -n 1 | awk '{print $1}')
+TOKEN=$(echo $(oc get secret $SECRET -n openshift-user-workload-monitoring -o json | jq -r '.data.token') | base64 -d)
+THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host')
+
+response=$(curl -k -H "Authorization: Bearer $TOKEN" -H "Content-type: application/json" "https://$THANOS_QUERIER_HOST/api/v1/query?query=gen")
+
+count=$(echo "$response" | jq -r '.data.result | length')
+
+if [[ $count -eq 0 ]]; then
+  echo "No telemetrygen metrics count with value present. Exiting with status 1."
+  exit 1
+else
+  echo "telemetrygen metrics with value is present."
+fi

tests/e2e-openshift/otlp-metrics-traces/check_traces.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+JAEGER_URL=$(oc -n kuttl-otlp-metrics get route jaeger-allinone -o json | jq '.spec.host' -r)
+SERVICE_NAME="telemetrygen"
+
+trace_exists=$(curl -ksSL "https://$JAEGER_URL/api/traces?service=$SERVICE_NAME&limit=1" | jq -r '.data | length')
+
+if [[ $trace_exists -gt 0 ]]; then
+  echo "Traces for $SERVICE_NAME exist in Jaeger."
+else
+  echo "Trace for $SERVICE_NAME does not exist in Jaeger."
+  exit 1  # Fail the test step if the trace doesn't exist
+fi