Create ServiceMonitors to gather metrics from the OTEL Operands (#1874)

* Allow the creation of ServiceMonitors to gather metrics from the OpenTelemetry Collector instances Signed-off-by: Israel Blancas <[email protected]> * Add missing changelog Signed-off-by: Israel Blancas <[email protected]> * Fix unprotected statement Signed-off-by: Israel Blancas <[email protected]> * Fix lint issues Signed-off-by: Israel Blancas <[email protected]> * Apply changes requested in code review Signed-off-by: Israel Blancas <[email protected]> * Add missing generated files Signed-off-by: Israel Blancas <[email protected]> * Change the way to enable the feature flag Signed-off-by: Israel Blancas <[email protected]> * Change the way to enable the feature flag Signed-off-by: Israel Blancas <[email protected]> * Fix merge * Fix enable feature flag * Change the name of the option and move the E2E tests to their own folder * Fix unit test * Fix docs * Fix CRD field * Fix CRD field * Add from version to feature gate Signed-off-by: Israel Blancas <[email protected]> * Move the E2E tests to their own section for the CI Signed-off-by: Israel Blancas <[email protected]> --------- Signed-off-by: Israel Blancas <[email protected]>
open-telemetry · Jul 25, 2023 · 42ff92f · 42ff92f
1 parent e21920f
commit 42ff92f
Show file tree

Hide file tree

Showing 25 changed files with 683 additions and 8 deletions.
diff --git a/.chloggen/1768-gather-metrics-from-collectors.yaml b/.chloggen/1768-gather-metrics-from-collectors.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, target allocator, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add the ability to the operator to create Service Monitors for the OpenTelemetry Collectors in order to gather the metrics they are generating
+
+# One or more tracking issues related to the change
+issues: [1768]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -29,10 +29,10 @@ jobs:
        - "1.27"
        group:
          - e2e e2e-upgrade
+         - e2e-prometheuscr
          - e2e-autoscale
 
     steps:
-
     - name: Set up Go
       uses: actions/setup-go@v4
       with:

diff --git a/Makefile b/Makefile
@@ -176,6 +176,11 @@ generate: controller-gen api-docs
 e2e:
 	$(KUTTL) test
 
+# end-to-end-test for PrometheusCR E2E tests
+.PHONY: e2e-prometheuscr
+e2e-prometheuscr:
+	$(KUTTL) test --config kuttl-test-prometheuscr.yaml
+
 # end-to-end-test for testing upgrading
 .PHONY: e2e-upgrade
 e2e-upgrade: undeploy
@@ -200,6 +205,11 @@ e2e-log-operator:
 prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server load-image-all deploy
 	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
 
+.PHONY: enable-prometheus-feature-flag
+enable-prometheus-feature-flag:
+	$(SED) -i "s#--feature-gates=+operator.autoinstrumentation.go#--feature-gates=+operator.autoinstrumentation.go,+operator.observability.prometheus#g" config/default/manager_auth_proxy_patch.yaml
+
+
 .PHONY: scorecard-tests
 scorecard-tests: operator-sdk
 	$(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1)
@@ -238,6 +248,10 @@ endif
 install-metrics-server:
 	./hack/install-metrics-server.sh
 
+.PHONY: install-prometheus-operator
+install-prometheus-operator:
+	./hack/install-prometheus-operator.sh
+
 .PHONY: load-image-all
 load-image-all: load-image-operator load-image-target-allocator load-image-operator-opamp-bridge
 

diff --git a/apis/v1alpha1/opentelemetrycollector_types.go b/apis/v1alpha1/opentelemetrycollector_types.go
@@ -190,6 +190,14 @@ type OpenTelemetryCollectorSpec struct {
 	// https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
 	// +optional
 	InitContainers []v1.Container `json:"initContainers,omitempty"`
+
+	// ObservabilitySpec defines how telemetry data gets handled.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Observability"
+	Observability ObservabilitySpec `json:"observability,omitempty"`
+
 	// TopologySpreadConstraints embedded kubernetes pod configuration option,
 	// controls how pods are spread across your cluster among failure-domains
 	// such as regions, zones, nodes, and other user-defined topology domains
@@ -371,6 +379,27 @@ type AutoscalerSpec struct {
 	TargetMemoryUtilization *int32 `json:"targetMemoryUtilization,omitempty"`
 }
 
+// MetricsConfigSpec defines a metrics config.
+type MetricsConfigSpec struct {
+	// EnableMetrics specifies if ServiceMonitors should be created for the OpenTelemetry Collector.
+	// The operator.observability.prometheus feature gate must be enabled to use this feature.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Create ServiceMonitors for OpenTelemetry Collector"
+	EnableMetrics bool `json:"enableMetrics,omitempty"`
+}
+
+// ObservabilitySpec defines how telemetry data gets handled.
+type ObservabilitySpec struct {
+	// Metrics defines the metrics configuration for operands.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Metrics Config"
+	Metrics MetricsConfigSpec `json:"metrics,omitempty"`
+}
+
 // Probe defines the OpenTelemetry's pod probe config. Only Liveness probe is supported currently.
 type Probe struct {
 	// Number of seconds after the container has started before liveness probes are initiated.

diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go
diff --git a/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml
@@ -76,6 +76,18 @@ spec:
       - kind: StatefulSets
         name: ""
         version: apps/v1
+      specDescriptors:
+      - description: ObservabilitySpec defines how telemetry data gets handled.
+        displayName: Observability
+        path: observability
+      - description: Metrics defines the metrics configuration for operands.
+        displayName: Metrics Config
+        path: observability.metrics
+      - description: EnableMetrics specifies if ServiceMonitors should be created
+          for the OpenTelemetry Collector. The operator.observability.prometheus feature
+          gate must be enabled to use this feature.
+        displayName: Create ServiceMonitors for OpenTelemetry Collector
+        path: observability.metrics.enableMetrics
       version: v1alpha1
   description: |-
     OpenTelemetry is a collection of tools, APIs, and SDKs. You use it to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) for analysis in order to understand your software's performance and behavior.
@@ -209,6 +221,18 @@ spec:
           - get
           - list
           - update
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - patch
+          - update
+          - watch
         - apiGroups:
           - networking.k8s.io
           resources:

diff --git a/bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml b/bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml
@@ -2949,6 +2949,19 @@ spec:
                   This is only relevant to daemonset, statefulset, and deployment
                   mode
                 type: object
+              observability:
+                description: ObservabilitySpec defines how telemetry data gets handled.
+                properties:
+                  metrics:
+                    description: Metrics defines the metrics configuration for operands.
+                    properties:
+                      enableMetrics:
+                        description: EnableMetrics specifies if ServiceMonitors should
+                          be created for the OpenTelemetry Collector. The operator.observability.prometheus
+                          feature gate must be enabled to use this feature.
+                        type: boolean
+                    type: object
+                type: object
               podAnnotations:
                 additionalProperties:
                   type: string

diff --git a/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml b/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml
@@ -2946,6 +2946,19 @@ spec:
                   This is only relevant to daemonset, statefulset, and deployment
                   mode
                 type: object
+              observability:
+                description: ObservabilitySpec defines how telemetry data gets handled.
+                properties:
+                  metrics:
+                    description: Metrics defines the metrics configuration for operands.
+                    properties:
+                      enableMetrics:
+                        description: EnableMetrics specifies if ServiceMonitors should
+                          be created for the OpenTelemetry Collector. The operator.observability.prometheus
+                          feature gate must be enabled to use this feature.
+                        type: boolean
+                    type: object
+                type: object
               podAnnotations:
                 additionalProperties:
                   type: string

diff --git a/config/manifests/bases/opentelemetry-operator.clusterserviceversion.yaml b/config/manifests/bases/opentelemetry-operator.clusterserviceversion.yaml
@@ -56,6 +56,18 @@ spec:
       - kind: StatefulSets
         name: ""
         version: apps/v1
+      specDescriptors:
+      - description: ObservabilitySpec defines how telemetry data gets handled.
+        displayName: Observability
+        path: observability
+      - description: Metrics defines the metrics configuration for operands.
+        displayName: Metrics Config
+        path: observability.metrics
+      - description: EnableMetrics specifies if ServiceMonitors should be created
+          for the OpenTelemetry Collector. The operator.observability.prometheus feature
+          gate must be enabled to use this feature.
+        displayName: Create ServiceMonitors for OpenTelemetry Collector
+        path: observability.metrics.enableMetrics
       version: v1alpha1
   description: |-
     OpenTelemetry is a collection of tools, APIs, and SDKs. You use it to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) for analysis in order to understand your software's performance and behavior.

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -119,6 +119,18 @@ rules:
   - get
   - list
   - update
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources:

diff --git a/controllers/opentelemetrycollector_controller.go b/controllers/opentelemetrycollector_controller.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 
 	"github.com/go-logr/logr"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2"
@@ -35,6 +36,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-operator/internal/config"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/autodetect"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/collector/reconcile"
+	"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
 )
 
 // OpenTelemetryCollectorReconciler reconciles a OpenTelemetryCollector object.
@@ -164,6 +166,11 @@ func NewReconciler(p Params) *OpenTelemetryCollectorReconciler {
 				"ingresses",
 				true,
 			},
+			{
+				reconcile.ServiceMonitors,
+				"service monitors",
+				true,
+			},
 			{
 				reconcile.Self,
 				"opentelemetry",
@@ -251,6 +258,10 @@ func (r *OpenTelemetryCollectorReconciler) SetupWithManager(mgr ctrl.Manager) er
 		Owns(&appsv1.DaemonSet{}).
 		Owns(&appsv1.StatefulSet{})
 
+	if featuregate.PrometheusOperatorIsAvailable.IsEnabled() {
+		builder.Owns(&monitoringv1.ServiceMonitor{})
+	}
+
 	autoscalingVersion := r.config.AutoscalingVersion()
 	if autoscalingVersion == autodetect.AutoscalingVersionV2 {
 		builder = builder.Owns(&autoscalingv2.HorizontalPodAutoscaler{})

diff --git a/docs/api.md b/docs/api.md
@@ -3736,6 +3736,13 @@ OpenTelemetryCollectorSpec defines the desired state of OpenTelemetryCollector.
           NodeSelector to schedule OpenTelemetry Collector pods. This is only relevant to daemonset, statefulset, and deployment mode<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b><a href="#opentelemetrycollectorspecobservability">observability</a></b></td>
+        <td>object</td>
+        <td>
+          ObservabilitySpec defines how telemetry data gets handled.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>podAnnotations</b></td>
         <td>map[string]string</td>
@@ -8985,6 +8992,60 @@ Liveness config for the OpenTelemetry Collector except the probe handler which i
 </table>
 
 
+### OpenTelemetryCollector.spec.observability
+<sup><sup>[↩ Parent](#opentelemetrycollectorspec)</sup></sup>
+
+
+
+ObservabilitySpec defines how telemetry data gets handled.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b><a href="#opentelemetrycollectorspecobservabilitymetrics">metrics</a></b></td>
+        <td>object</td>
+        <td>
+          Metrics defines the metrics configuration for operands.<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### OpenTelemetryCollector.spec.observability.metrics
+<sup><sup>[↩ Parent](#opentelemetrycollectorspecobservability)</sup></sup>
+
+
+
+Metrics defines the metrics configuration for operands.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>enableMetrics</b></td>
+        <td>boolean</td>
+        <td>
+          EnableMetrics specifies if ServiceMonitors should be created for the OpenTelemetry Collector. The operator.observability.prometheus feature gate must be enabled to use this feature.<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
 ### OpenTelemetryCollector.spec.podSecurityContext
 <sup><sup>[↩ Parent](#opentelemetrycollectorspec)</sup></sup>