feat: Introduce labels in agent and gateways for usage in NetworkPoli…

…cy (#1557)

Co-authored-by: Stanislav Khalash <[email protected]>
Co-authored-by: Nina Hingerl <[email protected]>

docs/user/04-metrics.md

@@ -766,6 +766,32 @@ To detect and fix such situations, check the pipeline status and check out [Trou
 
 **Remedy 2**: Define the application protocol in the Service port definition by either prefixing the port name with the protocol, like in `http-metrics` or define the `appProtocol` attribute.
 
+**Cause 3**: A deny-all `NetworkPolicy` was created in the workload namespace, which prevents that the agent can scrape metrics from annotated workloads.
+
+**Remedy 3**: Create a separate `NetworkPolicy` to explicitly let the agent scrape your workload using the `telemetry.kyma-project.io/metric-scrape` label.
+
+For example, see the following `NetworkPolicy` configuration:
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-traffic-from-agent
+spec:
+  podSelector: 
+    matchLabels:
+      app.kubernetes.io/name: "annotated-workload" # <your workload here>
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: kyma-system
+      podSelector:
+        matchLabels:
+          telemetry.kyma-project.io/metric-scrape: "true"
+  policyTypes:
+  - Ingress
+```
+
 ### Gateway Buffer Filling Up
 
 **Symptom**: In the MetricPipeline status, the `TelemetryFlowHealthy` condition has status **BufferFillingUp**.

internal/labels/labels.go

@@ -0,0 +1,43 @@
+package labels
+
+const (
+	selectorLabelKey            = "app.kubernetes.io/name"
+	traceGatewayIngestSelector  = "telemetry.kyma-project.io/trace-ingest"
+	traceGatewayExportSelector  = "telemetry.kyma-project.io/trace-export"
+	metricAgentScrapeSelector   = "telemetry.kyma-project.io/metric-scrape"
+	metricGatewayIngestSelector = "telemetry.kyma-project.io/metric-ingest"
+	metricGatewayExportSelector = "telemetry.kyma-project.io/metric-export"
+	istioSidecarInjectLabel     = "sidecar.istio.io/inject"
+)
+
+func MakeDefaultLabel(baseName string) map[string]string {
+	return map[string]string{
+		selectorLabelKey: baseName,
+	}
+}
+
+func MakeMetricAgentSelectorLabel(baseName string) map[string]string {
+	return map[string]string{
+		selectorLabelKey:          baseName,
+		metricAgentScrapeSelector: "true",
+		istioSidecarInjectLabel:   "true",
+	}
+}
+
+func MakeMetricGatewaySelectorLabel(baseName string) map[string]string {
+	return map[string]string{
+		selectorLabelKey:            baseName,
+		metricGatewayIngestSelector: "true",
+		metricGatewayExportSelector: "true",
+		istioSidecarInjectLabel:     "true",
+	}
+}
+
+func MakeTraceGatewaySelectorLabel(baseName string) map[string]string {
+	return map[string]string{
+		selectorLabelKey:           baseName,
+		traceGatewayIngestSelector: "true",
+		traceGatewayExportSelector: "true",
+		istioSidecarInjectLabel:    "true",
+	}
+}

internal/labels/labels_test.go

@@ -0,0 +1,42 @@
+package labels
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMakeDefaultLabel(t *testing.T) {
+	podLabel := MakeDefaultLabel("my-pod")
+	require.Equal(t, map[string]string{
+		"app.kubernetes.io/name": "my-pod",
+	}, podLabel)
+}
+
+func TestMakeMetricAgentSelectorLabel(t *testing.T) {
+	metricAgentSelectorLabel := MakeMetricAgentSelectorLabel("metric-agent")
+	require.Equal(t, map[string]string{
+		"app.kubernetes.io/name":                  "metric-agent",
+		"telemetry.kyma-project.io/metric-scrape": "true",
+		"sidecar.istio.io/inject":                 "true",
+	}, metricAgentSelectorLabel)
+}
+
+func TestMakeMetricGatewaySelectorLabel(t *testing.T) {
+	metricGatewaySelectorLabel := MakeMetricGatewaySelectorLabel("metric-gateway")
+	require.Equal(t, map[string]string{
+		"app.kubernetes.io/name":                  "metric-gateway",
+		"telemetry.kyma-project.io/metric-ingest": "true",
+		"telemetry.kyma-project.io/metric-export": "true",
+		"sidecar.istio.io/inject":                 "true",
+	}, metricGatewaySelectorLabel)
+}
+func TestMakeTraceGatewaySelectorLabel(t *testing.T) {
+	traceGatewaySelectorLabel := MakeTraceGatewaySelectorLabel("trace-gateway")
+	require.Equal(t, map[string]string{
+		"app.kubernetes.io/name":                 "trace-gateway",
+		"telemetry.kyma-project.io/trace-ingest": "true",
+		"telemetry.kyma-project.io/trace-export": "true",
+		"sidecar.istio.io/inject":                "true",
+	}, traceGatewaySelectorLabel)
+}

internal/reconciler/metricpipeline/reconciler.go

@@ -15,6 +15,7 @@ import (
 	telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1"
 	"github.com/kyma-project/telemetry-manager/internal/errortypes"
 	"github.com/kyma-project/telemetry-manager/internal/k8sutils"
+	"github.com/kyma-project/telemetry-manager/internal/labels"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config/metric/agent"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config/metric/gateway"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config/otlpexporter"
@@ -266,10 +267,13 @@ func (r *Reconciler) reconcileMetricGateway(ctx context.Context, pipeline *telem
 		allowedPorts = append(allowedPorts, ports.IstioEnvoy)
 	}
 
+	metricGatewaySelectorLabels := labels.MakeMetricGatewaySelectorLabel(r.config.GatewayName)
+
 	opts := otelcollector.GatewayApplyOptions{
 		AllowedPorts:                   allowedPorts,
 		CollectorConfigYAML:            string(collectorConfigYAML),
 		CollectorEnvVars:               collectorEnvVars,
+		ComponentSelectorLabels:        metricGatewaySelectorLabels,
 		IstioEnabled:                   isIstioActive,
 		IstioExcludePorts:              []int32{ports.Metrics},
 		Replicas:                       r.getReplicaCountFromTelemetry(ctx),
@@ -306,12 +310,15 @@ func (r *Reconciler) reconcileMetricAgents(ctx context.Context, pipeline *teleme
 		allowedPorts = append(allowedPorts, ports.IstioEnvoy)
 	}
 
+	metricAgentSelectorLabels := labels.MakeMetricAgentSelectorLabel(r.config.AgentName)
+
 	if err := r.agentApplierDeleter.ApplyResources(
 		ctx,
 		k8sutils.NewOwnerReferenceSetter(r.Client, pipeline),
 		otelcollector.AgentApplyOptions{
-			AllowedPorts:        allowedPorts,
-			CollectorConfigYAML: string(agentConfigYAML),
+			AllowedPorts:            allowedPorts,
+			CollectorConfigYAML:     string(agentConfigYAML),
+			ComponentSelectorLabels: metricAgentSelectorLabels,
 		},
 	); err != nil {
 		return fmt.Errorf("failed to apply agent resources: %w", err)

internal/reconciler/tracepipeline/reconciler.go

@@ -31,6 +31,7 @@ import (
 	telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1"
 	"github.com/kyma-project/telemetry-manager/internal/errortypes"
 	"github.com/kyma-project/telemetry-manager/internal/k8sutils"
+	"github.com/kyma-project/telemetry-manager/internal/labels"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config/otlpexporter"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config/trace/gateway"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/ports"
@@ -245,10 +246,13 @@ func (r *Reconciler) reconcileTraceGateway(ctx context.Context, pipeline *teleme
 		allowedPorts = append(allowedPorts, ports.IstioEnvoy)
 	}
 
+	traceGatewaySelectorLabels := labels.MakeTraceGatewaySelectorLabel(r.config.TraceGatewayName)
+
 	opts := otelcollector.GatewayApplyOptions{
 		AllowedPorts:                   allowedPorts,
 		CollectorConfigYAML:            string(collectorConfigYAML),
 		CollectorEnvVars:               collectorEnvVars,
+		ComponentSelectorLabels:        traceGatewaySelectorLabels,
 		IstioEnabled:                   isIstioActive,
 		IstioExcludePorts:              []int32{ports.Metrics},
 		Replicas:                       r.getReplicaCountFromTelemetry(ctx),

internal/resources/fluentbit/resources.go

@@ -19,6 +19,7 @@ import (
 
 const checksumAnnotationKey = "checksum/logpipeline-config"
 const istioExcludeInboundPorts = "traffic.sidecar.istio.io/excludeInboundPorts"
+const fluentbitExportSelector = "telemetry.kyma-project.io/log-export"
 
 type DaemonSetConfig struct {
 	FluentBitImage              string
@@ -61,6 +62,7 @@ func MakeDaemonSet(name types.NamespacedName, checksum string, dsConfig DaemonSe
 
 	podLabels := Labels()
 	podLabels["sidecar.istio.io/inject"] = "true"
+	podLabels[fluentbitExportSelector] = "true"
 
 	return &appsv1.DaemonSet{
 		TypeMeta: metav1.TypeMeta{},

internal/resources/fluentbit/resources_test.go

@@ -39,9 +39,10 @@ func TestMakeDaemonSet(t *testing.T) {
 		"app.kubernetes.io/instance": "telemetry",
 	}, daemonSet.Spec.Selector.MatchLabels)
 	require.Equal(t, map[string]string{
-		"app.kubernetes.io/name":     "fluent-bit",
-		"app.kubernetes.io/instance": "telemetry",
-		"sidecar.istio.io/inject":    "true",
+		"app.kubernetes.io/name":               "fluent-bit",
+		"app.kubernetes.io/instance":           "telemetry",
+		"sidecar.istio.io/inject":              "true",
+		"telemetry.kyma-project.io/log-export": "true",
 	}, daemonSet.Spec.Template.ObjectMeta.Labels)
 	require.NotEmpty(t, daemonSet.Spec.Template.Spec.Containers[0].EnvFrom)
 	require.NotNil(t, daemonSet.Spec.Template.Spec.Containers[0].LivenessProbe, "liveness probe must be defined")

internal/resources/otelcollector/agent.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/kyma-project/telemetry-manager/internal/configchecksum"
 	"github.com/kyma-project/telemetry-manager/internal/k8sutils"
+	"github.com/kyma-project/telemetry-manager/internal/labels"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/config"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/ports"
 	commonresources "github.com/kyma-project/telemetry-manager/internal/resources/common"
@@ -32,8 +33,9 @@ type AgentApplierDeleter struct {
 }
 
 type AgentApplyOptions struct {
-	AllowedPorts        []int32
-	CollectorConfigYAML string
+	AllowedPorts            []int32
+	CollectorConfigYAML     string
+	ComponentSelectorLabels map[string]string
 }
 
 func (aad *AgentApplierDeleter) ApplyResources(ctx context.Context, c client.Client, opts AgentApplyOptions) error {
@@ -49,7 +51,7 @@ func (aad *AgentApplierDeleter) ApplyResources(ctx context.Context, c client.Cli
 	}
 
 	configChecksum := configchecksum.Calculate([]corev1.ConfigMap{*configMap}, []corev1.Secret{})
-	if err := k8sutils.CreateOrUpdateDaemonSet(ctx, c, aad.makeAgentDaemonSet(configChecksum)); err != nil {
+	if err := k8sutils.CreateOrUpdateDaemonSet(ctx, c, aad.makeAgentDaemonSet(configChecksum, opts)); err != nil {
 		return fmt.Errorf("failed to create daemonset: %w", err)
 	}
 
@@ -83,10 +85,8 @@ func (aad *AgentApplierDeleter) DeleteResources(ctx context.Context, c client.Cl
 	return allErrors
 }
 
-func (aad *AgentApplierDeleter) makeAgentDaemonSet(configChecksum string) *appsv1.DaemonSet {
-	selectorLabels := defaultLabels(aad.Config.BaseName)
-	podLabels := maps.Clone(selectorLabels)
-	podLabels["sidecar.istio.io/inject"] = "true"
+func (aad *AgentApplierDeleter) makeAgentDaemonSet(configChecksum string, opts AgentApplyOptions) *appsv1.DaemonSet {
+	selectorLabels := labels.MakeDefaultLabel(aad.Config.BaseName)
 
 	annotations := map[string]string{"checksum/config": configChecksum}
 	maps.Copy(annotations, makeIstioTLSPodAnnotations(IstioCertPath))
@@ -123,7 +123,7 @@ func (aad *AgentApplierDeleter) makeAgentDaemonSet(configChecksum string) *appsv
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels:      podLabels,
+					Labels:      opts.ComponentSelectorLabels,
 					Annotations: annotations,
 				},
 				Spec: podSpec,

internal/resources/otelcollector/agent_test.go

@@ -14,6 +14,8 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"github.com/kyma-project/telemetry-manager/internal/labels"
 )
 
 var (
@@ -37,8 +39,9 @@ func TestApplyAgentResources(t *testing.T) {
 	}
 
 	err := sut.ApplyResources(ctx, client, AgentApplyOptions{
-		AllowedPorts:        []int32{5555, 6666},
-		CollectorConfigYAML: agentCfg,
+		AllowedPorts:            []int32{5555, 6666},
+		CollectorConfigYAML:     agentCfg,
+		ComponentSelectorLabels: labels.MakeMetricAgentSelectorLabel(agentName),
 	})
 	require.NoError(t, err)
 
@@ -205,8 +208,9 @@ func TestApplyAgentResources(t *testing.T) {
 			"app.kubernetes.io/name": agentName,
 		}, ds.Spec.Selector.MatchLabels, "must have expected daemonset selector labels")
 		require.Equal(t, map[string]string{
-			"app.kubernetes.io/name":  agentName,
-			"sidecar.istio.io/inject": "true",
+			"app.kubernetes.io/name":                  agentName,
+			"sidecar.istio.io/inject":                 "true",
+			"telemetry.kyma-project.io/metric-scrape": "true",
 		}, ds.Spec.Template.ObjectMeta.Labels, "must have expected pod labels")
 
 		// annotations
@@ -323,7 +327,7 @@ func createAgentRBAC() Rbac {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      agentName,
 			Namespace: agentNamespace,
-			Labels:    defaultLabels(agentName),
+			Labels:    labels.MakeDefaultLabel(agentName),
 		},
 		Rules: []rbacv1.PolicyRule{
 			{
@@ -338,7 +342,7 @@ func createAgentRBAC() Rbac {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      agentName,
 			Namespace: agentNamespace,
-			Labels:    defaultLabels(agentName),
+			Labels:    labels.MakeDefaultLabel(agentName),
 		},
 		Subjects: []rbacv1.Subject{{Name: agentName, Namespace: agentNamespace, Kind: rbacv1.ServiceAccountKind}},
 		RoleRef: rbacv1.RoleRef{

internal/resources/otelcollector/core.go

@@ -16,6 +16,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kyma-project/telemetry-manager/internal/k8sutils"
+	"github.com/kyma-project/telemetry-manager/internal/labels"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/ports"
 	commonresources "github.com/kyma-project/telemetry-manager/internal/resources/common"
 )
@@ -64,7 +65,7 @@ func applyCommonResources(ctx context.Context, c client.Client, name types.Names
 		return fmt.Errorf("failed to create metrics service: %w", err)
 	}
 
-	if err := k8sutils.CreateOrUpdateNetworkPolicy(ctx, c, commonresources.MakeNetworkPolicy(name, allowedPorts, defaultLabels(name.Name))); err != nil {
+	if err := k8sutils.CreateOrUpdateNetworkPolicy(ctx, c, commonresources.MakeNetworkPolicy(name, allowedPorts, labels.MakeDefaultLabel(name.Name))); err != nil {
 		return fmt.Errorf("failed to create network policy: %w", err)
 	}
 
@@ -118,18 +119,12 @@ func deleteCommonResources(ctx context.Context, c client.Client, name types.Name
 	return allErrors
 }
 
-func defaultLabels(baseName string) map[string]string {
-	return map[string]string{
-		"app.kubernetes.io/name": baseName,
-	}
-}
-
 func makeServiceAccount(name types.NamespacedName) *corev1.ServiceAccount {
 	serviceAccount := corev1.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name.Name,
 			Namespace: name.Namespace,
-			Labels:    defaultLabels(name.Name),
+			Labels:    labels.MakeDefaultLabel(name.Name),
 		},
 	}
 
@@ -141,7 +136,7 @@ func makeConfigMap(name types.NamespacedName, collectorConfigYAML string) *corev
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name.Name,
 			Namespace: name.Namespace,
-			Labels:    defaultLabels(name.Name),
+			Labels:    labels.MakeDefaultLabel(name.Name),
 		},
 		Data: map[string]string{
 			configMapKey: collectorConfigYAML,
@@ -154,14 +149,14 @@ func makeSecret(name types.NamespacedName, secretData map[string][]byte) *corev1
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name.Name,
 			Namespace: name.Namespace,
-			Labels:    defaultLabels(name.Name),
+			Labels:    labels.MakeDefaultLabel(name.Name),
 		},
 		Data: secretData,
 	}
 }
 
 func makeMetricsService(name types.NamespacedName) *corev1.Service {
-	labels := defaultLabels(name.Name)
+	labels := labels.MakeDefaultLabel(name.Name)
 	selectorLabels := make(map[string]string)
 	maps.Copy(selectorLabels, labels)
 	labels["telemetry.kyma-project.io/self-monitor"] = "enabled"