feat: Add dummy LogPipeline OTLP output (#1427)

Co-authored-by: Korbinian Stoemmer <[email protected]>

.golangci.yaml

@@ -78,6 +78,8 @@ linters-settings:
         alias: logparserwebhook
       - pkg: github.com/kyma-project/telemetry-manager/webhook/logpipeline
         alias: logpipelinewebhook
+      - pkg: github.com/kyma-project/telemetry-manager/internal/reconciler/logpipeline/fluentbit
+        alias: logpipelinefluentbit
       - pkg: github.com/kyma-project/telemetry-manager/webhook/logpipeline/validation/mocks
         alias: logpipelinevalidationmocks
       - pkg: github.com/prometheus/client_golang/api/prometheus/v1

.testcoverage.yml

@@ -19,11 +19,15 @@ threshold:
 # specific to more general rules.
 override:
   # Temporarily decrease coverage threshold for certain packages, we need to increase the coverage and remove these entries from the list
-  - threshold: 57
+  - threshold: 87
     path: ^internal/istiostatus$
-  - threshold: 86
+  - threshold: 87
     path: ^internal/otelcollector/config/otlpexporter$
-  - threshold: 73
+  - threshold: 77
+    path: ^internal/reconciler/logpipeline/fluentbit$
+  - threshold: 0
+    path: ^internal/reconciler/logpipeline/otel$
+  - threshold: 82
     path: ^internal/reconciler/logpipeline$
   - threshold: 77
     path: ^internal/reconciler/metricpipeline$
@@ -33,30 +37,30 @@ override:
     path: ^internal/reconciler/tracepipeline$
   - threshold: 82
     path: ^internal/resources/otelcollector$
-  - threshold: 75
+  - threshold: 78
     path: ^internal/resources/selfmonitor$
   - threshold: 75
     path: ^internal/resourcelock$
   - threshold: 85
     path: ^internal/webhookcert$
-  - threshold: 70
+  - threshold: 73
     path: ^webhook/logpipeline
 
 # Holds regexp rules which will exclude matched files or packages from coverage statistics
 exclude:
   # Exclude files or packages matching their paths
   paths:
-    - ^apis                # apis should in general only contain Go structs and not unit tested
-    - ^controllers         # controllers are covered by e2e tests
+    - ^apis # apis should in general only contain Go structs and not unit tested
+    - ^controllers # controllers are covered by e2e tests
     - main
     - mocks
     - stubs
-    - ^internal/k8sutils   # k8sutils has to be dissected into smaller single-purpose packages, for now excluding from coverage check
+    - ^internal/k8sutils # k8sutils has to be dissected into smaller single-purpose packages, for now excluding from coverage check
     - internal/logger
-    - ^internal/reconciler/logparser  # logparser reconciler will be removed soon
-    - ^test                # e2e and integration tests
-    - webhook/logparser    # logparser will be removed soon
-    - ^internal/testutils  # some functions in testutils are being used in e2e tests only
+    - ^internal/reconciler/logparser # logparser reconciler will be removed soon
+    - ^test # e2e and integration tests
+    - webhook/logparser # logparser will be removed soon
+    - ^internal/testutils # some functions in testutils are being used in e2e tests only
 
 # NOTES:
 # - symbol `/` in all path regexps will be replaced by

Makefile

@@ -48,6 +48,8 @@ GO_TEST_COVERAGE := $(TOOLS_BIN_DIR)/go-test-coverage
 KUSTOMIZE        := $(TOOLS_BIN_DIR)/kustomize
 MOCKERY          := $(TOOLS_BIN_DIR)/mockery
 TABLE_GEN        := $(TOOLS_BIN_DIR)/table-gen
+YQ               := $(TOOLS_BIN_DIR)/yq
+STRINGER         := $(TOOLS_BIN_DIR)/stringer
 
 # Sub-makefile
 include hack/make/provision.mk
@@ -90,18 +92,22 @@ crd-docs-gen: $(TABLE_GEN) manifests## Generates CRD spec into docs folder
 	$(TABLE_GEN) --crd-filename ./config/crd/bases/telemetry.kyma-project.io_metricpipelines.yaml --md-filename ./docs/user/resources/05-metricpipeline.md
 
 .PHONY: manifests
-manifests: $(CONTROLLER_GEN) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition for v1alpha1.
+manifests: $(CONTROLLER_GEN) $(YQ) $(YAMLFMT) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition for v1alpha1.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..."
 	$(CONTROLLER_GEN) crd paths="./apis/operator/v1alpha1" output:crd:artifacts:config=config/crd/bases
 	$(CONTROLLER_GEN) crd paths="./apis/telemetry/v1alpha1" output:crd:artifacts:config=config/crd/bases
+	$(YQ) eval 'del(.. | select(has("otlp")).otlp)' -i ./config/crd/bases/telemetry.kyma-project.io_logpipelines.yaml
+	$(YQ) eval 'del(.. | select(has("x-kubernetes-validations"))."x-kubernetes-validations"[] | select(.rule|contains("otlp")) )' -i ./config/crd/bases/telemetry.kyma-project.io_logpipelines.yaml
+
 
 .PHONY: manifests-dev
 manifests-dev: $(CONTROLLER_GEN) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition for v1alpha1 and v1beta1.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role webhook crd paths="./..." output:crd:artifacts:config=config/development/crd/bases
 
 .PHONY: generate
-generate: $(CONTROLLER_GEN) $(MOCKERY) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+generate: $(CONTROLLER_GEN) $(MOCKERY) $(STRINGER) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(MOCKERY)
+	$(STRINGER) --type OutputType internal/reconciler/logpipeline/reconciler.go
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 .PHONY: fmt

apis/telemetry/v1alpha1/logpipeline_conversion.go

@@ -55,6 +55,17 @@ func (lp *LogPipeline) ConvertTo(dstRaw conversion.Hub) error {
 		}
 	}
 
+	if srcOTLPOutput := src.Spec.Output.Otlp; srcOTLPOutput != nil {
+		dst.Spec.Output.OTLP = &telemetryv1beta1.OTLPOutput{
+			Protocol:       telemetryv1beta1.OTLPProtocol(srcOTLPOutput.Protocol),
+			Endpoint:       v1Alpha1ValueTypeToV1Beta1(srcOTLPOutput.Endpoint),
+			Path:           srcOTLPOutput.Path,
+			Authentication: v1Alpha1AuthenticationToV1Beta1(srcOTLPOutput.Authentication),
+			Headers:        v1Alpha1HeadersToV1Beta1(srcOTLPOutput.Headers),
+			TLS:            v1Alpha1OtlpTLSToV1Beta1(srcOTLPOutput.TLS),
+		}
+	}
+
 	if srcCustomOutput := src.Spec.Output.Custom; srcCustomOutput != "" {
 		dst.Spec.Output.Custom = srcCustomOutput
 	}
@@ -64,6 +75,67 @@ func (lp *LogPipeline) ConvertTo(dstRaw conversion.Hub) error {
 	return nil
 }
 
+func v1Alpha1OtlpTLSToV1Beta1(tls *OtlpTLS) *telemetryv1beta1.OTLPTLS {
+	if tls == nil {
+		return nil
+	}
+
+	betaTLS := &telemetryv1beta1.OTLPTLS{
+		Insecure:           tls.Insecure,
+		InsecureSkipVerify: tls.InsecureSkipVerify,
+	}
+
+	if tls.CA != nil {
+		ca := v1Alpha1ValueTypeToV1Beta1(*tls.CA)
+		betaTLS.CA = &ca
+	}
+	if tls.Key != nil {
+		key := v1Alpha1ValueTypeToV1Beta1(*tls.Key)
+		betaTLS.Key = &key
+	}
+	if tls.Cert != nil {
+		cert := v1Alpha1ValueTypeToV1Beta1(*tls.Cert)
+		betaTLS.Cert = &cert
+	}
+	return betaTLS
+}
+
+func v1Alpha1HeadersToV1Beta1(headers []Header) []telemetryv1beta1.Header {
+	var dst []telemetryv1beta1.Header
+	for _, h := range headers {
+		dst = append(dst, v1Alpha1HeaderToV1Beta1(h))
+	}
+	return dst
+
+}
+
+func v1Alpha1HeaderToV1Beta1(h Header) telemetryv1beta1.Header {
+	return telemetryv1beta1.Header{
+		Name:      h.Name,
+		ValueType: v1Alpha1ValueTypeToV1Beta1(h.ValueType),
+		Prefix:    h.Prefix,
+	}
+}
+
+func v1Alpha1AuthenticationToV1Beta1(authentication *AuthenticationOptions) *telemetryv1beta1.AuthenticationOptions {
+	if authentication == nil {
+		return nil
+	}
+	return &telemetryv1beta1.AuthenticationOptions{
+		Basic: v1Alpha1BasicAuthOptionsToV1Beta1(authentication.Basic),
+	}
+}
+
+func v1Alpha1BasicAuthOptionsToV1Beta1(basic *BasicAuthOptions) *telemetryv1beta1.BasicAuthOptions {
+	if basic == nil {
+		return nil
+	}
+	return &telemetryv1beta1.BasicAuthOptions{
+		User:     v1Alpha1ValueTypeToV1Beta1(basic.User),
+		Password: v1Alpha1ValueTypeToV1Beta1(basic.Password),
+	}
+}
+
 func v1Alpha1ValueTypeToV1Beta1(src ValueType) telemetryv1beta1.ValueType {
 	if src.ValueFrom != nil && src.ValueFrom.SecretKeyRef != nil {
 		return telemetryv1beta1.ValueType{
@@ -144,6 +216,17 @@ func (lp *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error {
 		}
 	}
 
+	if srcOTLPOutput := src.Spec.Output.OTLP; srcOTLPOutput != nil {
+		dst.Spec.Output.Otlp = &OtlpOutput{
+			Protocol:       (string)(srcOTLPOutput.Protocol),
+			Endpoint:       v1Beta1ValueTypeToV1Alpha1(srcOTLPOutput.Endpoint),
+			Path:           srcOTLPOutput.Path,
+			Authentication: v1Beta1AuthenticationToV1Alpha1(srcOTLPOutput.Authentication),
+			Headers:        v1Beta1HeadersToV1Alpha1(srcOTLPOutput.Headers),
+			TLS:            v1Beta1OtlpTLSToV1Alpha1(srcOTLPOutput.TLS),
+		}
+	}
+
 	if srcCustomOutput := src.Spec.Output.Custom; srcCustomOutput != "" {
 		dst.Spec.Output.Custom = srcCustomOutput
 	}
@@ -153,6 +236,66 @@ func (lp *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error {
 	return nil
 }
 
+func v1Beta1OtlpTLSToV1Alpha1(tls *telemetryv1beta1.OTLPTLS) *OtlpTLS {
+	if tls == nil {
+		return nil
+	}
+
+	alphaTLS := &OtlpTLS{
+		Insecure:           tls.Insecure,
+		InsecureSkipVerify: tls.InsecureSkipVerify,
+	}
+
+	if tls.CA != nil {
+		ca := v1Beta1ValueTypeToV1Alpha1(*tls.CA)
+		alphaTLS.CA = &ca
+	}
+	if tls.Key != nil {
+		key := v1Beta1ValueTypeToV1Alpha1(*tls.Key)
+		alphaTLS.Key = &key
+	}
+	if tls.Cert != nil {
+		cert := v1Beta1ValueTypeToV1Alpha1(*tls.Cert)
+		alphaTLS.Cert = &cert
+	}
+	return alphaTLS
+}
+
+func v1Beta1HeadersToV1Alpha1(headers []telemetryv1beta1.Header) []Header {
+	var dst []Header
+	for _, h := range headers {
+		dst = append(dst, v1Beta1HeaderToV1Alpha1(h))
+	}
+	return dst
+}
+
+func v1Beta1HeaderToV1Alpha1(h telemetryv1beta1.Header) Header {
+	return Header{
+		Name:      h.Name,
+		ValueType: v1Beta1ValueTypeToV1Alpha1(h.ValueType),
+		Prefix:    h.Prefix,
+	}
+}
+
+func v1Beta1AuthenticationToV1Alpha1(authentication *telemetryv1beta1.AuthenticationOptions) *AuthenticationOptions {
+	if authentication == nil {
+		return nil
+	}
+	return &AuthenticationOptions{
+		Basic: v1Beta1BasicAuthOptionsToV1Alpha1(authentication.Basic),
+	}
+}
+
+func v1Beta1BasicAuthOptionsToV1Alpha1(basic *telemetryv1beta1.BasicAuthOptions) *BasicAuthOptions {
+	if basic == nil {
+		return nil
+	}
+	return &BasicAuthOptions{
+		User:     v1Beta1ValueTypeToV1Alpha1(basic.User),
+		Password: v1Beta1ValueTypeToV1Alpha1(basic.Password),
+	}
+}
+
 func v1Beta1TLSToV1Alpha1(src telemetryv1beta1.LogPipelineHTTPOutputTLS) TLSConfig {
 	var dst TLSConfig
 	if src.CA != nil {

apis/telemetry/v1alpha1/logpipeline_conversion_test.go

@@ -76,6 +76,52 @@ func TestConvertTo(t *testing.T) {
 					},
 					Dedot: true,
 				},
+				Otlp: &OtlpOutput{
+					Protocol: OtlpProtocolGRPC,
+					Endpoint: ValueType{
+						Value: "localhost:4317",
+					},
+					Path: "/v1/logs",
+					Authentication: &AuthenticationOptions{
+						Basic: &BasicAuthOptions{
+							User: ValueType{
+								Value: "user",
+							},
+							Password: ValueType{
+								Value: "password",
+							},
+						},
+					},
+					Headers: []Header{
+						{
+							Name: "header1",
+							ValueType: ValueType{
+								Value: "value1",
+							},
+							Prefix: "prefix1",
+						},
+						{
+							Name: "header2",
+							ValueType: ValueType{
+								Value: "value2",
+							},
+							Prefix: "prefix2",
+						},
+					},
+					TLS: &OtlpTLS{
+						Insecure:           true,
+						InsecureSkipVerify: true,
+						CA: &ValueType{
+							Value: "ca",
+						},
+						Cert: &ValueType{
+							Value: "cert",
+						},
+						Key: &ValueType{
+							Value: "key",
+						},
+					},
+				},
 			},
 		},
 		Status: LogPipelineStatus{
@@ -169,6 +215,42 @@ func TestConvertFrom(t *testing.T) {
 					},
 					Dedot: true,
 				},
+				OTLP: &telemetryv1beta1.OTLPOutput{
+					Protocol: telemetryv1beta1.OTLPProtocolGRPC,
+					Endpoint: telemetryv1beta1.ValueType{Value: "localhost:4317"},
+					Path:     "/v1/logs",
+					Authentication: &telemetryv1beta1.AuthenticationOptions{Basic: &telemetryv1beta1.BasicAuthOptions{
+						User: telemetryv1beta1.ValueType{
+							Value: "user",
+						},
+						Password: telemetryv1beta1.ValueType{
+							Value: "password",
+						},
+					}},
+					Headers: []telemetryv1beta1.Header{
+						{
+							Name: "header1",
+							ValueType: telemetryv1beta1.ValueType{
+								Value: "value1",
+							},
+							Prefix: "prefix1",
+						},
+						{
+							Name: "header2",
+							ValueType: telemetryv1beta1.ValueType{
+								Value: "value2",
+							},
+							Prefix: "prefix2",
+						},
+					},
+					TLS: &telemetryv1beta1.OTLPTLS{
+						Insecure:           true,
+						InsecureSkipVerify: true,
+						CA:                 &telemetryv1beta1.ValueType{Value: "ca"},
+						Cert:               &telemetryv1beta1.ValueType{Value: "cert"},
+						Key:                &telemetryv1beta1.ValueType{Value: "key"},
+					},
+				},
 			},
 		},
 		Status: telemetryv1beta1.LogPipelineStatus{
@@ -235,6 +317,28 @@ func requireLogPipelinesEquivalent(t *testing.T, x *LogPipeline, y *telemetryv1b
 	require.Equal(t, xHTTP.TLSConfig.Cert.Value, yHTTP.TLSConfig.Cert.Value, "HTTP TLS cert mismatch")
 	require.Equal(t, xHTTP.TLSConfig.Key.Value, yHTTP.TLSConfig.Key.Value, "HTTP TLS key mismatch")
 
+	xOTLP := x.Spec.Output.Otlp
+	yOTLP := y.Spec.Output.OTLP
+	require.NotNil(t, xOTLP, "expected OTLP output")
+	require.NotNil(t, yOTLP, "expected OTLP output")
+	require.Equal(t, xOTLP.Protocol, string(yOTLP.Protocol), "OTLP protocol mismatch")
+	require.Equal(t, xOTLP.Endpoint.Value, yOTLP.Endpoint.Value, "OTLP endpoint mismatch")
+	require.Equal(t, xOTLP.Path, yOTLP.Path, "OTLP path mismatch")
+	require.Equal(t, xOTLP.Authentication.Basic.User.Value, yOTLP.Authentication.Basic.User.Value, "OTLP basic auth user mismatch")
+	require.Equal(t, xOTLP.Authentication.Basic.Password.Value, yOTLP.Authentication.Basic.Password.Value, "OTLP basic auth password mismatch")
+	require.Len(t, yOTLP.Headers, 2, "expected two headers")
+	require.Equal(t, xOTLP.Headers[0].Name, yOTLP.Headers[0].Name, "OTLP header name mismatch")
+	require.Equal(t, xOTLP.Headers[0].ValueType.Value, yOTLP.Headers[0].ValueType.Value, "OTLP header value mismatch")
+	require.Equal(t, xOTLP.Headers[0].Prefix, yOTLP.Headers[0].Prefix, "OTLP header prefix mismatch")
+	require.Equal(t, xOTLP.Headers[1].Name, yOTLP.Headers[1].Name, "OTLP header name mismatch")
+	require.Equal(t, xOTLP.Headers[1].ValueType.Value, yOTLP.Headers[1].ValueType.Value, "OTLP header value mismatch")
+	require.Equal(t, xOTLP.Headers[1].Prefix, yOTLP.Headers[1].Prefix, "OTLP header prefix mismatch")
+	require.Equal(t, xOTLP.TLS.Insecure, yOTLP.TLS.Insecure, "OTLP TLS insecure mismatch")
+	require.Equal(t, xOTLP.TLS.InsecureSkipVerify, yOTLP.TLS.InsecureSkipVerify, "OTLP TLS insecure skip verify mismatch")
+	require.Equal(t, xOTLP.TLS.CA.Value, yOTLP.TLS.CA.Value, "OTLP TLS CA mismatch")
+	require.Equal(t, xOTLP.TLS.Cert.Value, yOTLP.TLS.Cert.Value, "OTLP TLS cert mismatch")
+	require.Equal(t, xOTLP.TLS.Key.Value, yOTLP.TLS.Key.Value, "OTLP TLS key mismatch")
+
 	require.Equal(t, x.Status.UnsupportedMode, y.Status.UnsupportedMode, "status unsupported mode mismatch")
 	require.ElementsMatch(t, x.Status.Conditions, y.Status.Conditions, "status conditions mismatch")
 }