Update the log level for metric scrape failures of the smartagent/kub…

…ernetes-proxy receiver from error to debug (#832) Update the log level for metric scrape failures of the smartagent/kubernetes-proxy receiver from error to debug when distribution=''
signalfx · Jun 27, 2023 · 3ec2dfc · 3ec2dfc
1 parent 06d3588
commit 3ec2dfc
Show file tree

Hide file tree

Showing 35 changed files with 50 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@ This Splunk OpenTelemetry Collector for Kubernetes release adopts the [Splunk Op
 - Upgrade the Splunk OpenTelemetry Collector for Kubernetes subchart dependencies [#828](https://github.com/signalfx/splunk-otel-collector-chart/pull/828)
   - cert-manager upgraded from 1.11.1 to [1.12.2](https://github.com/cert-manager/cert-manager/releases/tag/v1.12.2)
   - opentelemetry-operator upgraded from 0.28.0 to [0.32.0)](https://github.com/open-telemetry/opentelemetry-helm-charts/releases/tag/opentelemetry-operator-0.32.0)
+- Update the log level for metric scrape failures of the smartagent/kubernetes-proxy receiver from error to debug when distribution='' [#832](https://github.com/signalfx/splunk-otel-collector-chart/pull/832)
 
 ## [0.79.1] - 2023-06-22
 

diff --git a/docs/advanced-configuration.md b/docs/advanced-configuration.md
@@ -379,14 +379,19 @@ agent:
 ### Known issues
 
 Kube Proxy
-* https://github.com/kubernetes/kops/issues/6472
-  * Problem
-    * When using a kops created Kubernetes cluster, a network connectivity issue has been reported that prevents proxy
-      metrics from being collected.
+* `10249: connect: connection refused`
+  * Issue
+    * When using a Kubernetes cluster with non-default configurations for kube proxy, there is a reported network connectivity issue that prevents the collection of proxy metrics.
   * Solution
-    * This issue can be addressed updating the kubeProxy metric bind address in the kops cluster spec:
-      * Set "kubeProxy.metricsBindAddress: 0.0.0.0" in the kops cluster spec.
-      * Deploy the change with "kops update cluster {cluster_name}" and "kops rolling-update cluster {cluster_name}".
+    * Update the kube proxy metric bind address (--metrics-bind-address) in the cluster spec.
+Set the kubeProxy metrics bind address to 0.0.0.0 or another value based on your Kubernetes cluster distribution.
+For this particular issue, the solution may vary depending on the Kubernetes cluster distribution. It is recommended to research what your Kubernetes distribution recommends for addressing this issue.
+  * Related Issue Links
+    * [kubernetes - Expose kube-proxy metrics on 0.0.0.0 by default ](https://github.com/kubernetes/kubernetes/pull/74300)
+    * [kubernetes - kube-proxy TLS support](https://github.com/kubernetes/kubernetes/issues/106870)
+    * [splunk-otel-collector-chart - Error connecting to kubernetes-proxy](https://github.com/signalfx/splunk-otel-collector-chart/issues/758)
+    * [kops - expose metrics-bind-address configuration for kube-proxy](https://github.com/kubernetes/kops/issues/6472)
+    * [prometheus - prometheus-kube-stack - kube-proxy metrics status with connection refused](https://github.com/prometheus-community/helm-charts/issues/977)
 
 ## Logs collection
 

diff --git a/examples/add-filter-processor/rendered_manifests/configmap-agent.yaml b/examples/add-filter-processor/rendered_manifests/configmap-agent.yaml
@@ -268,6 +268,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/add-filter-processor/rendered_manifests/daemonset.yaml b/examples/add-filter-processor/rendered_manifests/daemonset.yaml
@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: acf85b8fd8567f93c7aa10027e8a1670bfaba0deadcb3fa76fbf30c6a261fd78
+        checksum/config: 0d5e81dde2b89bae6cf60beb7443678fbfa3f7916aa235bbd32ef7156494f775
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/add-receiver-creator/rendered_manifests/configmap-agent.yaml b/examples/add-receiver-creator/rendered_manifests/configmap-agent.yaml
@@ -215,6 +215,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/add-receiver-creator/rendered_manifests/daemonset.yaml b/examples/add-receiver-creator/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 81b9a50255c250df5b28e166d5757aa2f6874582c04c8f481d0fd6a128859199
+        checksum/config: 5ff17e00f88e15acf0f5c507c9b32894fddf88b01f09745155e1d0fad185c324
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/add-sampler/rendered_manifests/configmap-agent.yaml b/examples/add-sampler/rendered_manifests/configmap-agent.yaml
@@ -212,6 +212,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/add-sampler/rendered_manifests/daemonset.yaml b/examples/add-sampler/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: f3332dc5cefba1eaead28a2cd0d13f8d0e9cfc90315490d0f00f7b83aa7ff7fd
+        checksum/config: f3406d9e0e942c956df031f76fb02081c51d7b34ffb7c9aa39f81be6772e7a05
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/autodetect-istio/rendered_manifests/configmap-agent.yaml b/examples/autodetect-istio/rendered_manifests/configmap-agent.yaml
@@ -260,6 +260,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/autodetect-istio/rendered_manifests/daemonset.yaml b/examples/autodetect-istio/rendered_manifests/daemonset.yaml
@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 22405d5dcb6df5e5fe9659e825eb000d3d9659707d4d4993e98685ac6f6b779c
+        checksum/config: d664606cbf1f109b692576cd806fdce467655d1e42e55ced0e9095f567b58771
         kubectl.kubernetes.io/default-container: otel-collector
         sidecar.istio.io/inject: "false"
     spec:

diff --git a/examples/collector-agent-only/rendered_manifests/configmap-agent.yaml b/examples/collector-agent-only/rendered_manifests/configmap-agent.yaml
@@ -209,6 +209,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/collector-agent-only/rendered_manifests/daemonset.yaml b/examples/collector-agent-only/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 8c7fd14875e411d30c6fd321e11d4ad6c14afbbc5aac856bb6a3b478579c4d41
+        checksum/config: 51255f7bce9baabbbf67f49087059580b307a3c5571854db33cb7b00f3f862f7
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/collector-all-modes/rendered_manifests/configmap-agent.yaml b/examples/collector-all-modes/rendered_manifests/configmap-agent.yaml
@@ -195,6 +195,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/collector-all-modes/rendered_manifests/daemonset.yaml b/examples/collector-all-modes/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: e1f90a5b31a22c6890747f9658b61b6246ec147f1293765c20107512a4af914e
+        checksum/config: f9ef5b2dcc6c5ea66916acf315f96346be3f871e378d377c5c922289fbbddf92
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/crio-logging/rendered_manifests/configmap-agent.yaml b/examples/crio-logging/rendered_manifests/configmap-agent.yaml
@@ -209,6 +209,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/crio-logging/rendered_manifests/daemonset.yaml b/examples/crio-logging/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 8c7fd14875e411d30c6fd321e11d4ad6c14afbbc5aac856bb6a3b478579c4d41
+        checksum/config: 51255f7bce9baabbbf67f49087059580b307a3c5571854db33cb7b00f3f862f7
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/default/rendered_manifests/configmap-agent.yaml b/examples/default/rendered_manifests/configmap-agent.yaml
@@ -209,6 +209,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/default/rendered_manifests/daemonset.yaml b/examples/default/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 8c7fd14875e411d30c6fd321e11d4ad6c14afbbc5aac856bb6a3b478579c4d41
+        checksum/config: 51255f7bce9baabbbf67f49087059580b307a3c5571854db33cb7b00f3f862f7
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/enable-operator-and-auto-instrumentation/rendered_manifests/configmap-agent.yaml b/examples/enable-operator-and-auto-instrumentation/rendered_manifests/configmap-agent.yaml
@@ -222,6 +222,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/enable-operator-and-auto-instrumentation/rendered_manifests/daemonset.yaml b/examples/enable-operator-and-auto-instrumentation/rendered_manifests/daemonset.yaml
@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 73a646d4229e25be8250c6289f8e118aee8589e128599f8e9915b87a10c44210
+        checksum/config: e82bab9a1a5a64c9c2c95f1fb0fbaa2128174f9d66574b47809bae926d9c527b
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/enabled-pprof-extension/rendered_manifests/configmap-agent.yaml b/examples/enabled-pprof-extension/rendered_manifests/configmap-agent.yaml
@@ -210,6 +210,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/enabled-pprof-extension/rendered_manifests/daemonset.yaml b/examples/enabled-pprof-extension/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: c6987712e8d37650dc50386a7814a3f23cef6c11283927ceba32cf7269453fb5
+        checksum/config: a560c670f80a5fe39697bb880f70e6af50ffc9759ee8a09d9e6cbdb5480eb2a6
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/configmap-agent.yaml b/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/configmap-agent.yaml
@@ -217,6 +217,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/daemonset.yaml b/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/daemonset.yaml
@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 304bc9eddc453b88e355a9a350b1d195e7973c942cb52a9490e6b206bbef12d1
+        checksum/config: 04a6c8511fee6a54c7f1368df1cfbb90e7d8eb78989fcd7e2d27ee0f8fd26a44
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/kubernetes-windows-nodes/rendered_manifests/configmap-agent.yaml b/examples/kubernetes-windows-nodes/rendered_manifests/configmap-agent.yaml
@@ -222,6 +222,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/kubernetes-windows-nodes/rendered_manifests/daemonset.yaml b/examples/kubernetes-windows-nodes/rendered_manifests/daemonset.yaml
@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: b1a11d2e331e74c87f99b7052b3f60dfd444d0eb305c630ec50f4cf514259ffe
+        checksum/config: a0675a23e7de0c43c110b15fdb7b702093ec5a592e1fa2a6f0a613fc8203f478
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       dnsPolicy: ClusterFirstWithHostNet

diff --git a/examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml b/examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml
@@ -216,6 +216,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/only-metrics-platform/rendered_manifests/daemonset.yaml b/examples/only-metrics-platform/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: b225aba42e0123a33594314517f27c88a692e5c39ecfc2ec8c533b8acdafd492
+        checksum/config: 915fdd3ba4159ed02f375bbac3a518479a2ddb9dabc4e489084430458b657eec
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/only-metrics/rendered_manifests/configmap-agent.yaml b/examples/only-metrics/rendered_manifests/configmap-agent.yaml
@@ -200,6 +200,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/only-metrics/rendered_manifests/daemonset.yaml b/examples/only-metrics/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 37808ef183178056f0957ee73878c9a1171dc09ce60466c5a15092cf06abe1e3
+        checksum/config: b99e72496b5965538304a4b73241b0a28373ac6bd6da666cc37eeead18e96a5c
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/...es/route-data-through-gateway-deployed-separately/rendered_manifests/configmap-agent.yaml b/...es/route-data-through-gateway-deployed-separately/rendered_manifests/configmap-agent.yaml
@@ -213,6 +213,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/route-data-through-gateway-deployed-separately/rendered_manifests/daemonset.yaml b/examples/route-data-through-gateway-deployed-separately/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: e445b88f98dbbebbf4df3e4bbffbf3ebffcdde354206043c96ae168a5fa155b2
+        checksum/config: e46f5efeb437d8e6f5283379f8d02fc50c08ab489d9040af5f62943305ca8416
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/examples/use-proxy/rendered_manifests/configmap-agent.yaml b/examples/use-proxy/rendered_manifests/configmap-agent.yaml
@@ -209,6 +209,7 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:

diff --git a/examples/use-proxy/rendered_manifests/daemonset.yaml b/examples/use-proxy/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 8c7fd14875e411d30c6fd321e11d4ad6c14afbbc5aac856bb6a3b478579c4d41
+        checksum/config: 51255f7bce9baabbbf67f49087059580b307a3c5571854db33cb7b00f3f862f7
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
@@ -181,6 +181,11 @@ receivers:
           extraDimensions:
             metric_source: kubernetes-proxy
           type: kubernetes-proxy
+          # Connecting to kube proxy in unknown Kubernetes distributions can be troublesome and generate log noise
+          # For now, set the scrape failure log level to debug when no specific distribution is selected
+          {{- if eq .Values.distribution "" }}
+          scrapeFailureLogLevel: debug
+          {{- end }}
           {{- if eq .Values.distribution "openshift" }}
           skipVerify: true
           useHTTPS: true