feat(metrics): Define allowlist for histogram metrics

.changelog/3821.added.txt

@@ -0,0 +1 @@
+feat(metrics): Define allowlist for histogram metrics

deploy/helm/sumologic/README.md

@@ -148,6 +148,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `sumologic.metrics.collector.otelcol.config.override`                                 | Configuration for otelcol metrics collector, replaces defaults. See also https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/configuration.md.                                                                                                                                            | {}                                                                                                                                                                                   |
 | `sumologic.metrics.collector.otelcol.targetAllocator.resources`                       | Resource requests and limits for Metrics Collector Target Allocator.                                                                                                                                                                                                                                       | {}                                                                                                                                                                                   |
 | `sumologic.metrics.dropHistogramBuckets`                                              | Drop buckets from histogram and summary metrics, leaving only the sum and count components.                                                                                                                                                                                                                | `true`                                                                                                                                                                               |
+| `sumologic.metrics.allowHistogramRegex`                                               | Allowlist for Histogram metrics, including the buckets                                                                                                                                                                                                                                                     | `"^$"`                                                                                                                                                                               |
 | `sumologic.metrics.sourceType`                                                        | The type of the Sumo Logic source being used for metrics ingestion. Can be `http` or `otlp`.                                                                                                                                                                                                               | `otlp`                                                                                                                                                                               |
 | `sumologic.traces.enabled`                                                            | Set the enabled flag to true to enable tracing ingestion. _Tracing must be enabled for the account first. Please contact your Sumo representative for activation details_                                                                                                                                  | `true`                                                                                                                                                                               |
 | `sumologic.traces.spans_per_request`                                                  | Maximum number of spans sent in single batch                                                                                                                                                                                                                                                               | `100`                                                                                                                                                                                |

deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml

@@ -57,8 +57,8 @@ processors:
     metric_statements:
       - context: metric
         statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, {{ .Values.sumologic.metrics.allowHistogramRegex | quote }})) and (type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, {{ .Values.sumologic.metrics.allowHistogramRegex | quote }})) and (type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
 {{- end }}
 
 receivers:

deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml

@@ -16,7 +16,7 @@ filter/drop_unnecessary_metrics:
       - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
 {{- if .Values.sumologic.metrics.dropHistogramBuckets }}
       # drop histograms we've extracted sums and counts from, but don't want the full thing
-      - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+      - (not IsMatch(name, {{ .Values.sumologic.metrics.allowHistogramRegex | quote }})) and (type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket"))
 {{- end }}
 
 # Prometheus receiver puts all labels in record-level attributes, and we need them in resource

deploy/helm/sumologic/values.yaml

@@ -599,6 +599,9 @@ sumologic:
     ## - kubelet_runtime_operations_duration_seconds
     dropHistogramBuckets: true
 
+    ## A regular expression to allow selected histogram metrics, including the buckets.
+    allowHistogramRegex: "^$"
+
     ## A regular expression for namespaces.
     ## Metrics that match these namespaces will be excluded from Sumo.
     excludeNamespaceRegex: ""

tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml

@@ -44,8 +44,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type
+            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

tests/helm/testdata/goldenfile/metadata_metrics_otc/allow_histograms.input.yaml

@@ -0,0 +1,3 @@
+sumologic:
+  metrics:
+    allowHistogramRegex: "^(apiserver_request_duration_seconds)$"

tests/helm/testdata/goldenfile/metadata_metrics_otc/allow_histograms.output.yaml

@@ -0,0 +1,204 @@
+---
+# Source: sumologic/templates/metrics/otelcol/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: RELEASE-NAME-sumologic-otelcol-metrics
+  namespace: sumologic
+  labels:
+    app: RELEASE-NAME-sumologic-otelcol-metrics
+    chart: "sumologic-%CURRENT_CHART_VERSION%"
+    release: "RELEASE-NAME"
+    heritage: "Helm"
+data:
+  config.yaml: |
+    exporters:
+      sumologic/default:
+        client: k8s_%CURRENT_CHART_VERSION%
+        decompose_otlp_histograms: true
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE}
+        max_request_body_size: 16777216
+        metric_format: otlp
+        sending_queue:
+          enabled: true
+          num_consumers: 10
+          queue_size: 10000
+          storage: file_storage
+        timeout: 30s
+    extensions:
+      file_storage:
+        compaction:
+          directory: /tmp
+          on_rebound: true
+        directory: /var/lib/storage/otc
+        timeout: 10s
+      health_check: {}
+      pprof: {}
+    processors:
+      batch:
+        send_batch_max_size: 2048
+        send_batch_size: 1024
+        timeout: 1s
+      filter/drop_unnecessary_metrics:
+        error_mode: ignore
+        metrics:
+          metric:
+          - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
+          - (not IsMatch(name, "^(apiserver_request_duration_seconds)$")) and (type ==
+            METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
+            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket"))
+      groupbyattrs:
+        keys:
+        - container
+        - namespace
+        - pod
+        - service
+      groupbyattrs/group_by_name:
+        keys:
+        - __name__
+        - job
+      k8s_tagger:
+        extract:
+          delimiter: _
+          labels:
+          - key: '*'
+            tag_name: pod_labels_%s
+          metadata:
+          - daemonSetName
+          - deploymentName
+          - nodeName
+          - replicaSetName
+          - serviceName
+          - statefulSetName
+        owner_lookup_enabled: true
+        passthrough: false
+        pod_association:
+        - from: build_hostname
+      memory_limiter:
+        check_interval: 5s
+        limit_percentage: 90
+        spike_limit_percentage: 20
+      metricstransform:
+        transforms:
+          action: update
+          include: ^prometheus_remote_write_(.*)$$
+          match_type: regexp
+          new_name: $$1
+      resource:
+        attributes:
+        - action: upsert
+          from_attribute: namespace
+          key: k8s.namespace.name
+        - action: delete
+          key: namespace
+        - action: upsert
+          from_attribute: pod
+          key: k8s.pod.name
+        - action: delete
+          key: pod
+        - action: upsert
+          from_attribute: container
+          key: k8s.container.name
+        - action: delete
+          key: container
+        - action: upsert
+          from_attribute: node
+          key: k8s.node.name
+        - action: delete
+          key: node
+        - action: upsert
+          from_attribute: service
+          key: prometheus_service
+        - action: delete
+          key: service
+        - action: upsert
+          from_attribute: service.name
+          key: job
+        - action: delete
+          key: service.name
+        - action: upsert
+          key: _origin
+          value: kubernetes
+        - action: upsert
+          key: cluster
+          value: kubernetes
+      resource/delete_source_metadata:
+        attributes:
+        - action: delete
+          key: _sourceCategory
+        - action: delete
+          key: _sourceHost
+        - action: delete
+          key: _sourceName
+      resource/remove_k8s_pod_pod_name:
+        attributes:
+        - action: delete
+          key: k8s.pod.pod_name
+      source:
+        collector: kubernetes
+        exclude:
+          k8s.namespace.name: ""
+      sumologic:
+        add_cloud_namespace: false
+      transform/remove_name:
+        error_mode: ignore
+        metric_statements:
+        - context: resource
+          statements:
+          - delete_key(attributes, "__name__")
+      transform/set_name:
+        error_mode: ignore
+        metric_statements:
+        - context: datapoint
+          statements:
+          - set(attributes["__name__"], metric.name) where IsMatch(metric.name, "^cloudprovider_.*")
+    receivers:
+      otlp:
+        protocols:
+          http:
+            endpoint: 0.0.0.0:4318
+      telegraf:
+        agent_config: |
+          [agent]
+            interval = "30s"
+            flush_interval = "30s"
+            omit_hostname = true
+          [[inputs.http_listener_v2]]
+            # wait longer than prometheus
+            read_timeout = "30s"
+            write_timeout = "30s"
+            service_address = ":9888"
+            data_format = "prometheusremotewrite"
+            paths = [
+              "/prometheus.metrics"
+            ]
+    service:
+      extensions:
+      - health_check
+      - file_storage
+      - pprof
+      pipelines:
+        metrics:
+          exporters:
+          - sumologic/default
+          processors:
+          - memory_limiter
+          - metricstransform
+          - groupbyattrs
+          - resource
+          - k8s_tagger
+          - source
+          - sumologic
+          - resource/remove_k8s_pod_pod_name
+          - resource/delete_source_metadata
+          - transform/set_name
+          - groupbyattrs/group_by_name
+          - transform/remove_name
+          - filter/drop_unnecessary_metrics
+          - batch
+          receivers:
+          - telegraf
+          - otlp
+      telemetry:
+        logs:
+          level: info

tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml

@@ -44,8 +44,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type
+            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml

@@ -121,8 +121,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type
+            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

tests/helm/testdata/goldenfile/metadata_metrics_otc/debug.output.yaml

@@ -46,8 +46,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type
+            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

tests/helm/testdata/goldenfile/metadata_metrics_otc/debug_with_sumologic_mock.output.yaml

@@ -69,8 +69,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type
+            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

tests/helm/testdata/goldenfile/metadata_metrics_otc/debug_with_sumologic_mock_http.output.yaml

@@ -146,8 +146,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type
+            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml

@@ -68,8 +68,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type
+            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml

@@ -115,10 +115,10 @@ spec:
         metric_statements:
         - context: metric
           statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
     receivers:
       prometheus:
         config:

tests/helm/testdata/goldenfile/metrics_collector_otc/debug.output.yaml

@@ -117,10 +117,10 @@ spec:
         metric_statements:
         - context: metric
           statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
     receivers:
       prometheus:
         config:

tests/helm/testdata/goldenfile/metrics_collector_otc/kubelet.output.yaml

@@ -115,10 +115,10 @@ spec:
         metric_statements:
         - context: metric
           statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, "^$")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
     receivers:
       prometheus:
         config: