feat(metrics): Define allowlist for histogram metrics

SumoLogic · Aug 7, 2024 · 9a30fba · 9a30fba
1 parent b6c474c
commit 9a30fba
Show file tree

Hide file tree

Showing 15 changed files with 41 additions and 29 deletions.
diff --git a/.changelog/3821.added.txt b/.changelog/3821.added.txt
@@ -0,0 +1 @@
+feat(metrics): Define allowlist for histogram metrics
diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md
@@ -148,6 +148,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `sumologic.metrics.collector.otelcol.config.override`                                 | Configuration for otelcol metrics collector, replaces defaults. See also https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/configuration.md.                                                                                                                                            | {}                                                                                                                                                                                   |
 | `sumologic.metrics.collector.otelcol.targetAllocator.resources`                       | Resource requests and limits for Metrics Collector Target Allocator.                                                                                                                                                                                                                                       | {}                                                                                                                                                                                   |
 | `sumologic.metrics.dropHistogramBuckets`                                              | Drop buckets from histogram and summary metrics, leaving only the sum and count components.                                                                                                                                                                                                                | `true`                                                                                                                                                                               |
+| `sumologic.metrics.allowHistogramRegex`                                               | Allowlist for Histogram metrics, including the buckets                                                                                                                                                                                                                                                     | `""`                                                                                                                                                                                 |
 | `sumologic.metrics.sourceType`                                                        | The type of the Sumo Logic source being used for metrics ingestion. Can be `http` or `otlp`.                                                                                                                                                                                                               | `otlp`                                                                                                                                                                               |
 | `sumologic.traces.enabled`                                                            | Set the enabled flag to true to enable tracing ingestion. _Tracing must be enabled for the account first. Please contact your Sumo representative for activation details_                                                                                                                                  | `true`                                                                                                                                                                               |
 | `sumologic.traces.spans_per_request`                                                  | Maximum number of spans sent in single batch                                                                                                                                                                                                                                                               | `100`                                                                                                                                                                                |

diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
@@ -57,8 +57,8 @@ processors:
     metric_statements:
       - context: metric
         statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, {{ .Values.sumologic.metrics.allowHistogramRegex | quote }})) and (type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, {{ .Values.sumologic.metrics.allowHistogramRegex | quote }})) and (type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
 {{- end }}
 
 receivers:

diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml
@@ -16,7 +16,7 @@ filter/drop_unnecessary_metrics:
       - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
 {{- if .Values.sumologic.metrics.dropHistogramBuckets }}
       # drop histograms we've extracted sums and counts from, but don't want the full thing
-      - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+      - (not IsMatch(name, {{ .Values.sumologic.metrics.allowHistogramRegex | quote }})) and (type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket"))
 {{- end }}
 
 # Prometheus receiver puts all labels in record-level attributes, and we need them in resource

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -599,6 +599,9 @@ sumologic:
     ## - kubelet_runtime_operations_duration_seconds
     dropHistogramBuckets: true
 
+    # A regular expression to allow selected histogram metrics, including the buckets.
+    allowHistogramRegex: ""
+
     ## A regular expression for namespaces.
     ## Metrics that match these namespaces will be excluded from Sumo.
     excludeNamespaceRegex: ""

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml
@@ -44,8 +44,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type ==
+            METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml
@@ -44,8 +44,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type ==
+            METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml
@@ -121,8 +121,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type ==
+            METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/debug.output.yaml
@@ -46,8 +46,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type ==
+            METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/debug_with_sumologic_mock.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/debug_with_sumologic_mock.output.yaml
@@ -69,8 +69,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type ==
+            METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

diff --git a/.../helm/testdata/goldenfile/metadata_metrics_otc/debug_with_sumologic_mock_http.output.yaml b/.../helm/testdata/goldenfile/metadata_metrics_otc/debug_with_sumologic_mock_http.output.yaml
@@ -146,8 +146,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type ==
+            METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml
@@ -68,8 +68,9 @@ data:
         metrics:
           metric:
           - resource.attributes["job"] != "pod-annotations" and IsMatch(name, "scrape_.*")
-          - type == METRIC_DATA_TYPE_HISTOGRAM or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM
-            or type == METRIC_DATA_TYPE_SUMMARY or IsMatch(name, ".*_bucket")
+          - (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM or type ==
+            METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+            or IsMatch(name, ".*_bucket"))
       groupbyattrs:
         keys:
         - container

diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
@@ -115,10 +115,10 @@ spec:
         metric_statements:
         - context: metric
           statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
     receivers:
       prometheus:
         config:

diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/debug.output.yaml
@@ -117,10 +117,10 @@ spec:
         metric_statements:
         - context: metric
           statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
     receivers:
       prometheus:
         config:

diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/kubelet.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/kubelet.output.yaml
@@ -115,10 +115,10 @@ spec:
         metric_statements:
         - context: metric
           statements:
-          - extract_sum_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
-          - extract_count_metric(true) where type == METRIC_DATA_TYPE_HISTOGRAM or type
-            == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY
+          - extract_sum_metric(true) where (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
+          - extract_count_metric(true) where (not IsMatch(name, "")) and (type == METRIC_DATA_TYPE_HISTOGRAM
+            or type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM or type == METRIC_DATA_TYPE_SUMMARY)
     receivers:
       prometheus:
         config: