[8.16](backport #5826) Update otel samples to send otel native data

internal/pkg/otel/samples/darwin/logs_metrics_traces.yml

@@ -44,6 +44,239 @@ extensions:
   file_storage:
     directory: ${env:STORAGE_DIR}
 
+connectors:
+  signaltometrics:
+    logs:
+      - name: service_summary
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: metricset.name
+            default_value: service_summary
+        sum:
+          value: "1"
+    datapoints:
+      - name: service_summary
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: metricset.name
+            default_value: service_summary
+        sum:
+          value: "1"
+    spans:
+      - name: service_summary
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: metricset.name
+            default_value: service_summary
+        sum:
+          value: Int(AdjustedCount())
+      - name: transaction.duration.histogram
+        description: APM service transaction aggregated metrics as histogram
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: transaction.root
+          - key: transaction.type
+          - key: metricset.name
+            default_value: service_transaction
+          - key: elasticsearch.mapping.hints
+            default_value: [_doc_count]
+        unit: us
+        exponential_histogram:
+          value: Microseconds(end_time - start_time)
+      - name: transaction.duration.summary
+        description: APM service transaction aggregated metrics as summary
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: transaction.root
+          - key: transaction.type
+          - key: metricset.name
+            default_value: service_transaction
+          - key: elasticsearch.mapping.hints
+            default_value: [aggregate_metric_double]
+        unit: us
+        histogram:
+          buckets: [1]
+          value: Microseconds(end_time - start_time)
+      - name: transaction.duration.histogram
+        description: APM transaction aggregated metrics as histogram
+        ephemeral_resource_attribute: true
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+          - key: container.id
+          - key: k8s.pod.name
+          - key: service.version
+          - key: service.instance.id # service.node.name
+          - key: process.runtime.name # service.runtime.name
+          - key: process.runtime.version # service.runtime.version
+          - key: telemetry.sdk.version # service.language.version??
+          - key: host.name
+          - key: os.type # host.os.platform
+          - key: faas.instance
+          - key: faas.name
+          - key: faas.version
+          - key: cloud.provider
+          - key: cloud.region
+          - key: cloud.availability_zone
+          - key: cloud.platform # cloud.servicename
+          - key: cloud.account.id
+        attributes:
+          - key: transaction.root
+          - key: transaction.name
+          - key: transaction.type
+          - key: transaction.result
+          - key: event.outcome
+          - key: metricset.name
+            default_value: transaction
+          - key: elasticsearch.mapping.hints
+            default_value: [_doc_count]
+        unit: us
+        exponential_histogram:
+          value: Microseconds(end_time - start_time)
+      - name: transaction.duration.summary
+        description: APM transaction aggregated metrics as summary
+        ephemeral_resource_attribute: true
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+          - key: container.id
+          - key: k8s.pod.name
+          - key: service.version
+          - key: service.instance.id # service.node.name
+          - key: process.runtime.name # service.runtime.name
+          - key: process.runtime.version # service.runtime.version
+          - key: telemetry.sdk.version # service.language.version??
+          - key: host.name
+          - key: os.type # host.os.platform
+          - key: faas.instance
+          - key: faas.name
+          - key: faas.version
+          - key: cloud.provider
+          - key: cloud.region
+          - key: cloud.availability_zone
+          - key: cloud.platform # cloud.servicename
+          - key: cloud.account.id
+        attributes:
+          - key: transaction.root
+          - key: transaction.name
+          - key: transaction.type
+          - key: transaction.result
+          - key: event.outcome
+          - key: metricset.name
+            default_value: transaction
+          - key: elasticsearch.mapping.hints
+            default_value: [aggregate_metric_double]
+        unit: us
+        histogram:
+          buckets: [1]
+          value: Microseconds(end_time - start_time)
+      - name: span.destination.service.response_time.sum.us
+        description: APM span destination metrics
+        ephemeral_resource_attribute: true
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: span.name
+          - key: event.outcome
+          - key: service.target.type
+          - key: service.target.name
+          - key: span.destination.service.resource
+          - key: metricset.name
+            default_value: service_destination
+        unit: us
+        sum:
+          value: Double(Microseconds(end_time - start_time))
+      - name: span.destination.service.response_time.count
+        description: APM span destination metrics
+        ephemeral_resource_attribute: true
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: span.name
+          - key: event.outcome
+          - key: service.target.type
+          - key: service.target.name
+          - key: span.destination.service.resource
+          - key: metricset.name
+            default_value: service_destination
+        sum:
+          value: Int(AdjustedCount())
+      # event.success_count is populated using 2 metric definition with different conditions
+      # and value for the histogram bucket based on event outcome. Both metric definition
+      # are created using same name and attribute and will result in a single histogram.
+      # We use mapping hint of aggregate_metric_double, so, only the sum and the count
+      # values are required and the actual histogram bucket is ignored.
+      - name: event.success_count
+        description: Success count as a metric for service transaction
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: transaction.root
+          - key: transaction.type
+          - key: metricset.name
+            default_value: service_transaction
+          - key: elasticsearch.mapping.hints
+            default_value: [aggregate_metric_double]
+        conditions:
+          - attributes["event.outcome"] != nil and attributes["event.outcome"] == "success"
+        histogram:
+          buckets: [1]
+          count: Int(AdjustedCount())
+          value: Int(AdjustedCount())
+      - name: event.success_count
+        description: Success count as a metric for service transaction
+        include_resource_attributes:
+          - key: service.name
+          - key: deployment.environment # service.environment
+          - key: telemetry.sdk.language # service.language.name
+          - key: agent.name # set via elastictraceprocessor
+        attributes:
+          - key: transaction.root
+          - key: transaction.type
+          - key: metricset.name
+            default_value: service_transaction
+          - key: elasticsearch.mapping.hints
+            default_value: [aggregate_metric_double]
+        conditions:
+          - attributes["event.outcome"] != nil and attributes["event.outcome"] != "success"
+        histogram:
+          buckets: [0]
+          count: Int(AdjustedCount())
+          value: Double(0)
+
 processors:
   elasticinframetrics:
   resourcedetection:
@@ -89,14 +322,27 @@ processors:
       - key: process.executable.path
         action: delete
 
-exporters:
-  otlp/apm:
-    endpoint: "${env:APM_ENDPOINT}"
-    headers:
-    # Elastic APM Server secret token or API key
-      Authorization: "Bearer ${env:APM_SECRET_KEY}"
+  elastictrace:
+  lsminterval:
+    intervals:
+      - duration: 1m
+        statements:
+          - set(resource.attributes["metricset.interval"], "1m")
+          - set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "1m"], "."))
+          - set(attributes["processor.event"], "metric")
+      - duration: 10m
+        statements:
+          - set(resource.attributes["metricset.interval"], "10m")
+          - set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "10m"], "."))
+          - set(attributes["processor.event"], "metric")
+      - duration: 60m
+        statements:
+          - set(resource.attributes["metricset.interval"], "60m")
+          - set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "60m"], "."))
+          - set(attributes["processor.event"], "metric")
 
-  elasticsearch:
+exporters:
+  elasticsearch/ecs:
     endpoints: ["${env:ELASTIC_ENDPOINT}"]
     api_key: ${env:ELASTIC_API_KEY}
     mapping:
@@ -108,30 +354,47 @@ exporters:
     traces_dynamic_index:
       enabled: true
 
+  elasticsearch/otel:
+    endpoints: [ "${env:ELASTIC_ENDPOINT}" ]
+    api_key: ${env:ELASTIC_API_KEY}
+    mapping:
+      mode: otel
+    logs_dynamic_index:
+      enabled: true
+    metrics_dynamic_index:
+      enabled: true
+    traces_dynamic_index:
+      enabled: true
+
 service:
   extensions: [file_storage]
   pipelines:
     traces/fromsdk:
       receivers: [otlp/fromsdk]
-      processors: []
-      exporters: [otlp/apm]
+      processors: [elastictrace]
+      exporters: [signaltometrics, elasticsearch/otel]
 
     metrics/fromsdk:
       receivers: [otlp/fromsdk]
       processors: []
-      exporters: [otlp/apm]
+      exporters: [signaltometrics, elasticsearch/otel]
 
     metrics/hostmetrics:
       receivers: [hostmetrics/system]
       processors: [elasticinframetrics, resourcedetection, attributes/dataset, resource/process]
-      exporters: [elasticsearch]
+      exporters: [elasticsearch/ecs]
+
+    metrics/aggregated-metrics:
+      receivers: [signaltometrics]
+      processors: [lsminterval]
+      exporters: [elasticsearch/otel]
 
     logs/fromsdk:
       receivers: [otlp/fromsdk]
       processors: []
-      exporters: [otlp/apm]
+      exporters: [signaltometrics, elasticsearch/otel]
 
     logs/platformlogs:
       receivers: [filelog/platformlogs]
       processors: [resourcedetection]
-      exporters: [elasticsearch]
+      exporters: [elasticsearch/otel]

internal/pkg/otel/samples/darwin/platformlogs.yml

@@ -11,7 +11,7 @@ receivers:
 extensions:
   file_storage:
     directory: ${env:STORAGE_DIR}
-  
+
 processors:
   resourcedetection:
     detectors: ["system"]
@@ -53,11 +53,11 @@ exporters:
     sampling_thereafter: 1000
 
   # Exporter to send logs and metrics to Elasticsearch
-  elasticsearch:
+  elasticsearch/otel:
     endpoints: ["${env:ELASTIC_ENDPOINT}"]
     api_key: ${env:ELASTIC_API_KEY}
     mapping:
-      mode: ecs
+      mode: otel
     logs_dynamic_index:
       enabled: true
     metrics_dynamic_index:
@@ -71,4 +71,4 @@ service:
     logs/platformlogs:
       receivers: [filelog/platformlogs]
       processors: [resourcedetection]
-      exporters: [debug, elasticsearch]
+      exporters: [debug, elasticsearch/otel]

internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml

@@ -85,7 +85,7 @@ processors:
 
 exporters:
   # Exporter to send logs and metrics to Elasticsearch
-  elasticsearch:
+  elasticsearch/ecs:
     endpoints: ["${env:ELASTIC_ENDPOINT}"]
     api_key: ${env:ELASTIC_API_KEY}
     mapping:
@@ -96,15 +96,26 @@ exporters:
       enabled: true
     traces_dynamic_index:
       enabled: true
+  elasticsearch/otel:
+    endpoints: [ "${env:ELASTIC_ENDPOINT}" ]
+    api_key: ${env:ELASTIC_API_KEY}
+    mapping:
+      mode: otel
+    logs_dynamic_index:
+      enabled: true
+    metrics_dynamic_index:
+      enabled: true
+    traces_dynamic_index:
+      enabled: true
 
 service:
   extensions: [file_storage]
   pipelines:
     metrics/hostmetrics:
       receivers: [hostmetrics/system]
       processors: [elasticinframetrics, resourcedetection, attributes/dataset, resource/process]
-      exporters: [elasticsearch]
+      exporters: [elasticsearch/ecs]
     logs/platformlogs:
       receivers: [filelog/platformlogs]
       processors: [resourcedetection]
-      exporters: [elasticsearch]
+      exporters: [elasticsearch/otel]