Skip to content

Commit

Permalink
Update otel samples to send otel native data (#5826) (#5836)
Browse files Browse the repository at this point in the history
(cherry picked from commit 100f433)

Co-authored-by: Carson Ip <[email protected]>
  • Loading branch information
mergify[bot] and carsonip authored Oct 28, 2024
1 parent f30fdc4 commit fc75656
Show file tree
Hide file tree
Showing 6 changed files with 587 additions and 40 deletions.
289 changes: 276 additions & 13 deletions internal/pkg/otel/samples/darwin/logs_metrics_traces.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,239 @@ extensions:
file_storage:
directory: ${env:STORAGE_DIR}

connectors:
signaltometrics:
logs:
- name: service_summary
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: metricset.name
default_value: service_summary
sum:
value: "1"
datapoints:
- name: service_summary
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: metricset.name
default_value: service_summary
sum:
value: "1"
spans:
- name: service_summary
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: metricset.name
default_value: service_summary
sum:
value: Int(AdjustedCount())
- name: transaction.duration.histogram
description: APM service transaction aggregated metrics as histogram
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: transaction.root
- key: transaction.type
- key: metricset.name
default_value: service_transaction
- key: elasticsearch.mapping.hints
default_value: [_doc_count]
unit: us
exponential_histogram:
value: Microseconds(end_time - start_time)
- name: transaction.duration.summary
description: APM service transaction aggregated metrics as summary
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: transaction.root
- key: transaction.type
- key: metricset.name
default_value: service_transaction
- key: elasticsearch.mapping.hints
default_value: [aggregate_metric_double]
unit: us
histogram:
buckets: [1]
value: Microseconds(end_time - start_time)
- name: transaction.duration.histogram
description: APM transaction aggregated metrics as histogram
ephemeral_resource_attribute: true
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
- key: container.id
- key: k8s.pod.name
- key: service.version
- key: service.instance.id # service.node.name
- key: process.runtime.name # service.runtime.name
- key: process.runtime.version # service.runtime.version
- key: telemetry.sdk.version # service.language.version??
- key: host.name
- key: os.type # host.os.platform
- key: faas.instance
- key: faas.name
- key: faas.version
- key: cloud.provider
- key: cloud.region
- key: cloud.availability_zone
- key: cloud.platform # cloud.servicename
- key: cloud.account.id
attributes:
- key: transaction.root
- key: transaction.name
- key: transaction.type
- key: transaction.result
- key: event.outcome
- key: metricset.name
default_value: transaction
- key: elasticsearch.mapping.hints
default_value: [_doc_count]
unit: us
exponential_histogram:
value: Microseconds(end_time - start_time)
- name: transaction.duration.summary
description: APM transaction aggregated metrics as summary
ephemeral_resource_attribute: true
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
- key: container.id
- key: k8s.pod.name
- key: service.version
- key: service.instance.id # service.node.name
- key: process.runtime.name # service.runtime.name
- key: process.runtime.version # service.runtime.version
- key: telemetry.sdk.version # service.language.version??
- key: host.name
- key: os.type # host.os.platform
- key: faas.instance
- key: faas.name
- key: faas.version
- key: cloud.provider
- key: cloud.region
- key: cloud.availability_zone
- key: cloud.platform # cloud.servicename
- key: cloud.account.id
attributes:
- key: transaction.root
- key: transaction.name
- key: transaction.type
- key: transaction.result
- key: event.outcome
- key: metricset.name
default_value: transaction
- key: elasticsearch.mapping.hints
default_value: [aggregate_metric_double]
unit: us
histogram:
buckets: [1]
value: Microseconds(end_time - start_time)
- name: span.destination.service.response_time.sum.us
description: APM span destination metrics
ephemeral_resource_attribute: true
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: span.name
- key: event.outcome
- key: service.target.type
- key: service.target.name
- key: span.destination.service.resource
- key: metricset.name
default_value: service_destination
unit: us
sum:
value: Double(Microseconds(end_time - start_time))
- name: span.destination.service.response_time.count
description: APM span destination metrics
ephemeral_resource_attribute: true
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: span.name
- key: event.outcome
- key: service.target.type
- key: service.target.name
- key: span.destination.service.resource
- key: metricset.name
default_value: service_destination
sum:
value: Int(AdjustedCount())
# event.success_count is populated using 2 metric definition with different conditions
# and value for the histogram bucket based on event outcome. Both metric definition
# are created using same name and attribute and will result in a single histogram.
# We use mapping hint of aggregate_metric_double, so, only the sum and the count
# values are required and the actual histogram bucket is ignored.
- name: event.success_count
description: Success count as a metric for service transaction
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: transaction.root
- key: transaction.type
- key: metricset.name
default_value: service_transaction
- key: elasticsearch.mapping.hints
default_value: [aggregate_metric_double]
conditions:
- attributes["event.outcome"] != nil and attributes["event.outcome"] == "success"
histogram:
buckets: [1]
count: Int(AdjustedCount())
value: Int(AdjustedCount())
- name: event.success_count
description: Success count as a metric for service transaction
include_resource_attributes:
- key: service.name
- key: deployment.environment # service.environment
- key: telemetry.sdk.language # service.language.name
- key: agent.name # set via elastictraceprocessor
attributes:
- key: transaction.root
- key: transaction.type
- key: metricset.name
default_value: service_transaction
- key: elasticsearch.mapping.hints
default_value: [aggregate_metric_double]
conditions:
- attributes["event.outcome"] != nil and attributes["event.outcome"] != "success"
histogram:
buckets: [0]
count: Int(AdjustedCount())
value: Double(0)

processors:
elasticinframetrics:
resourcedetection:
Expand Down Expand Up @@ -89,14 +322,27 @@ processors:
- key: process.executable.path
action: delete

exporters:
otlp/apm:
endpoint: "${env:APM_ENDPOINT}"
headers:
# Elastic APM Server secret token or API key
Authorization: "Bearer ${env:APM_SECRET_KEY}"
elastictrace:
lsminterval:
intervals:
- duration: 1m
statements:
- set(resource.attributes["metricset.interval"], "1m")
- set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "1m"], "."))
- set(attributes["processor.event"], "metric")
- duration: 10m
statements:
- set(resource.attributes["metricset.interval"], "10m")
- set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "10m"], "."))
- set(attributes["processor.event"], "metric")
- duration: 60m
statements:
- set(resource.attributes["metricset.interval"], "60m")
- set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "60m"], "."))
- set(attributes["processor.event"], "metric")

elasticsearch:
exporters:
elasticsearch/ecs:
endpoints: ["${env:ELASTIC_ENDPOINT}"]
api_key: ${env:ELASTIC_API_KEY}
mapping:
Expand All @@ -108,30 +354,47 @@ exporters:
traces_dynamic_index:
enabled: true

elasticsearch/otel:
endpoints: [ "${env:ELASTIC_ENDPOINT}" ]
api_key: ${env:ELASTIC_API_KEY}
mapping:
mode: otel
logs_dynamic_index:
enabled: true
metrics_dynamic_index:
enabled: true
traces_dynamic_index:
enabled: true

service:
extensions: [file_storage]
pipelines:
traces/fromsdk:
receivers: [otlp/fromsdk]
processors: []
exporters: [otlp/apm]
processors: [elastictrace]
exporters: [signaltometrics, elasticsearch/otel]

metrics/fromsdk:
receivers: [otlp/fromsdk]
processors: []
exporters: [otlp/apm]
exporters: [signaltometrics, elasticsearch/otel]

metrics/hostmetrics:
receivers: [hostmetrics/system]
processors: [elasticinframetrics, resourcedetection, attributes/dataset, resource/process]
exporters: [elasticsearch]
exporters: [elasticsearch/ecs]

metrics/aggregated-metrics:
receivers: [signaltometrics]
processors: [lsminterval]
exporters: [elasticsearch/otel]

logs/fromsdk:
receivers: [otlp/fromsdk]
processors: []
exporters: [otlp/apm]
exporters: [signaltometrics, elasticsearch/otel]

logs/platformlogs:
receivers: [filelog/platformlogs]
processors: [resourcedetection]
exporters: [elasticsearch]
exporters: [elasticsearch/otel]
8 changes: 4 additions & 4 deletions internal/pkg/otel/samples/darwin/platformlogs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ receivers:
extensions:
file_storage:
directory: ${env:STORAGE_DIR}

processors:
resourcedetection:
detectors: ["system"]
Expand Down Expand Up @@ -53,11 +53,11 @@ exporters:
sampling_thereafter: 1000

# Exporter to send logs and metrics to Elasticsearch
elasticsearch:
elasticsearch/otel:
endpoints: ["${env:ELASTIC_ENDPOINT}"]
api_key: ${env:ELASTIC_API_KEY}
mapping:
mode: ecs
mode: otel
logs_dynamic_index:
enabled: true
metrics_dynamic_index:
Expand All @@ -71,4 +71,4 @@ service:
logs/platformlogs:
receivers: [filelog/platformlogs]
processors: [resourcedetection]
exporters: [debug, elasticsearch]
exporters: [debug, elasticsearch/otel]
17 changes: 14 additions & 3 deletions internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ processors:

exporters:
# Exporter to send logs and metrics to Elasticsearch
elasticsearch:
elasticsearch/ecs:
endpoints: ["${env:ELASTIC_ENDPOINT}"]
api_key: ${env:ELASTIC_API_KEY}
mapping:
Expand All @@ -96,15 +96,26 @@ exporters:
enabled: true
traces_dynamic_index:
enabled: true
elasticsearch/otel:
endpoints: [ "${env:ELASTIC_ENDPOINT}" ]
api_key: ${env:ELASTIC_API_KEY}
mapping:
mode: otel
logs_dynamic_index:
enabled: true
metrics_dynamic_index:
enabled: true
traces_dynamic_index:
enabled: true

service:
extensions: [file_storage]
pipelines:
metrics/hostmetrics:
receivers: [hostmetrics/system]
processors: [elasticinframetrics, resourcedetection, attributes/dataset, resource/process]
exporters: [elasticsearch]
exporters: [elasticsearch/ecs]
logs/platformlogs:
receivers: [filelog/platformlogs]
processors: [resourcedetection]
exporters: [elasticsearch]
exporters: [elasticsearch/otel]
Loading

0 comments on commit fc75656

Please sign in to comment.