diff --git a/examples/add-filter-processor/README.md b/examples/add-filter-processor/README.md new file mode 100644 index 000000000..fda3e9ba3 --- /dev/null +++ b/examples/add-filter-processor/README.md @@ -0,0 +1,7 @@ +# Example of chart configuration + +## Filter Processor +The +[filter processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor#filter-processor) +determines when telemetry data should be dropped. If any condition +is met, the telemetry data is dropped. diff --git a/examples/add-filter-processor/add-filter-processor-values.yaml b/examples/add-filter-processor/add-filter-processor-values.yaml new file mode 100644 index 000000000..abbc41372 --- /dev/null +++ b/examples/add-filter-processor/add-filter-processor-values.yaml @@ -0,0 +1,82 @@ +clusterName: CHANGEME +splunkObservability: + realm: CHANGEME + accessToken: CHANGEME + metricsEnabled: true + tracesEnabled: true + logsEnabled: true + +agent: + config: + processors: + # Exclude specific metrics from containers named 'containerXName' or 'containerYName' + filter/exclude_metrics_from_container: + metrics: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.container.name + value: '^(containerXName|containerYName)$' + # Exclude logs from pods named 'podNameX' + filter/exclude_logs_from_pod: + logs: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.pod.name + value: '^(podNameX)$' + # Exclude logs from nodes named 'nodeNameX' + filter/exclude_logs_from_node: + logs: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.node.name + value: '^(nodeNameX)$' + # Exclude spans from traces for services housed in containers named 'containerXName' or 'containerYName' + filter/exclude_spans_from_traces_from_container: + spans: + exclude: + match_type: regexp + attributes: + - key: k8s.container.name + value: '^(containerXName|containerYName)$' + # Exclude all telemetry data (metrics, logs, traces) from a namespace named 'namespaceX' + filter/exclude_all_telemetry_data_from_namespace: + logs: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.namespace.name + value: '^(namespaceX)$' + metrics: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.namespace.name + value: '^(namespaceX)$' + traces: + span: + - 'attributes["k8s.namespace.name"] != "namespaceX"' + # Exclude metrics from a cluster named 'clusterX' + filter/exclude_metrics_from_cluster: + metrics: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.cluster.name + value: '^(clusterX)$' + # Define the data processing pipelines for logs, metrics, and traces + service: + pipelines: + logs: + processors: + - memory_limiter + - k8sattributes + - filter/logs + - batch + - resourcedetection + - resource + - resource/logs + - filter/exclude_logs_from_pod + - filter/exclude_logs_from_node diff --git a/examples/filter-container-metrics/rendered_manifests/clusterRole.yaml b/examples/add-filter-processor/rendered_manifests/clusterRole.yaml similarity index 100% rename from examples/filter-container-metrics/rendered_manifests/clusterRole.yaml rename to examples/add-filter-processor/rendered_manifests/clusterRole.yaml diff --git a/examples/filter-container-metrics/rendered_manifests/clusterRoleBinding.yaml b/examples/add-filter-processor/rendered_manifests/clusterRoleBinding.yaml similarity index 100% rename from examples/filter-container-metrics/rendered_manifests/clusterRoleBinding.yaml rename to examples/add-filter-processor/rendered_manifests/clusterRoleBinding.yaml diff --git a/examples/filter-container-metrics/rendered_manifests/configmap-agent.yaml b/examples/add-filter-processor/rendered_manifests/configmap-agent.yaml similarity index 71% rename from examples/filter-container-metrics/rendered_manifests/configmap-agent.yaml rename to examples/add-filter-processor/rendered_manifests/configmap-agent.yaml index 3147466be..75a5d0460 100644 --- a/examples/filter-container-metrics/rendered_manifests/configmap-agent.yaml +++ b/examples/add-filter-processor/rendered_manifests/configmap-agent.yaml @@ -17,12 +17,21 @@ metadata: data: relay: | exporters: + sapm: + access_token: ${SPLUNK_OBSERVABILITY_ACCESS_TOKEN} + endpoint: https://ingest.CHANGEME.signalfx.com/v2/trace signalfx: access_token: ${SPLUNK_OBSERVABILITY_ACCESS_TOKEN} api_url: https://api.CHANGEME.signalfx.com correlation: null ingest_url: https://ingest.CHANGEME.signalfx.com sync_host_metadata: true + splunk_hec/o11y: + disable_compression: true + endpoint: https://ingest.CHANGEME.signalfx.com/v1/log + log_data_enabled: true + profiling_data_enabled: false + token: ${SPLUNK_OBSERVABILITY_ACCESS_TOKEN} extensions: health_check: null k8s_observer: @@ -33,13 +42,57 @@ data: zpages: null processors: batch: null - filter/exclude_containers: + filter/exclude_all_telemetry_data_from_namespace: + logs: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.namespace.name + value: ^(namespaceX)$ + metrics: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.namespace.name + value: ^(namespaceX)$ + traces: + span: + - attributes["k8s.namespace.name"] != "namespaceX" + filter/exclude_logs_from_node: + logs: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.node.name + value: ^(nodeNameX)$ + filter/exclude_logs_from_pod: + logs: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.pod.name + value: ^(podNameX)$ + filter/exclude_metrics_from_cluster: metrics: exclude: match_type: regexp resource_attributes: - - Key: k8s.container.name - Value: ^(containerX|containerY)$ + - key: k8s.cluster.name + value: ^(clusterX)$ + filter/exclude_metrics_from_container: + metrics: + exclude: + match_type: regexp + resource_attributes: + - key: k8s.container.name + value: ^(containerXName|containerYName)$ + filter/exclude_spans_from_traces_from_container: + spans: + exclude: + attributes: + - key: k8s.container.name + value: ^(containerXName|containerYName)$ + match_type: regexp filter/logs: logs: exclude: @@ -140,6 +193,8 @@ data: override: true timeout: 10s receivers: + fluentforward: + endpoint: 0.0.0.0:8006 hostmetrics: collection_interval: 10s scrapers: @@ -151,6 +206,12 @@ data: network: null paging: null processes: null + jaeger: + protocols: + grpc: + endpoint: 0.0.0.0:14250 + thrift_http: + endpoint: 0.0.0.0:14268 kubeletstats: auth_type: serviceAccount collection_interval: 10s @@ -224,6 +285,11 @@ data: - k8s_observer signalfx: endpoint: 0.0.0.0:9943 + smartagent/signalfx-forwarder: + listenAddress: 0.0.0.0:9080 + type: signalfx-forwarder + zipkin: + endpoint: 0.0.0.0:9411 service: extensions: - health_check @@ -231,6 +297,22 @@ data: - memory_ballast - zpages pipelines: + logs: + exporters: + - splunk_hec/o11y + processors: + - memory_limiter + - k8sattributes + - filter/logs + - batch + - resourcedetection + - resource + - resource/logs + - filter/exclude_logs_from_pod + - filter/exclude_logs_from_node + receivers: + - fluentforward + - otlp metrics: exporters: - signalfx @@ -239,7 +321,6 @@ data: - batch - resourcedetection - resource - - filter/exclude_containers receivers: - hostmetrics - kubeletstats @@ -257,6 +338,21 @@ data: - resource receivers: - prometheus/agent + traces: + exporters: + - sapm + - signalfx + processors: + - memory_limiter + - k8sattributes + - batch + - resourcedetection + - resource + receivers: + - otlp + - jaeger + - smartagent/signalfx-forwarder + - zipkin telemetry: metrics: address: 0.0.0.0:8889 diff --git a/examples/filter-container-metrics/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-filter-processor/rendered_manifests/configmap-cluster-receiver.yaml similarity index 91% rename from examples/filter-container-metrics/rendered_manifests/configmap-cluster-receiver.yaml rename to examples/add-filter-processor/rendered_manifests/configmap-cluster-receiver.yaml index 22b204b61..1ce9658ae 100644 --- a/examples/filter-container-metrics/rendered_manifests/configmap-cluster-receiver.yaml +++ b/examples/add-filter-processor/rendered_manifests/configmap-cluster-receiver.yaml @@ -29,13 +29,6 @@ data: processors: batch: send_batch_max_size: 32768 - filter/exclude_containers: - metrics: - exclude: - match_type: regexp - resource_attributes: - - Key: k8s.container.name - Value: ^(containerX|containerY)$ memory_limiter: check_interval: 2s limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB} @@ -98,7 +91,6 @@ data: - batch - resource - resource/k8s_cluster - - filter/exclude_containers receivers: - k8s_cluster metrics/collector: diff --git a/examples/add-filter-processor/rendered_manifests/configmap-fluentd-json.yaml b/examples/add-filter-processor/rendered_manifests/configmap-fluentd-json.yaml new file mode 100644 index 000000000..df0c5954e --- /dev/null +++ b/examples/add-filter-processor/rendered_manifests/configmap-fluentd-json.yaml @@ -0,0 +1,24 @@ +--- +# Source: splunk-otel-collector/templates/configmap-fluentd-json.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: default-splunk-otel-collector-fluentd-json + labels: + app.kubernetes.io/name: splunk-otel-collector + helm.sh/chart: splunk-otel-collector-0.79.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: default + app.kubernetes.io/version: "0.79.0" + app: splunk-otel-collector + chart: splunk-otel-collector-0.79.0 + release: default + heritage: Helm +data: + source.containers.parse.conf: |- + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + output.filter.conf: "" + + output.transform.conf: "" diff --git a/examples/add-filter-processor/rendered_manifests/configmap-fluentd.yaml b/examples/add-filter-processor/rendered_manifests/configmap-fluentd.yaml new file mode 100644 index 000000000..8821a4518 --- /dev/null +++ b/examples/add-filter-processor/rendered_manifests/configmap-fluentd.yaml @@ -0,0 +1,352 @@ +--- +# Source: splunk-otel-collector/templates/configmap-fluentd.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: default-splunk-otel-collector-fluentd + labels: + app.kubernetes.io/name: splunk-otel-collector + helm.sh/chart: splunk-otel-collector-0.79.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: default + app.kubernetes.io/version: "0.79.0" + app: splunk-otel-collector + chart: splunk-otel-collector-0.79.0 + release: default + heritage: Helm +data: + fluent.conf: |- + @include system.conf + @include source.containers.conf + @include source.files.conf + @include source.journald.conf + @include output.conf + @include prometheus.conf + + system.conf: |- + # system wide configurations + + log_level info + root_dir /tmp/fluentd + + + prometheus.conf: |- + # input plugin that exports metrics + + @type prometheus + + + # input plugin that collects metrics from MonitorAgent + + @type prometheus_monitor + + + # input plugin that collects metrics for output plugin + + @type prometheus_output_monitor + + + source.containers.conf: |- + # This configuration file for Fluentd / td-agent is used + # to watch changes to Docker log files. The kubelet creates symlinks that + # capture the pod name, namespace, container name & Docker container ID + # to the docker logs for pods in the /var/log/containers directory on the host. + # If running this fluentd configuration in a Docker container, the /var/log + # directory should be mounted in the container. + # reading kubelet logs from journal + # + # Reference: + # https://github.com/kubernetes/community/blob/20d2f6f5498a5668bae2aea9dcaf4875b9c06ccb/contributors/design-proposals/node/kubelet-cri-logging.md + # + # Json Log Example: + # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} + # CRI Log Example (not supported): + # 2016-02-17T00:04:05.931087621Z stdout P { 'long': { 'json', 'object output' }, + # 2016-02-17T00:04:05.931087621Z stdout F 'splitted': 'partial-lines' } + # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here + + @id containers.log + @type tail + @label @CONCAT + tag tail.containers.* + path /var/log/containers/*.log + pos_file /var/log/splunk-fluentd-containers.log.pos + path_key source + read_from_head true + + @include source.containers.parse.conf + time_key time + time_type string + localtime false + + + + source.files.conf: |- + # This fluentd conf file contains sources for log files other than container logs. + + @id tail.file.kube-audit + @type tail + @label @CONCAT + tag tail.file.kube:apiserver-audit + path /var/log/kube-apiserver-audit.log + pos_file /var/log/splunk-fluentd-kube-audit.pos + read_from_head true + path_key source + + @type regexp + expression /^(?.*)$/ + time_key time + time_type string + time_format %Y-%m-%dT%H:%M:%SZ + + + + source.journald.conf: |- + # This fluentd conf file contains configurations for reading logs from systemd journal. + + @id journald-docker + @type systemd + @label @CONCAT + tag journald.kube:docker + path "/run/log/journal" + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + read_from_head true + + @type local + persistent true + path /var/log/splunkd-fluentd-journald-docker.pos.json + + + field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} + field_map_strict true + + + + @id journald-kubelet + @type systemd + @label @CONCAT + tag journald.kube:kubelet + path "/run/log/journal" + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + read_from_head true + + @type local + persistent true + path /var/log/splunkd-fluentd-journald-kubelet.pos.json + + + field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} + field_map_strict true + + + + output.conf: |- + #Events are emitted to the CONCAT label from the container, file and journald sources for multiline processing. + + + source.containers.parse.conf: |- + @type regexp + expression /^(?