From 81745f67341e5106eea09e22e43b09dff4c3b888 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 25 Sep 2024 12:33:30 +0200 Subject: [PATCH] add json logging example (#106) * add json logging example * add logback json example * add logback json example * add logback json example * add logback json example * update oats --- .github/workflows/acceptance-tests.yml | 2 +- docker/run-all.sh | 3 +- examples/java/json-logging-ecs/Dockerfile | 22 +++ examples/java/json-logging-ecs/README.md | 10 + examples/java/json-logging-ecs/build.sh | 5 + examples/java/json-logging-ecs/k3d.sh | 15 ++ .../k8s/collector-configmap.yaml | 172 ++++++++++++++++ examples/java/json-logging-ecs/k8s/dice.yaml | 44 +++++ examples/java/json-logging-ecs/k8s/lgtm.yaml | 86 ++++++++ .../java/json-logging-ecs/logback-spring.xml | 7 + examples/java/json-logging-ecs/oats.yaml | 38 ++++ examples/java/json-logging-logback/Dockerfile | 21 ++ examples/java/json-logging-logback/README.md | 10 + examples/java/json-logging-logback/build.sh | 5 + examples/java/json-logging-logback/k3d.sh | 15 ++ .../k8s/collector-configmap.yaml | 186 ++++++++++++++++++ .../java/json-logging-logback/k8s/dice.yaml | 44 +++++ .../java/json-logging-logback/k8s/lgtm.yaml | 86 ++++++++ .../json-logging-logback/logback-spring.xml | 19 ++ examples/java/json-logging-logback/oats.yaml | 41 ++++ run-lgtm.sh | 2 +- scripts/run-acceptance-tests.sh | 1 + 22 files changed, 830 insertions(+), 4 deletions(-) create mode 100644 examples/java/json-logging-ecs/Dockerfile create mode 100644 examples/java/json-logging-ecs/README.md create mode 100755 examples/java/json-logging-ecs/build.sh create mode 100755 examples/java/json-logging-ecs/k3d.sh create mode 100644 examples/java/json-logging-ecs/k8s/collector-configmap.yaml create mode 100644 examples/java/json-logging-ecs/k8s/dice.yaml create mode 100644 examples/java/json-logging-ecs/k8s/lgtm.yaml create mode 100644 examples/java/json-logging-ecs/logback-spring.xml create mode 100644 examples/java/json-logging-ecs/oats.yaml create mode 100644 examples/java/json-logging-logback/Dockerfile create mode 100644 examples/java/json-logging-logback/README.md create mode 100755 examples/java/json-logging-logback/build.sh create mode 100755 examples/java/json-logging-logback/k3d.sh create mode 100644 examples/java/json-logging-logback/k8s/collector-configmap.yaml create mode 100644 examples/java/json-logging-logback/k8s/dice.yaml create mode 100644 examples/java/json-logging-logback/k8s/lgtm.yaml create mode 100644 examples/java/json-logging-logback/logback-spring.xml create mode 100644 examples/java/json-logging-logback/oats.yaml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 24104d4..62b990d 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v4 with: repository: grafana/oats - ref: d07befda3cfa162865d8081aa64501ea26578870 + ref: ebc9c34c3928dacf64042d129f73f814035a0241 path: oats - name: Set up Go uses: actions/setup-go@v5 diff --git a/docker/run-all.sh b/docker/run-all.sh index 7d30c56..0b12cfa 100755 --- a/docker/run-all.sh +++ b/docker/run-all.sh @@ -24,8 +24,7 @@ wait_ready "Loki" "http://localhost:3100/ready" wait_ready "Prometheus" "http://localhost:9090/api/v1/status/runtimeinfo" wait_ready "Tempo" "http://localhost:3200/ready" -# collector may not have a prometheus endpoint exposed if the config has been replaced, -# so we query the otelcol_process_uptime_total metric instead, which checks if the collector is up, +# we query the otelcol_process_uptime_total metric instead, which checks if the collector is up, # and indirectly checks if the prometheus endpoint is up. while ! curl -sg 'http://localhost:9090/api/v1/query?query=otelcol_process_uptime_total{}' | jq -r .data.result[0].value[1] | grep '[0-9]' > /dev/null ; do echo "Waiting for the OpenTelemetry collector to start up..." diff --git a/examples/java/json-logging-ecs/Dockerfile b/examples/java/json-logging-ecs/Dockerfile new file mode 100644 index 0000000..9a7923e --- /dev/null +++ b/examples/java/json-logging-ecs/Dockerfile @@ -0,0 +1,22 @@ +FROM eclipse-temurin:21-jdk AS builder + +WORKDIR /usr/src/app/ + +COPY ./mvnw pom.xml ./ +COPY ./.mvn ./.mvn +COPY ./src ./src +COPY json-logging-ecs/logback-spring.xml ./src/main/resources/logback-spring.xml +RUN sed -i '//a co.elastic.logginglogback-ecs-encoder1.6.0' pom.xml +RUN --mount=type=cache,target=/root/.m2 ./mvnw install -DskipTests + +FROM eclipse-temurin:21-jre + +WORKDIR /usr/src/app/ + +COPY --from=builder /usr/src/app/target/rolldice.jar ./app.jar +# we ignore the version (which is from upstream) and use the latest version of the grafana distribution +ADD --chmod=644 https://github.com/grafana/grafana-opentelemetry-java/releases/latest/download/grafana-opentelemetry-java.jar /usr/src/app/opentelemetry-javaagent.jar +ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar + +EXPOSE 8080 +ENTRYPOINT [ "java", "-jar", "./app.jar" ] diff --git a/examples/java/json-logging-ecs/README.md b/examples/java/json-logging-ecs/README.md new file mode 100644 index 0000000..eb0b279 --- /dev/null +++ b/examples/java/json-logging-ecs/README.md @@ -0,0 +1,10 @@ +# Exporting Application logs using JSON logging in Kubernetes + +## Running the example + +1. Build the Docker image using using `build.sh` +2. Deploy the manifest using `kubectl apply -f k8s/` (e.g. using [k3d.sh](k3d.sh)) +3. Generate traffic using [generate-traffic.sh](../../../generate-traffic.sh) +4. Log in to [http://localhost:3000](http://localhost:3000) with user _admin_ and password _admin_. +5. Go to "Explore" +6. Select "Loki" as data source diff --git a/examples/java/json-logging-ecs/build.sh b/examples/java/json-logging-ecs/build.sh new file mode 100755 index 0000000..7cbdda6 --- /dev/null +++ b/examples/java/json-logging-ecs/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +docker build -f Dockerfile -t "dice:1.1-SNAPSHOT" .. diff --git a/examples/java/json-logging-ecs/k3d.sh b/examples/java/json-logging-ecs/k3d.sh new file mode 100755 index 0000000..36e446a --- /dev/null +++ b/examples/java/json-logging-ecs/k3d.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +./build.sh +k3d cluster create jsonlogging || k3d cluster start jsonlogging +k3d image import -c jsonlogging dice:1.1-SNAPSHOT + +kubectl apply -f k8s/ + +kubectl wait --for=condition=ready pod -l app=dice +kubectl wait --for=condition=ready --timeout=5m pod -l app=lgtm + +kubectl port-forward service/dice 8080:8080 & +kubectl port-forward service/lgtm 3000:3000 & diff --git a/examples/java/json-logging-ecs/k8s/collector-configmap.yaml b/examples/java/json-logging-ecs/k8s/collector-configmap.yaml new file mode 100644 index 0000000..8236d91 --- /dev/null +++ b/examples/java/json-logging-ecs/k8s/collector-configmap.yaml @@ -0,0 +1,172 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config +data: + otel-collector-config.yaml: |- + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus/collector: # needed if you use the docker-lgtm image + config: + scrape_configs: + - job_name: 'opentelemetry-collector' + static_configs: + - targets: [ 'localhost:8888' ] + filelog/json-ecs: + include: + - /var/log/pods/*/dice/*.log + include_file_path: true + operators: + - id: container-parser + type: container + - id: router + type: router + routes: + - output: json_parser + expr: 'body matches "\\{[^{}]*\\}" == true' + - id: json_parser + type: json_parser + on_error: drop # TODO use 'drop_quiet' once available: + body: attributes.message + timestamp: + parse_from: attributes["@timestamp"] + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + severity: + parse_from: attributes["log.level"] + trace: + trace_id: + parse_from: attributes.trace_id + span_id: + parse_from: attributes.span_id + scope_name: + parse_from: attributes["log.logger"] + - id: move_service_namespace + type: move + if: 'attributes["service.namespace"] != nil' + from: attributes["service.namespace"] + to: resource["service.namespace"] + - id: move_service_name + type: move + from: attributes["service.name"] + to: resource["service.name"] + - id: move_service_instance_id + type: move + if: 'attributes["service.instance.id"] != nil' + from: attributes["service.instance.id"] + to: resource["service.instance.id"] + - id: move_deployment_environment + type: move + if: 'attributes["deployment.environment"] != nil' + from: attributes["deployment.environment"] + to: resource["deployment.environment"] + - id: move_thread_name + type: move + from: attributes["process.thread.name"] + to: attributes["thread.name"] + - id: move_error_message + type: move + if: 'attributes["error.message"] != nil' + from: attributes["error.message"] + to: attributes["exception.message"] + - id: move_error_type + type: move + if: 'attributes["error.type"] != nil' + from: attributes["error.type"] + to: attributes["exception.type"] + - id: move_throwable_stacktrace + type: move + if: 'len(attributes["error.stack_trace"]) > 0' + from: attributes["error.stack_trace"] + to: attributes["exception.stacktrace"] + - id: remove_logger_name + type: remove + field: attributes["log.logger"] + - id: remove_timestamp + type: remove + field: attributes["@timestamp"] + - id: remove_level + type: remove + field: attributes["log.level"] + - id: remove_span_id + if: 'attributes["span_id"] != nil' + type: remove + field: attributes.span_id + - id: remove_trace_id + if: 'attributes["trace_id"] != nil' + type: remove + field: attributes.trace_id + - id: remove_message + type: remove + field: attributes.message + - id: remove_ecs_version + type: remove + field: attributes["ecs.version"] + - id: remove_ecs_event_dataset + type: remove + field: attributes["event.dataset"] + - id: remove_trace_flags + type: remove + field: attributes["trace_flags"] + - id: remove_logtag + type: remove + field: attributes.logtag + - id: remove_file + type: remove + field: attributes["log.file.path"] + - id: remove_filename + type: remove + field: attributes["log.file.name"] + - id: remove_stream + type: remove + field: attributes["log.iostream"] + - id: remove_time + type: remove + field: attributes.time + + processors: + batch: + resourcedetection: + detectors: [ "env", "system" ] + override: false + + exporters: + otlphttp/metrics: + endpoint: http://localhost:9090/api/v1/otlp + otlphttp/traces: + endpoint: http://localhost:4418 + otlphttp/logs: + endpoint: http://localhost:3100/otlp + debug/metrics: + verbosity: detailed + debug/traces: + verbosity: detailed + debug/logs: + verbosity: detailed + nop: + + service: + pipelines: + traces: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ otlphttp/traces ] + metrics: + receivers: [ otlp, prometheus/collector ] + processors: [ batch ] + exporters: [ otlphttp/metrics ] + logs/otlp: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ otlphttp/logs ] + logs/json-ecs: + receivers: [ filelog/json-ecs ] + processors: [ batch ] + exporters: [ otlphttp/logs ] + # exporters: [ otlphttp/logs, debug/logs ] # Uncomment this line to enable debug logging + + diff --git a/examples/java/json-logging-ecs/k8s/dice.yaml b/examples/java/json-logging-ecs/k8s/dice.yaml new file mode 100644 index 0000000..e463628 --- /dev/null +++ b/examples/java/json-logging-ecs/k8s/dice.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: dice +spec: + selector: + app: dice + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dice +spec: + replicas: 1 + selector: + matchLabels: + app: dice + template: + metadata: + labels: + app: dice + spec: + containers: + - name: dice + image: dice:1.1-SNAPSHOT + imagePullPolicy: Never + ports: + - containerPort: 8080 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://lgtm:4318" + - name: OTEL_LOGS_EXPORTER + value: "none" # to avoid duplicate logs + - name: OTEL_RESOURCE_ATTRIBUTES + value: service.name=dice,service.namespace=shop,service.version=1.1,deployment.environment=staging + - name: OTEL_INSTRUMENTATION_COMMON_MDC_RESOURCE_ATTRIBUTES + value: "service.namespace,service.instance.id,deployment.environment" + - name: SERVICE_NAME + value: dice + diff --git a/examples/java/json-logging-ecs/k8s/lgtm.yaml b/examples/java/json-logging-ecs/k8s/lgtm.yaml new file mode 100644 index 0000000..7d8f6cc --- /dev/null +++ b/examples/java/json-logging-ecs/k8s/lgtm.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: Service +metadata: + name: lgtm +spec: + selector: + app: lgtm + ports: + - name: grafana + protocol: TCP + port: 3000 + targetPort: 3000 + - name: otel-grpc + protocol: TCP + port: 4317 + targetPort: 4317 + - name: otel-http + protocol: TCP + port: 4318 + targetPort: 4318 + - name: prometheus # needed for automated tests + protocol: TCP + port: 9090 + targetPort: 9090 + - name: loki # needed for automated tests + protocol: TCP + port: 3100 + targetPort: 3100 + - name: tempo # needed for automated tests + protocol: TCP + port: 3200 + targetPort: 3200 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lgtm +spec: + replicas: 1 + selector: + matchLabels: + app: lgtm + template: + metadata: + labels: + app: lgtm + spec: + containers: + - name: lgtm + image: grafana/otel-lgtm:latest + ports: + - containerPort: 3000 + - containerPort: 4317 + - containerPort: 4318 + - containerPort: 9090 # needed for automated tests + - containerPort: 3100 # needed for automated tests + - containerPort: 3200 # needed for automated tests + readinessProbe: + exec: + command: + - cat + - /tmp/ready + volumeMounts: + - mountPath: /otel-lgtm/otelcol-config.yaml + name: otel-collector-config + subPath: otel-collector-config.yaml + readOnly: true + - mountPath: /var/log + name: varlog + readOnly: true + - mountPath: /var/lib/docker/containers + name: varlibdockercontainers + readOnly: true + env: + - name: ENABLE_LOGS_OTELCOL + value: "true" + volumes: + - name: otel-collector-config + configMap: + name: otel-collector-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/examples/java/json-logging-ecs/logback-spring.xml b/examples/java/json-logging-ecs/logback-spring.xml new file mode 100644 index 0000000..bd3e027 --- /dev/null +++ b/examples/java/json-logging-ecs/logback-spring.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/examples/java/json-logging-ecs/oats.yaml b/examples/java/json-logging-ecs/oats.yaml new file mode 100644 index 0000000..0edfb2e --- /dev/null +++ b/examples/java/json-logging-ecs/oats.yaml @@ -0,0 +1,38 @@ +# OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats/tree/main/yaml +kubernetes: + dir: k8s + app-service: dice + app-docker-file: Dockerfile + app-docker-context: .. + app-docker-tag: dice:1.1-SNAPSHOT + app-docker-port: 8080 +input: + - path: /rolldice +expected: + logs: + - logql: '{service_name="dice"} |~ `.*simulating an error.*`' + equals: + - 'Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed: java.lang.RuntimeException: simulating an error] with root cause' + attributes: + deployment_environment: staging + exception_message: "simulating an error" + exception_type: "java.lang.RuntimeException" + scope_name: "org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/].[dispatcherServlet]" + service_name: dice + service_namespace: shop + severity_number: 17 + severity_text: ERROR + k8s_container_name: dice + k8s_namespace_name: default + attribute-regexp: + detected_level: ".*" # from loki + observed_timestamp: ".*" # from loki + thread_name: ".*" + span_id: ".*" + trace_id: ".*" + k8s_pod_name: dice-.*-.* + k8s_pod_uid: ".*" + k8s_container_restart_count: ".*" + service_instance_id: ".*" + exception_stacktrace: ".*" + no-extra-attributes: true diff --git a/examples/java/json-logging-logback/Dockerfile b/examples/java/json-logging-logback/Dockerfile new file mode 100644 index 0000000..6b158df --- /dev/null +++ b/examples/java/json-logging-logback/Dockerfile @@ -0,0 +1,21 @@ +FROM eclipse-temurin:21-jdk AS builder + +WORKDIR /usr/src/app/ + +COPY ./mvnw pom.xml ./ +COPY ./.mvn ./.mvn +COPY ./src ./src +COPY json-logging-logback/logback-spring.xml ./src/main/resources/logback-spring.xml +RUN --mount=type=cache,target=/root/.m2 ./mvnw install -DskipTests + +FROM eclipse-temurin:21-jre + +WORKDIR /usr/src/app/ + +COPY --from=builder /usr/src/app/target/rolldice.jar ./app.jar +# we ignore the version (which is from upstream) and use the latest version of the grafana distribution +ADD --chmod=644 https://github.com/grafana/grafana-opentelemetry-java/releases/latest/download/grafana-opentelemetry-java.jar /usr/src/app/opentelemetry-javaagent.jar +ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar + +EXPOSE 8080 +ENTRYPOINT [ "java", "-jar", "./app.jar" ] diff --git a/examples/java/json-logging-logback/README.md b/examples/java/json-logging-logback/README.md new file mode 100644 index 0000000..eb0b279 --- /dev/null +++ b/examples/java/json-logging-logback/README.md @@ -0,0 +1,10 @@ +# Exporting Application logs using JSON logging in Kubernetes + +## Running the example + +1. Build the Docker image using using `build.sh` +2. Deploy the manifest using `kubectl apply -f k8s/` (e.g. using [k3d.sh](k3d.sh)) +3. Generate traffic using [generate-traffic.sh](../../../generate-traffic.sh) +4. Log in to [http://localhost:3000](http://localhost:3000) with user _admin_ and password _admin_. +5. Go to "Explore" +6. Select "Loki" as data source diff --git a/examples/java/json-logging-logback/build.sh b/examples/java/json-logging-logback/build.sh new file mode 100755 index 0000000..7cbdda6 --- /dev/null +++ b/examples/java/json-logging-logback/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +docker build -f Dockerfile -t "dice:1.1-SNAPSHOT" .. diff --git a/examples/java/json-logging-logback/k3d.sh b/examples/java/json-logging-logback/k3d.sh new file mode 100755 index 0000000..36e446a --- /dev/null +++ b/examples/java/json-logging-logback/k3d.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +./build.sh +k3d cluster create jsonlogging || k3d cluster start jsonlogging +k3d image import -c jsonlogging dice:1.1-SNAPSHOT + +kubectl apply -f k8s/ + +kubectl wait --for=condition=ready pod -l app=dice +kubectl wait --for=condition=ready --timeout=5m pod -l app=lgtm + +kubectl port-forward service/dice 8080:8080 & +kubectl port-forward service/lgtm 3000:3000 & diff --git a/examples/java/json-logging-logback/k8s/collector-configmap.yaml b/examples/java/json-logging-logback/k8s/collector-configmap.yaml new file mode 100644 index 0000000..541f335 --- /dev/null +++ b/examples/java/json-logging-logback/k8s/collector-configmap.yaml @@ -0,0 +1,186 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config +data: + otel-collector-config.yaml: |- + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus/collector: # needed if you use the docker-lgtm image + config: + scrape_configs: + - job_name: 'opentelemetry-collector' + static_configs: + - targets: [ 'localhost:8888' ] + filelog/json-logback: + include: + - /var/log/pods/*/dice/*.log + include_file_path: true + operators: + - id: container-parser + type: container + - id: router + type: router + routes: + - output: json_parser + expr: 'body matches "\\{[^{}]*\\}" == true' + - id: json_parser + type: json_parser + on_error: drop # TODO use 'drop_quiet' once available: + body: attributes.message + timestamp: + parse_from: attributes.timestamp + layout_type: 'epoch' + layout: 'ms' + severity: + parse_from: attributes.level + trace: + trace_id: + parse_from: attributes.mdc.trace_id + span_id: + parse_from: attributes.mdc.span_id + trace_flags: + parse_from: attributes.mdc.trace_flags + scope_name: + parse_from: attributes.loggerName + - id: move_service_namespace + type: move + if: 'attributes.mdc["service.namespace"] != nil' + from: attributes.mdc["service.namespace"] + to: resource["service.namespace"] + - id: move_service_name + type: move + from: attributes.mdc["service.name"] + to: resource["service.name"] + - id: move_service_version + type: move + from: attributes.mdc["service.version"] + to: resource["service.version"] + - id: move_service_instance_id + type: move + if: 'attributes.mdc["service.instance.id"] != nil' + from: attributes.mdc["service.instance.id"] + to: resource["service.instance.id"] + - id: move_deployment_environment + type: move + if: 'attributes.mdc["deployment.environment"] != nil' + from: attributes.mdc["deployment.environment"] + to: resource["deployment.environment"] + - id: move_thread_name + type: move + from: attributes.threadName + to: attributes["thread.name"] + - id: move_throwable_class_name + type: move + if: "attributes.throwable?.className != nil" + from: attributes.throwable.className + to: attributes["exception.type"] + - id: move_throwable_message + type: move + if: "attributes.throwable?.message != nil" + from: attributes.throwable.message + to: attributes["exception.message"] + # FIXME "stepArray" is a json array eroding the visualization in Loki + # [{ "className": "a.b.C", "methodName": "do", "fileName": "C.java", "lineNumber": 123},...] + # It would help if logBack had a raw toString of the stack trace + - id: move_throwable_stack_trace + type: move + if: "attributes.throwable?.stepArray != nil" + from: attributes.throwable.stepArray + to: attributes["exception.stacktrace"] + - id: remove_throwable + type: remove + field: attributes.throwable + - id: remove_logger_name + type: remove + field: attributes.loggerName + - id: remove_timestamp + type: remove + field: attributes.timestamp + - id: remove_nanoseconds + type: remove + field: attributes.nanoseconds + - id: remove_observed_timestamp + type: remove + field: attributes["observed.timestamp"] + - id: remove_level + type: remove + field: attributes.level + - id: remove_detected_level + type: remove + field: attributes["detected.level"] + - id: remove_mdc + type: remove + field: attributes.mdc + - id: remove_context + type: remove + field: attributes.context + - id: remove_message + type: remove + field: attributes.message + - id: remove_logtag + type: remove + field: attributes.logtag + - id: remove_file + type: remove + field: attributes["log.file.path"] + - id: remove_filename + type: remove + field: attributes["log.file.name"] + - id: remove_stream + type: remove + field: attributes["log.iostream"] + - id: remove_time + type: remove + field: attributes.time + - id: remove_sequenceNumber + type: remove + field: attributes.sequenceNumber + + processors: + batch: + resourcedetection: + detectors: [ "env", "system" ] + override: false + + exporters: + otlphttp/metrics: + endpoint: http://localhost:9090/api/v1/otlp + otlphttp/traces: + endpoint: http://localhost:4418 + otlphttp/logs: + endpoint: http://localhost:3100/otlp + debug/metrics: + verbosity: detailed + debug/traces: + verbosity: detailed + debug/logs: + verbosity: detailed + nop: + + service: + pipelines: + traces: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ otlphttp/traces ] + metrics: + receivers: [ otlp, prometheus/collector ] + processors: [ batch ] + exporters: [ otlphttp/metrics ] + logs/otlp: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ otlphttp/logs ] + logs/json-elastic: + receivers: [ filelog/json-logback ] + processors: [ batch ] + exporters: [ otlphttp/logs ] + # exporters: [ otlphttp/logs, debug/logs ] # Uncomment this line to enable debug logging + + diff --git a/examples/java/json-logging-logback/k8s/dice.yaml b/examples/java/json-logging-logback/k8s/dice.yaml new file mode 100644 index 0000000..296fd6a --- /dev/null +++ b/examples/java/json-logging-logback/k8s/dice.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: dice +spec: + selector: + app: dice + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dice +spec: + replicas: 1 + selector: + matchLabels: + app: dice + template: + metadata: + labels: + app: dice + spec: + containers: + - name: dice + image: dice:1.1-SNAPSHOT + imagePullPolicy: Never + ports: + - containerPort: 8080 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://lgtm:4318" + - name: OTEL_LOGS_EXPORTER + value: "none" # to avoid duplicate logs + - name: OTEL_RESOURCE_ATTRIBUTES + value: service.name=dice,service.namespace=shop,service.version=1.1,deployment.environment=staging + - name: OTEL_INSTRUMENTATION_COMMON_MDC_RESOURCE_ATTRIBUTES + value: "service.namespace,service.name,service.instance.id,service.version,deployment.environment" + - name: SERVICE_NAME + value: dice + diff --git a/examples/java/json-logging-logback/k8s/lgtm.yaml b/examples/java/json-logging-logback/k8s/lgtm.yaml new file mode 100644 index 0000000..7d8f6cc --- /dev/null +++ b/examples/java/json-logging-logback/k8s/lgtm.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: Service +metadata: + name: lgtm +spec: + selector: + app: lgtm + ports: + - name: grafana + protocol: TCP + port: 3000 + targetPort: 3000 + - name: otel-grpc + protocol: TCP + port: 4317 + targetPort: 4317 + - name: otel-http + protocol: TCP + port: 4318 + targetPort: 4318 + - name: prometheus # needed for automated tests + protocol: TCP + port: 9090 + targetPort: 9090 + - name: loki # needed for automated tests + protocol: TCP + port: 3100 + targetPort: 3100 + - name: tempo # needed for automated tests + protocol: TCP + port: 3200 + targetPort: 3200 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lgtm +spec: + replicas: 1 + selector: + matchLabels: + app: lgtm + template: + metadata: + labels: + app: lgtm + spec: + containers: + - name: lgtm + image: grafana/otel-lgtm:latest + ports: + - containerPort: 3000 + - containerPort: 4317 + - containerPort: 4318 + - containerPort: 9090 # needed for automated tests + - containerPort: 3100 # needed for automated tests + - containerPort: 3200 # needed for automated tests + readinessProbe: + exec: + command: + - cat + - /tmp/ready + volumeMounts: + - mountPath: /otel-lgtm/otelcol-config.yaml + name: otel-collector-config + subPath: otel-collector-config.yaml + readOnly: true + - mountPath: /var/log + name: varlog + readOnly: true + - mountPath: /var/lib/docker/containers + name: varlibdockercontainers + readOnly: true + env: + - name: ENABLE_LOGS_OTELCOL + value: "true" + volumes: + - name: otel-collector-config + configMap: + name: otel-collector-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/examples/java/json-logging-logback/logback-spring.xml b/examples/java/json-logging-logback/logback-spring.xml new file mode 100644 index 0000000..1e5867c --- /dev/null +++ b/examples/java/json-logging-logback/logback-spring.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + true + false + false + false + false + + + diff --git a/examples/java/json-logging-logback/oats.yaml b/examples/java/json-logging-logback/oats.yaml new file mode 100644 index 0000000..f0bdba1 --- /dev/null +++ b/examples/java/json-logging-logback/oats.yaml @@ -0,0 +1,41 @@ +# OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats/tree/main/yaml +kubernetes: + dir: k8s + app-service: dice + app-docker-file: Dockerfile + app-docker-context: .. + app-docker-tag: dice:1.1-SNAPSHOT + app-docker-port: 8080 +input: + - path: /rolldice +expected: + logs: + - logql: '{service_name="dice"} |~ `.*simulating an error.*`' + equals: + - 'Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed: java.lang.RuntimeException: simulating an error] with root cause' + attributes: + deployment_environment: staging + exception_message: "simulating an error" + exception_type: "java.lang.RuntimeException" + scope_name: "org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/].[dispatcherServlet]" + service_name: dice + service_namespace: shop + service_version: 1.1 + severity_number: 17 + severity_text: ERROR + k8s_container_name: dice + k8s_namespace_name: default + attribute-regexp: + flags: ".*" # from loki + detected_level: ".*" # from loki + observed_timestamp: ".*" # from loki + thread_name: ".*" + span_id: ".*" + trace_id: ".*" + k8s_pod_name: dice-.*-.* + k8s_pod_uid: ".*" + k8s_container_restart_count: ".*" + service_instance_id: ".*" + exception_stacktrace: ".*" # TODO: transform to sanitized stacktrace + no-extra-attributes: true + diff --git a/run-lgtm.sh b/run-lgtm.sh index f380803..bfa6906 100755 --- a/run-lgtm.sh +++ b/run-lgtm.sh @@ -13,4 +13,4 @@ docker run \ -v $PWD/container/prometheus:/data/prometheus \ -v $PWD/container/loki:/loki \ -e GF_PATHS_DATA=/data/grafana \ - docker.io/grafana/otel-lgtm:${RELEASE} \ No newline at end of file + docker.io/grafana/otel-lgtm:${RELEASE} diff --git a/scripts/run-acceptance-tests.sh b/scripts/run-acceptance-tests.sh index 94c9cd8..ed95721 100755 --- a/scripts/run-acceptance-tests.sh +++ b/scripts/run-acceptance-tests.sh @@ -2,6 +2,7 @@ set -euo pipefail +wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash cd oats/yaml go install github.com/onsi/ginkgo/v2/ginkgo@latest export TESTCASE_TIMEOUT=5m