From 64110957ec2360ba8881383a402dcd4fdb7e9968 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Sep 2024 11:00:16 +0200 Subject: [PATCH] add json logging example --- examples/java/json-logging/Dockerfile | 22 +++ examples/java/json-logging/README.md | 14 ++ examples/java/json-logging/build.sh | 5 + examples/java/json-logging/k3d.sh | 14 ++ .../json-logging/k8s/collector-configmap.yaml | 166 ++++++++++++++++++ examples/java/json-logging/k8s/dice.yaml | 44 +++++ examples/java/json-logging/k8s/lgtm.yaml | 66 +++++++ examples/java/json-logging/logback.xml | 7 + 8 files changed, 338 insertions(+) create mode 100644 examples/java/json-logging/Dockerfile create mode 100644 examples/java/json-logging/README.md create mode 100755 examples/java/json-logging/build.sh create mode 100755 examples/java/json-logging/k3d.sh create mode 100644 examples/java/json-logging/k8s/collector-configmap.yaml create mode 100644 examples/java/json-logging/k8s/dice.yaml create mode 100644 examples/java/json-logging/k8s/lgtm.yaml create mode 100644 examples/java/json-logging/logback.xml diff --git a/examples/java/json-logging/Dockerfile b/examples/java/json-logging/Dockerfile new file mode 100644 index 0000000..9e1bec8 --- /dev/null +++ b/examples/java/json-logging/Dockerfile @@ -0,0 +1,22 @@ +FROM eclipse-temurin:21-jdk AS builder + +WORKDIR /usr/src/app/ + +COPY ./mvnw pom.xml ./ +COPY ./.mvn ./.mvn +COPY ./src ./src +COPY json-logging/logback.xml ./src/main/resources/logback.xml +RUN sed -i '//a co.elastic.logginglogback-ecs-encoder1.6.0' pom.xml +RUN --mount=type=cache,target=/root/.m2 ./mvnw install -DskipTests + +FROM eclipse-temurin:21-jre + +WORKDIR /usr/src/app/ + +COPY --from=builder /usr/src/app/target/rolldice.jar ./app.jar +# we ignore the version (which is from upstream) and use the latest version of the grafana distribution +ADD --chmod=644 https://github.com/grafana/grafana-opentelemetry-java/releases/latest/download/grafana-opentelemetry-java.jar /usr/src/app/opentelemetry-javaagent.jar +ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar + +EXPOSE 8080 +ENTRYPOINT [ "java", "-jar", "./app.jar" ] diff --git a/examples/java/json-logging/README.md b/examples/java/json-logging/README.md new file mode 100644 index 0000000..d5abcfb --- /dev/null +++ b/examples/java/json-logging/README.md @@ -0,0 +1,14 @@ +# Exporting Application logs using JSON logging in Kubernetes + +## Introduction + +todo + +## Running the example + +1. Build the Docker image using using `build.sh` +2. Deploy the manifest using `kubectl apply -f k8s/` (e.g. using [k3d.sh](k3d.sh)) +3. Generate traffic using [generate-traffic.sh](../../../generate-traffic.sh) +4. Log in to [http://localhost:3000](http://localhost:3000) with user _admin_ and password _admin_. +5. Go to "Explore" +6. Select "Loki" as data source diff --git a/examples/java/json-logging/build.sh b/examples/java/json-logging/build.sh new file mode 100755 index 0000000..7cbdda6 --- /dev/null +++ b/examples/java/json-logging/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +docker build -f Dockerfile -t "dice:1.1-SNAPSHOT" .. diff --git a/examples/java/json-logging/k3d.sh b/examples/java/json-logging/k3d.sh new file mode 100755 index 0000000..ed370f0 --- /dev/null +++ b/examples/java/json-logging/k3d.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +k3d cluster create jsonlogging || true +k3d image import -c jsonlogging dice:1.1-SNAPSHOT + +kubectl apply -f k8s/ + +kubectl wait --for=condition=ready pod -l app=dice +kubectl wait --for=condition=ready pod -l app=lgtm + +kubectl port-forward service/dice 8080:8080 & +kubectl port-forward service/lgtm 3000:3000 & diff --git a/examples/java/json-logging/k8s/collector-configmap.yaml b/examples/java/json-logging/k8s/collector-configmap.yaml new file mode 100644 index 0000000..c587c7b --- /dev/null +++ b/examples/java/json-logging/k8s/collector-configmap.yaml @@ -0,0 +1,166 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config +data: + otel-collector-config.yaml: |- + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + filelog/json-elastic: + include: + - /var/log/pods/*/dice/*.log + include_file_path: true + operators: + - id: container-parser + type: container + - id: router + type: router + routes: + - output: json_parser + expr: 'body matches "\\{[^{}]*\\}" == true' + - id: json_parser + type: json_parser + on_error: drop # TODO use 'drop_quiet' once available: + body: attributes.message + timestamp: + parse_from: attributes["@timestamp"] + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + severity: + parse_from: attributes["log.level"] + trace: + trace_id: + parse_from: attributes.trace_id + span_id: + parse_from: attributes.span_id + scope_name: + parse_from: attributes["log.logger"] + - id: move_service_namespace + type: move + if: 'attributes["service.namespace"] != nil' + from: attributes["service.namespace"] + to: resource["service.namespace"] + - id: move_service_name + type: move + from: attributes["service.name"] + to: resource["service.name"] + - id: move_service_instance_id + type: move + if: 'attributes["service.instance.id"] != nil' + from: attributes["service.instance.id"] + to: resource["service.instance.id"] + - id: move_deployment_environment + type: move + if: 'attributes["deployment.environment"] != nil' + from: attributes["deployment.environment"] + to: resource["deployment.environment"] + - id: move_thread_name + type: move + from: attributes["process.thread.name"] + to: attributes["thread.name"] + - id: move_error_message + type: move + if: 'attributes["error.message"] != nil' + from: attributes["error.message"] + to: attributes["exception.message"] + - id: move_error_type + type: move + if: 'attributes["error.type"] != nil' + from: attributes["error.type"] + to: attributes["exception.type"] + - id: move_throwable_stacktrace + type: move + if: 'len(attributes["error.stack_trace"]) > 0' + from: attributes["error.stack_trace"] + to: attributes["exception.stacktrace"] + - id: remove_logger_name + type: remove + field: attributes["log.logger"] + - id: remove_timestamp + type: remove + field: attributes["@timestamp"] + - id: remove_level + type: remove + field: attributes["log.level"] + - id: remove_span_id + if: 'attributes["span_id"] != nil' + type: remove + field: attributes.span_id + - id: remove_trace_id + if: 'attributes["trace_id"] != nil' + type: remove + field: attributes.trace_id + - id: remove_message + type: remove + field: attributes.message + - id: remove_ecs_version + type: remove + field: attributes["ecs.version"] + - id: remove_ecs_event_dataset + type: remove + field: attributes["event.dataset"] + - id: remove_trace_flags + type: remove + field: attributes["trace_flags"] + - id: remove_logtag + type: remove + field: attributes.logtag + - id: remove_file + type: remove + field: attributes["log.file.path"] + - id: remove_filename + type: remove + field: attributes["log.file.name"] + - id: remove_stream + type: remove + field: attributes["log.iostream"] + - id: remove_time + type: remove + field: attributes.time + + processors: + batch: + resourcedetection: + detectors: ["env", "system"] + override: false + + exporters: + otlphttp/metrics: + endpoint: http://localhost:9090/api/v1/otlp + otlphttp/traces: + endpoint: http://localhost:4418 + otlphttp/logs: + endpoint: http://localhost:3100/otlp + debug/metrics: + verbosity: detailed + debug/traces: + verbosity: detailed + debug/logs: + verbosity: detailed + nop: + + service: + pipelines: + traces: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ otlphttp/traces ] + metrics: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ otlphttp/metrics ] + logs/otlp: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ otlphttp/logs ] + logs/json-elastic: + receivers: [ filelog/json-elastic ] + processors: [ batch ] + exporters: [ otlphttp/logs ] + # exporters: [ otlphttp/logs, debug/logs ] # Uncomment this line to enable debug logging + + diff --git a/examples/java/json-logging/k8s/dice.yaml b/examples/java/json-logging/k8s/dice.yaml new file mode 100644 index 0000000..e463628 --- /dev/null +++ b/examples/java/json-logging/k8s/dice.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: dice +spec: + selector: + app: dice + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dice +spec: + replicas: 1 + selector: + matchLabels: + app: dice + template: + metadata: + labels: + app: dice + spec: + containers: + - name: dice + image: dice:1.1-SNAPSHOT + imagePullPolicy: Never + ports: + - containerPort: 8080 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://lgtm:4318" + - name: OTEL_LOGS_EXPORTER + value: "none" # to avoid duplicate logs + - name: OTEL_RESOURCE_ATTRIBUTES + value: service.name=dice,service.namespace=shop,service.version=1.1,deployment.environment=staging + - name: OTEL_INSTRUMENTATION_COMMON_MDC_RESOURCE_ATTRIBUTES + value: "service.namespace,service.instance.id,deployment.environment" + - name: SERVICE_NAME + value: dice + diff --git a/examples/java/json-logging/k8s/lgtm.yaml b/examples/java/json-logging/k8s/lgtm.yaml new file mode 100644 index 0000000..2dd3c45 --- /dev/null +++ b/examples/java/json-logging/k8s/lgtm.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: Service +metadata: + name: lgtm +spec: + selector: + app: lgtm + ports: + - name: grafana + protocol: TCP + port: 3000 + targetPort: 3000 + - name: otel-grpc + protocol: TCP + port: 4317 + targetPort: 4317 + - name: otel-http + protocol: TCP + port: 4318 + targetPort: 4318 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lgtm +spec: + replicas: 1 + selector: + matchLabels: + app: lgtm + template: + metadata: + labels: + app: lgtm + spec: + containers: + - name: lgtm + image: grafana/otel-lgtm:latest + ports: + - containerPort: 3000 + - containerPort: 4317 + - containerPort: 4318 + volumeMounts: + - mountPath: /otel-lgtm/otelcol-config.yaml + name: otel-collector-config + subPath: otel-collector-config.yaml + readOnly: true + - mountPath: /var/log + name: varlog + readOnly: true + - mountPath: /var/lib/docker/containers + name: varlibdockercontainers + readOnly: true + env: + - name: ENABLE_LOGS_OTELCOL + value: "true" + volumes: + - name: otel-collector-config + configMap: + name: otel-collector-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/examples/java/json-logging/logback.xml b/examples/java/json-logging/logback.xml new file mode 100644 index 0000000..bd3e027 --- /dev/null +++ b/examples/java/json-logging/logback.xml @@ -0,0 +1,7 @@ + + + + + + +