add json logging example (#106)

* add json logging example

* add logback json example

* add logback json example

* add logback json example

* add logback json example

* update oats

.github/workflows/acceptance-tests.yml

@@ -12,7 +12,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: grafana/oats
-          ref: d07befda3cfa162865d8081aa64501ea26578870
+          ref: ebc9c34c3928dacf64042d129f73f814035a0241
           path: oats
       - name: Set up Go
         uses: actions/setup-go@v5

docker/run-all.sh

@@ -24,8 +24,7 @@ wait_ready "Loki" "http://localhost:3100/ready"
 wait_ready "Prometheus" "http://localhost:9090/api/v1/status/runtimeinfo"
 wait_ready "Tempo" "http://localhost:3200/ready"
 
-# collector may not have a prometheus endpoint exposed if the config has been replaced,
-# so we query the otelcol_process_uptime_total metric instead, which checks if the collector is up,
+# we query the otelcol_process_uptime_total metric instead, which checks if the collector is up,
 # and indirectly checks if the prometheus endpoint is up.
 while ! curl -sg 'http://localhost:9090/api/v1/query?query=otelcol_process_uptime_total{}' | jq -r .data.result[0].value[1] | grep '[0-9]' > /dev/null ; do
   echo "Waiting for the OpenTelemetry collector to start up..."

examples/java/json-logging-ecs/Dockerfile

@@ -0,0 +1,22 @@
+FROM eclipse-temurin:21-jdk AS builder
+
+WORKDIR /usr/src/app/
+
+COPY ./mvnw pom.xml ./
+COPY ./.mvn ./.mvn
+COPY ./src ./src
+COPY json-logging-ecs/logback-spring.xml ./src/main/resources/logback-spring.xml
+RUN sed -i '/<dependencies>/a <dependency><groupId>co.elastic.logging</groupId><artifactId>logback-ecs-encoder</artifactId><version>1.6.0</version></dependency>' pom.xml
+RUN --mount=type=cache,target=/root/.m2 ./mvnw install -DskipTests
+
+FROM eclipse-temurin:21-jre
+
+WORKDIR /usr/src/app/
+
+COPY --from=builder /usr/src/app/target/rolldice.jar ./app.jar
+# we ignore the version (which is from upstream) and use the latest version of the grafana distribution
+ADD --chmod=644 https://github.com/grafana/grafana-opentelemetry-java/releases/latest/download/grafana-opentelemetry-java.jar /usr/src/app/opentelemetry-javaagent.jar
+ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar
+
+EXPOSE 8080
+ENTRYPOINT [ "java", "-jar", "./app.jar" ]

examples/java/json-logging-ecs/README.md

@@ -0,0 +1,10 @@
+# Exporting Application logs using JSON logging in Kubernetes
+
+## Running the example
+
+1. Build the Docker image using using `build.sh`
+2. Deploy the manifest using `kubectl apply -f k8s/` (e.g. using [k3d.sh](k3d.sh))
+3. Generate traffic using [generate-traffic.sh](../../../generate-traffic.sh)
+4. Log in to [http://localhost:3000](http://localhost:3000) with user _admin_ and password _admin_.
+5. Go to "Explore"
+6. Select "Loki" as data source

examples/java/json-logging-ecs/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+docker build -f Dockerfile -t "dice:1.1-SNAPSHOT" ..

examples/java/json-logging-ecs/k3d.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+./build.sh
+k3d cluster create jsonlogging || k3d cluster start jsonlogging
+k3d image import -c jsonlogging dice:1.1-SNAPSHOT
+
+kubectl apply -f k8s/
+
+kubectl wait --for=condition=ready pod -l app=dice
+kubectl wait --for=condition=ready --timeout=5m pod -l app=lgtm
+
+kubectl port-forward service/dice 8080:8080 &
+kubectl port-forward service/lgtm 3000:3000 &

examples/java/json-logging-ecs/k8s/collector-configmap.yaml

@@ -0,0 +1,172 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-config
+data:
+  otel-collector-config.yaml: |-
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+      prometheus/collector: # needed if you use the docker-lgtm image
+        config:
+          scrape_configs:
+            - job_name: 'opentelemetry-collector'
+              static_configs:
+                - targets: [ 'localhost:8888' ]
+      filelog/json-ecs:
+        include:
+          - /var/log/pods/*/dice/*.log
+        include_file_path: true
+        operators:
+          - id: container-parser
+            type: container
+          - id: router
+            type: router
+            routes:
+              - output: json_parser
+                expr: 'body matches "\\{[^{}]*\\}" == true'
+          - id: json_parser
+            type: json_parser
+            on_error: drop # TODO use 'drop_quiet' once available:
+            body: attributes.message
+            timestamp:
+              parse_from: attributes["@timestamp"]
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+            severity:
+              parse_from: attributes["log.level"]
+            trace:
+              trace_id:
+                parse_from: attributes.trace_id
+              span_id:
+                parse_from: attributes.span_id
+            scope_name:
+              parse_from: attributes["log.logger"]
+          - id: move_service_namespace
+            type: move
+            if: 'attributes["service.namespace"] != nil'
+            from: attributes["service.namespace"]
+            to: resource["service.namespace"]
+          - id: move_service_name
+            type: move
+            from: attributes["service.name"]
+            to: resource["service.name"]
+          - id: move_service_instance_id
+            type: move
+            if: 'attributes["service.instance.id"] != nil'
+            from: attributes["service.instance.id"]
+            to: resource["service.instance.id"]
+          - id: move_deployment_environment
+            type: move
+            if: 'attributes["deployment.environment"] != nil'
+            from: attributes["deployment.environment"]
+            to: resource["deployment.environment"]
+          - id: move_thread_name
+            type: move
+            from: attributes["process.thread.name"]
+            to: attributes["thread.name"]
+          - id: move_error_message
+            type: move
+            if: 'attributes["error.message"] != nil'
+            from: attributes["error.message"]
+            to: attributes["exception.message"]
+          - id: move_error_type
+            type: move
+            if: 'attributes["error.type"] != nil'
+            from: attributes["error.type"]
+            to: attributes["exception.type"]
+          - id: move_throwable_stacktrace
+            type: move
+            if: 'len(attributes["error.stack_trace"]) > 0'
+            from: attributes["error.stack_trace"]
+            to: attributes["exception.stacktrace"]
+          - id: remove_logger_name
+            type: remove
+            field: attributes["log.logger"]
+          - id: remove_timestamp
+            type: remove
+            field: attributes["@timestamp"]
+          - id: remove_level
+            type: remove
+            field: attributes["log.level"]
+          - id: remove_span_id
+            if: 'attributes["span_id"] != nil'
+            type: remove
+            field: attributes.span_id
+          - id: remove_trace_id
+            if: 'attributes["trace_id"] != nil'
+            type: remove
+            field: attributes.trace_id
+          - id: remove_message
+            type: remove
+            field: attributes.message
+          - id: remove_ecs_version
+            type: remove
+            field: attributes["ecs.version"]
+          - id: remove_ecs_event_dataset
+            type: remove
+            field: attributes["event.dataset"]
+          - id: remove_trace_flags
+            type: remove
+            field: attributes["trace_flags"]
+          - id: remove_logtag
+            type: remove
+            field: attributes.logtag
+          - id: remove_file
+            type: remove
+            field: attributes["log.file.path"]
+          - id: remove_filename
+            type: remove
+            field: attributes["log.file.name"]
+          - id: remove_stream
+            type: remove
+            field: attributes["log.iostream"]
+          - id: remove_time
+            type: remove
+            field: attributes.time
+    
+    processors:
+      batch:
+      resourcedetection:
+        detectors: [ "env", "system" ]
+        override: false
+    
+    exporters:
+      otlphttp/metrics:
+        endpoint: http://localhost:9090/api/v1/otlp
+      otlphttp/traces:
+        endpoint: http://localhost:4418
+      otlphttp/logs:
+        endpoint: http://localhost:3100/otlp
+      debug/metrics:
+        verbosity: detailed
+      debug/traces:
+        verbosity: detailed
+      debug/logs:
+        verbosity: detailed
+      nop:
+    
+    service:
+      pipelines:
+        traces:
+          receivers: [ otlp ]
+          processors: [ batch ]
+          exporters: [ otlphttp/traces ]
+        metrics:
+          receivers: [ otlp, prometheus/collector ]
+          processors: [ batch ]
+          exporters: [ otlphttp/metrics ]
+        logs/otlp:
+          receivers: [ otlp ]
+          processors: [ batch ]
+          exporters: [ otlphttp/logs ]
+        logs/json-ecs:
+          receivers: [ filelog/json-ecs ]
+          processors: [ batch ]
+          exporters: [ otlphttp/logs ]
+          # exporters: [ otlphttp/logs, debug/logs ]  # Uncomment this line to enable debug logging
+          
+          

examples/java/json-logging-ecs/k8s/dice.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: dice
+spec:
+  selector:
+    app: dice
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dice
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dice
+  template:
+    metadata:
+      labels:
+        app: dice
+    spec:
+      containers:
+      - name: dice
+        image: dice:1.1-SNAPSHOT
+        imagePullPolicy: Never
+        ports:
+        - containerPort: 8080
+        env:
+          - name: OTEL_EXPORTER_OTLP_ENDPOINT
+            value: "http://lgtm:4318"
+          - name: OTEL_LOGS_EXPORTER
+            value: "none" # to avoid duplicate logs
+          - name: OTEL_RESOURCE_ATTRIBUTES
+            value: service.name=dice,service.namespace=shop,service.version=1.1,deployment.environment=staging
+          - name: OTEL_INSTRUMENTATION_COMMON_MDC_RESOURCE_ATTRIBUTES
+            value: "service.namespace,service.instance.id,deployment.environment"
+          - name: SERVICE_NAME
+            value: dice
+

examples/java/json-logging-ecs/k8s/lgtm.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: lgtm
+spec:
+  selector:
+    app: lgtm
+  ports:
+    - name: grafana
+      protocol: TCP
+      port: 3000
+      targetPort: 3000
+    - name: otel-grpc
+      protocol: TCP
+      port: 4317
+      targetPort: 4317
+    - name: otel-http
+      protocol: TCP
+      port: 4318
+      targetPort: 4318
+    - name: prometheus # needed for automated tests
+      protocol: TCP
+      port: 9090
+      targetPort: 9090
+    - name: loki # needed for automated tests
+      protocol: TCP
+      port: 3100
+      targetPort: 3100
+    - name: tempo # needed for automated tests
+      protocol: TCP
+      port: 3200
+      targetPort: 3200
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lgtm
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: lgtm
+  template:
+    metadata:
+      labels:
+        app: lgtm
+    spec:
+      containers:
+        - name: lgtm
+          image: grafana/otel-lgtm:latest
+          ports:
+            - containerPort: 3000
+            - containerPort: 4317
+            - containerPort: 4318
+            - containerPort: 9090 # needed for automated tests
+            - containerPort: 3100 # needed for automated tests
+            - containerPort: 3200 # needed for automated tests
+          readinessProbe:
+            exec:
+              command:
+                - cat
+                - /tmp/ready
+          volumeMounts:
+            - mountPath: /otel-lgtm/otelcol-config.yaml
+              name: otel-collector-config
+              subPath: otel-collector-config.yaml
+              readOnly: true
+            - mountPath: /var/log
+              name: varlog
+              readOnly: true
+            - mountPath: /var/lib/docker/containers
+              name: varlibdockercontainers
+              readOnly: true
+          env:
+            - name: ENABLE_LOGS_OTELCOL
+              value: "true"
+      volumes:
+        - name: otel-collector-config
+          configMap:
+            name: otel-collector-config
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers

examples/java/json-logging-ecs/logback-spring.xml

@@ -0,0 +1,7 @@
+<configuration>
+    <include resource="co/elastic/logging/logback/boot/ecs-console-appender.xml" />
+
+    <root level="info">
+        <appender-ref ref="ECS_JSON_CONSOLE"/>
+    </root>
+</configuration>