How to write connect logs to a kafka topic

vdesabou · Sep 22, 2023 · a999088 · a999088
1 parent 6373af1
commit a999088
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 0 deletions.
diff --git a/docs/content-template.md b/docs/content-template.md
@@ -298,6 +298,7 @@ There is also a Confluent Cloud version available [here](https://github.com/vdes
 ## 👾 Other Playgrounds
 
 - 📃 [How to write logs to files when using docker-compose](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/write-logs-to-files) :other/write-logs-to-files:
+- 📃 [How to write connect logs to a kafka topic](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/write-logs-to-topic) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet)
 - 📢 [Publish logs to kafka with Elastic Filebeat](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/filebeat-to-kafka) :other/filebeat-to-kafka:
 - 💠 [.NET](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/client-dotnet) client (producer/consumer) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet)
 - 🐚 [KafkaJS](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/client-kafkajs) client (producer/consumer) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet)

diff --git a/other/write-logs-to-topic/docker-compose.plaintext.yml b/other/write-logs-to-topic/docker-compose.plaintext.yml
@@ -0,0 +1,9 @@
+---
+version: '3.5'
+services:
+
+  connect:
+    volumes:
+      - ../../other/write-logs-to-topic/log4j.properties:/tmp/connect/log4j.properties
+    environment:
+      KAFKA_LOG4J_OPTS: "-Dlog4j.configuration=file:/tmp/connect/log4j.properties"
diff --git a/other/write-logs-to-topic/log4j.properties b/other/write-logs-to-topic/log4j.properties
@@ -0,0 +1,24 @@
+log4j.rootLogger=INFO, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern =[%d] %p %X{connector.context}%m (%c:%L)%n
+
+
+# loggers from CONNECT_LOG4J_LOGGERS env variable
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.I0Itec.zkclient=ERROR
+log4j.logger.org.reflections=ERROR
+
+
+# Send the logs to a kafka topic
+# there will be no key for the kafka records by default
+log4j.logger.org.apache.kafka.connect=INFO, kafka_appender
+log4j.appender.kafka_appender=org.apache.kafka.log4jappender.KafkaLog4jAppender
+log4j.appender.kafka_appender.layout=org.apache.log4j.PatternLayout
+log4j.appender.kafka_appender.layout.ConversionPattern=[%d] %p %X{connector.context}%m (%c:%L)%n
+log4j.appender.kafka_appender.BrokerList=broker:9092
+log4j.appender.kafka_appender.Topic=connect-logs
+log4j.logger.processing=INFO, kafka_appender
+
+
diff --git a/other/write-logs-to-topic/start.sh b/other/write-logs-to-topic/start.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+source ${DIR}/../../scripts/utils.sh
+
+${DIR}/../../environment/plaintext/start.sh "${PWD}/docker-compose.plaintext.yml"
+
+playground topic consume --topic connect-logs --tail
diff --git a/other/write-logs-to-topic/stop.sh b/other/write-logs-to-topic/stop.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+source ${DIR}/../../scripts/utils.sh
+
+stop_all "$DIR"