From df9a7cfe7e6b83fb85ae0e53ba0c59ef1731a811 Mon Sep 17 00:00:00 2001
From: Dmitrii Anoshin <anoshindx@gmail.com>
Date: Mon, 19 Dec 2022 17:53:24 -0800
Subject: [PATCH] Fix log collection in GKE Autopilot with fluentd (#609)

This change starts the transition towards assuming that containerd is the container engine used by default instead of trying to detect it. We are confident that at least GKE Autopilot don't use the docker engine anymore so we can exclude the autodection for that distribution. We will be adding other distributions to the exclusion list going forward, and eventually we can drop support of docker engine.
---
 CHANGELOG.md                                  |  3 +-
 .../templates/configmap-fluentd-cri.yaml      | 51 -------------------
 .../templates/configmap-fluentd.yaml          | 36 +++++++++++++
 .../templates/daemonset.yaml                  | 17 +++----
 .../logs-only/configmap-fluentd-cri.yaml      | 51 -------------------
 .../logs-only/configmap-fluentd.yaml          | 31 +++++++++++
 rendered/manifests/logs-only/daemonset.yaml   | 14 ++---
 7 files changed, 80 insertions(+), 123 deletions(-)
 delete mode 100644 helm-charts/splunk-otel-collector/templates/configmap-fluentd-cri.yaml
 delete mode 100644 rendered/manifests/logs-only/configmap-fluentd-cri.yaml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 17da56b757..5b9c12fb08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Fixed
 
-- Make sure the daemonset can start in GKE Autopiot (#602)
+- Make sure the daemonset can start in GKE Autopiot (#608)
+- Make containerd engine default in for fluentd logs and use always use it in GKE Autopiot (#609)
 
 ## [0.66.1] - 2022-12-08
 
diff --git a/helm-charts/splunk-otel-collector/templates/configmap-fluentd-cri.yaml b/helm-charts/splunk-otel-collector/templates/configmap-fluentd-cri.yaml
deleted file mode 100644
index 85ef3bb6f1..0000000000
--- a/helm-charts/splunk-otel-collector/templates/configmap-fluentd-cri.yaml
+++ /dev/null
@@ -1,51 +0,0 @@
-{{ $agent := fromYaml (include "splunk-otel-collector.agent" .) }}
-{{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") $agent.enabled (eq .Values.logsEngine "fluentd") }}
-{{/*
-Fluentd config parts applied only to clusters with containerd/cri-o runtime.
-*/}}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ template "splunk-otel-collector.fullname" . }}-fluentd-cri
-  labels:
-    {{- include "splunk-otel-collector.commonLabels" . | nindent 4 }}
-    app: {{ template "splunk-otel-collector.name" . }}
-    chart: {{ template "splunk-otel-collector.chart" . }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-data:
-
-  source.containers.parse.conf: |-
-    @type regexp
-    expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
-    time_format {{ .Values.fluentd.config.containers.criTimeFormat }}
-
-  output.filter.conf: |-
-    # = handle cri/containerd multiline format =
-    <filter tail.containers.var.log.containers.**>
-      @type concat
-      key log
-      partial_key partial_flag
-      partial_value P
-      separator ''
-      timeout_label @SPLUNK
-    </filter>
-
-  output.transform.conf: |-
-    # extract pod_uid and container_name for CRIO runtime
-    # currently CRI does not produce log paths with all the necessary
-    # metadata to parse out pod, namespace, container_name, container_id.
-    # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
-    <filter tail.containers.var.log.pods.**>
-      @type jq_transformer
-      jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
-    </filter>
-    # rename pod_uid and container_name to otel semantics.
-    <filter tail.containers.var.log.pods.**>
-      @type record_transformer
-      <record>
-        k8s.pod.uid ${record["pod_uid"]}
-        k8s.container.name ${record["container_name"]}
-      </record>
-    </filter>
-{{- end }}
diff --git a/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml b/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml
index c8535b0ccf..c6cf77cf57 100644
--- a/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml
+++ b/helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml
@@ -282,4 +282,40 @@ data:
         </format>
       </match>
     </label>
+
+  {{- /*
+    Default configuration files for CRI container engine.
+    Can be overridden for docker engine by prepare-fluentd-config container.
+  */}}
+  source.containers.parse.conf: |-
+    @type regexp
+    expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
+    time_format {{ .Values.fluentd.config.containers.criTimeFormat }}
+  output.filter.conf: |-
+    # = handle cri/containerd multiline format =
+    <filter tail.containers.var.log.containers.**>
+      @type concat
+      key log
+      partial_key partial_flag
+      partial_value P
+      separator ''
+      timeout_label @SPLUNK
+    </filter>
+  output.transform.conf: |-
+    # extract pod_uid and container_name for CRIO runtime
+    # currently CRI does not produce log paths with all the necessary
+    # metadata to parse out pod, namespace, container_name, container_id.
+    # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
+    <filter tail.containers.var.log.pods.**>
+      @type jq_transformer
+      jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
+    </filter>
+    # rename pod_uid and container_name to otel semantics.
+    <filter tail.containers.var.log.pods.**>
+      @type record_transformer
+      <record>
+        k8s.pod.uid ${record["pod_uid"]}
+        k8s.container.name ${record["container_name"]}
+      </record>
+    </filter>
 {{- end }}
diff --git a/helm-charts/splunk-otel-collector/templates/daemonset.yaml b/helm-charts/splunk-otel-collector/templates/daemonset.yaml
index 90c8dcff61..4bfbbbc12a 100644
--- a/helm-charts/splunk-otel-collector/templates/daemonset.yaml
+++ b/helm-charts/splunk-otel-collector/templates/daemonset.yaml
@@ -70,7 +70,7 @@ spec:
       {{- end }}
       {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (not .Values.isWindows) }}
       initContainers:
-        {{- if (eq .Values.logsEngine "fluentd") }}
+        {{- if and (eq .Values.logsEngine "fluentd") (not (eq .Values.distribution "gke/autopilot")) }}
         - name: prepare-fluentd-config
           image: {{ template "splunk-otel-collector.image.fluentd" . }}
           imagePullPolicy: {{ .Values.image.fluentd.pullPolicy }}
@@ -78,14 +78,11 @@ spec:
           securityContext:
             runAsUser: 0
           args:
-            - if [ -z "${LOG_FORMAT_TYPE}" ]; then
-                if [ "$(ls {{ .Values.fluentd.config.containers.pathDest }}/*/*json.log 2>/dev/null | wc -l)" != "0" ]; then
-                  export LOG_FORMAT_TYPE=json;
-                else
-                  export LOG_FORMAT_TYPE=cri;
-                fi;
+            - cp /fluentd/etc/common/* /fluentd/etc/;
+              {{- /* Update fluentd configs if docker engine is configured or discovered from existing log files. */}}
+              if [ "${LOG_FORMAT_TYPE}" == "json" ] || [ "$(ls {{ .Values.fluentd.config.containers.pathDest }}/*/*json.log 2>/dev/null | wc -l)" != "0" ]; then
+                  cp /fluentd/etc/json/* /fluentd/etc/;
               fi;
-              cp /fluentd/etc/common/* /fluentd/etc/${LOG_FORMAT_TYPE}/* /fluentd/etc/
           env:
             - name: LOG_FORMAT_TYPE
               value: "{{ .Values.fluentd.config.containers.logFormatType }}"
@@ -99,8 +96,6 @@ spec:
               mountPath: /fluentd/etc/common
             - name: fluentd-config-json
               mountPath: /fluentd/etc/json
-            - name: fluentd-config-cri
-              mountPath: /fluentd/etc/cri
         {{- else }}
         {{- if not (eq .Values.distribution "gke/autopilot") }}
         - name: migrate-checkpoint
@@ -412,8 +407,10 @@ spec:
         hostPath:
           path: {{ .Values.fluentd.config.journalLogPath | quote }}
       - name: fluentd-config
+      {{- if (not (eq .Values.distribution "gke/autopilot")) }}
         emptyDir: {}
       - name: fluentd-config-common
+      {{- end }}
         configMap:
           name: {{ template "splunk-otel-collector.fullname" . }}-fluentd
       - name: fluentd-config-cri
diff --git a/rendered/manifests/logs-only/configmap-fluentd-cri.yaml b/rendered/manifests/logs-only/configmap-fluentd-cri.yaml
deleted file mode 100644
index 6374b39628..0000000000
--- a/rendered/manifests/logs-only/configmap-fluentd-cri.yaml
+++ /dev/null
@@ -1,51 +0,0 @@
----
-# Source: splunk-otel-collector/templates/configmap-fluentd-cri.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: default-splunk-otel-collector-fluentd-cri
-  labels:
-    app.kubernetes.io/name: splunk-otel-collector
-    helm.sh/chart: splunk-otel-collector-0.66.1
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: default
-    app.kubernetes.io/version: "0.66.0"
-    app: splunk-otel-collector
-    chart: splunk-otel-collector-0.66.1
-    release: default
-    heritage: Helm
-data:
-
-  source.containers.parse.conf: |-
-    @type regexp
-    expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
-    time_format %Y-%m-%dT%H:%M:%S.%N%:z
-
-  output.filter.conf: |-
-    # = handle cri/containerd multiline format =
-    <filter tail.containers.var.log.containers.**>
-      @type concat
-      key log
-      partial_key partial_flag
-      partial_value P
-      separator ''
-      timeout_label @SPLUNK
-    </filter>
-
-  output.transform.conf: |-
-    # extract pod_uid and container_name for CRIO runtime
-    # currently CRI does not produce log paths with all the necessary
-    # metadata to parse out pod, namespace, container_name, container_id.
-    # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
-    <filter tail.containers.var.log.pods.**>
-      @type jq_transformer
-      jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
-    </filter>
-    # rename pod_uid and container_name to otel semantics.
-    <filter tail.containers.var.log.pods.**>
-      @type record_transformer
-      <record>
-        k8s.pod.uid ${record["pod_uid"]}
-        k8s.container.name ${record["container_name"]}
-      </record>
-    </filter>
diff --git a/rendered/manifests/logs-only/configmap-fluentd.yaml b/rendered/manifests/logs-only/configmap-fluentd.yaml
index ed730c3cdb..532a6f522c 100644
--- a/rendered/manifests/logs-only/configmap-fluentd.yaml
+++ b/rendered/manifests/logs-only/configmap-fluentd.yaml
@@ -319,3 +319,34 @@ data:
         </format>
       </match>
     </label>
+  source.containers.parse.conf: |-
+    @type regexp
+    expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<partial_flag>[FP]))? (?<log>.*)$/
+    time_format %Y-%m-%dT%H:%M:%S.%N%:z
+  output.filter.conf: |-
+    # = handle cri/containerd multiline format =
+    <filter tail.containers.var.log.containers.**>
+      @type concat
+      key log
+      partial_key partial_flag
+      partial_value P
+      separator ''
+      timeout_label @SPLUNK
+    </filter>
+  output.transform.conf: |-
+    # extract pod_uid and container_name for CRIO runtime
+    # currently CRI does not produce log paths with all the necessary
+    # metadata to parse out pod, namespace, container_name, container_id.
+    # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
+    <filter tail.containers.var.log.pods.**>
+      @type jq_transformer
+      jq '.record | . + (.source | capture("/var/log/pods/(?<pod_uid>[^/]+)/(?<container_name>[^/]+)/(?<container_retry>[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)'
+    </filter>
+    # rename pod_uid and container_name to otel semantics.
+    <filter tail.containers.var.log.pods.**>
+      @type record_transformer
+      <record>
+        k8s.pod.uid ${record["pod_uid"]}
+        k8s.container.name ${record["container_name"]}
+      </record>
+    </filter>
diff --git a/rendered/manifests/logs-only/daemonset.yaml b/rendered/manifests/logs-only/daemonset.yaml
index 8b21c2773f..27f9274eef 100644
--- a/rendered/manifests/logs-only/daemonset.yaml
+++ b/rendered/manifests/logs-only/daemonset.yaml
@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 0640e5c5b99d34b55fbd125d1c34fab8387b37d8c3c5a1e7c6857867a6ad7dfc
+        checksum/config: 0dd2001a5452e85b22ed2a419f03895f9b0a0382fb9d035be838299393f6080b
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true
@@ -50,14 +50,10 @@ spec:
           securityContext:
             runAsUser: 0
           args:
-            - if [ -z "${LOG_FORMAT_TYPE}" ]; then
-                if [ "$(ls /var/lib/docker/containers/*/*json.log 2>/dev/null | wc -l)" != "0" ]; then
-                  export LOG_FORMAT_TYPE=json;
-                else
-                  export LOG_FORMAT_TYPE=cri;
-                fi;
+            - cp /fluentd/etc/common/* /fluentd/etc/;
+              if [ "${LOG_FORMAT_TYPE}" == "json" ] || [ "$(ls /var/lib/docker/containers/*/*json.log 2>/dev/null | wc -l)" != "0" ]; then
+                  cp /fluentd/etc/json/* /fluentd/etc/;
               fi;
-              cp /fluentd/etc/common/* /fluentd/etc/${LOG_FORMAT_TYPE}/* /fluentd/etc/
           env:
             - name: LOG_FORMAT_TYPE
               value: ""
@@ -71,8 +67,6 @@ spec:
               mountPath: /fluentd/etc/common
             - name: fluentd-config-json
               mountPath: /fluentd/etc/json
-            - name: fluentd-config-cri
-              mountPath: /fluentd/etc/cri
       containers:
       - name: fluentd
         image: splunk/fluentd-hec:1.2.8