feat: add containerExclusions

[clavinjune]

add containerExclusions to specifically exclude container logs.

Since we use td-agent 4.3.2 fluentd 0.14.25 we can't use regex matcher, so we redirect the exclusions to @type null instead

helm template render result:

values.yaml

+apps:
+  - name: with-namespace-with-exclusions
+    namespace: bar
+    applicationGroupSecret: appSecret
+    produceUrl: https://produceUrl
+    containerExclusions:
+      - container1
+      - container2
+  - name: no-namespace-with-exclusions
+    applicationGroupSecret: appSecret
+    produceUrl: https://produceUrl
+    containerExclusions:
+      - container1
+  - name: with-namespace-no-exclusions
+    namespace: bar
+    applicationGroupSecret: appSecret
+    produceUrl: https://produceUrl
+    containerExclusions: []
+  - name: no-namespace-no-exclusions
+    applicationGroupSecret: appSecret
+    produceUrl: https://produceUrl
+    containerExclusions: []

before

    <match kubernetes.var.log.containers.with-namespace-with-exclusions-**_bar_**.log>
      @type barito_batch_k8s
      name with-namespace-with-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>
    <match kubernetes.var.log.containers.no-namespace-with-exclusions-**.log>
      @type barito_batch_k8s
      name no-namespace-with-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>
    <match kubernetes.var.log.containers.with-namespace-no-exclusions-**_bar_**.log>
      @type barito_batch_k8s
      name with-namespace-no-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>
    <match kubernetes.var.log.containers.no-namespace-no-exclusions-**.log>
      @type barito_batch_k8s
      name no-namespace-no-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>

after

    <match kubernetes.var.log.containers.with-namespace-with-exclusions-**_bar_{container1,container2}**.log>
      @type null
    </match>
    <match kubernetes.var.log.containers.with-namespace-with-exclusions-**_bar_**.log>
      @type barito_batch_k8s
      name with-namespace-with-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>
    <match kubernetes.var.log.containers.no-namespace-with-exclusions-**_**_{container1}**.log>
      @type null
    </match>
    <match kubernetes.var.log.containers.no-namespace-with-exclusions-**.log>
      @type barito_batch_k8s
      name no-namespace-with-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>
    <match kubernetes.var.log.containers.with-namespace-no-exclusions-**_bar_**.log>
      @type barito_batch_k8s
      name with-namespace-no-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>
    <match kubernetes.var.log.containers.no-namespace-no-exclusions-**.log>
      @type barito_batch_k8s
      name no-namespace-no-exclusions
      cluster_name 
      application_name 
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>

diff

--- /tmp/a.yml	2023-04-03 14:37:35.000000000 +0700
+++ /tmp/b.yml	2023-04-03 14:37:59.000000000 +0700
@@ -23,6 +23,9 @@
       @type kubernetes_metadata
       de_dot false
     </filter>
+    <match kubernetes.var.log.containers.with-namespace-with-exclusions-**_bar_{container1,container2}**.log>
+      @type null
+    </match>
     <match kubernetes.var.log.containers.with-namespace-with-exclusions-**_bar_**.log>
       @type barito_batch_k8s
       name with-namespace-with-exclusions
@@ -47,6 +50,9 @@
         disable_chunk_backup true
       </buffer>
     </match>
+    <match kubernetes.var.log.containers.no-namespace-with-exclusions-**_**_{container1}**.log>
+      @type null
+    </match>
     <match kubernetes.var.log.containers.no-namespace-with-exclusions-**.log>
       @type barito_batch_k8s
       name no-namespace-with-exclusions

[clavinjune]

Caveat 1

Using this method, the namespaced apps will be safe, but not with the non-namespaced

let's say we have 2 apps:

• app foo on namespace ns1 has container1
• app foo on namespace ns2 has container1

if we set the apps like below:

- name: foo
    applicationGroupSecret: appSecret
    produceUrl: https://produceUrl
    containerExclusions:
      - container1
- name: foo
    namespace: ns2
    applicationGroupSecret: appSecret
    produceUrl: https://produceUrl
    containerExclusions: []

the container1 from foo on ns2 will be excluded because the foo on ns1 doesn't have any namespace defined

Caveat 2

if we have 2 containers container1 and container2 and we only define containerExclusions: ["container"], both of the container will be excluded since the rendered result will be {container}**.log

[clavinjune]

values.yaml

+# these sidecars logs will be stored in the new application name
+separatedSidecars:
+  - istio
+
+apps:
+  - name: with-namespace-with-exclusions
+    namespace: bar
+    applicationGroupSecret: appSecret
+    produceUrl: https://produceUrl
+  - name: no-namespace-with-exclusions
+    applicationGroupSecret: appSecret
+    produceUrl: https://produceUrl

Using .Values.separatedContainers

    <match kubernetes.var.log.containers.with-namespace-with-exclusions-**_bar_{istio}**.log>
      @type barito_batch_k8s
      name with-namespace-with-exclusions
      cluster_name
      application_name istio
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>

    <match kubernetes.var.log.containers.with-namespace-with-exclusions-**_bar_**.log>
      @type barito_batch_k8s
      name with-namespace-with-exclusions
      cluster_name
      application_name
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>

    <match kubernetes.var.log.containers.no-namespace-with-exclusions-**_**_{istio}**.log>
      @type barito_batch_k8s
      name no-namespace-with-exclusions
      cluster_name
      application_name istio
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>

    <match kubernetes.var.log.containers.no-namespace-with-exclusions-**_**_**.log>
      @type barito_batch_k8s
      name no-namespace-with-exclusions
      cluster_name
      application_name
      application_group_secret appSecret
      produce_url https://produceUrl
      <buffer>
        flush_at_shutdown true
        flush_thread_count 8
        flush_thread_interval 1
        flush_thread_burst_interval 1
        flush_mode interval
        flush_interval 1s
        queued_chunks_limit_size 1
        overflow_action drop_oldest_chunk
        retry_timeout 0s
        retry_max_times 3
        retry_type exponential_backoff
        retry_wait 1s
        retry_exponential_backoff_base 2
        disable_chunk_backup true
      </buffer>
    </match>