Bfs/vz 10302

.github/dependabot.yml

@@ -13,7 +13,7 @@ updates:
     schedule:
       interval: "daily"
   - package-ecosystem: "docker"
-    directory: "/cmd/fluent-watcher/fluentd/base"
+    directory: "/cmd/fluent-watcher/fluentd"
     schedule:
       interval: "daily"
   - package-ecosystem: "docker"

.github/workflows/build-fb-image.yaml

@@ -13,8 +13,8 @@ on:
       - "pkg/filenotify/**"
 
 env:
-  FB_IMG: 'kubesphere/fluent-bit:v2.0.11'
-  FB_IMG_DEBUG: 'kubesphere/fluent-bit:v2.0.11-debug'
+  FB_IMG: 'kubesphere/fluent-bit:v2.1.6'
+  FB_IMG_DEBUG: 'kubesphere/fluent-bit:v2.1.6-debug'
 
 jobs:
   build:

.github/workflows/build-fd-image.yaml

@@ -51,7 +51,7 @@ jobs:
 
   build-arm64:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    timeout-minutes: 90 
     name: Build arm64 Image for Fluentd
     steps:
       - name: Install Go
@@ -79,7 +79,7 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Build and Push arm64 Image for Fluentd
+      - name: Build and Push arm64 base Image for Fluentd
         run: |
           make build-fd-arm64-base -e FD_IMG_BASE=${{ env.FD_IMG_BASE }}
 

.github/workflows/lint-test.yaml

@@ -24,7 +24,7 @@ jobs:
           python-version: 3.7
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.1.0
+        uses: helm/chart-testing-action@v2.4.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -38,7 +38,7 @@ jobs:
         run: ct lint --chart-dirs charts/fluent-operator --config charts/ct.yaml
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.5.0
+        uses: helm/kind-action@v1.7.0
         if: steps.list-changed.outputs.changed == 'true'
 
       - name: Run chart-testing (install)

CHANGELOG.md

@@ -1,3 +1,49 @@
+## 2.3.0 / 2023-06-05
+### Features
+- Feat: Adding influxdb plugin (#690)
+- Feat: Add EnvVars support to FluentD (#697)
+- Feat: Add Pod Annotations support to FluentD (#698)
+- Feat: Fluent operator & fluentbit: Added tolerations, nodeSelector + more (#704)
+- Feat: Add fluentbit.affinity configuration (#726)
+- Feat: Allow setting fluentd PodSecurityContext (#744)
+- Feat: Fluentd in_tail plugin (#753)
+- Feat: Add missing fluentd buffer options (#757)
+- Feat: Add AWS Kinesis Data Streams output plugin for Fluent Bit (#768)
+- Feat: Add global log_level support for fluentd (#770)
+- Feat: Add scheduler support for fluentbit & fluentd (#771)
+
+### ENHANCEMENT
+- EnvVars support in fluentbit helm template (#706) 
+- Add uri field for each telemetry type in opentelemetry plugin, remove old uri field (#708)
+- Adjust fluentd watcher dependabot (#716)
+- remove the deprecated -i flag in go build (#720) 
+- Adjust fluentd arm64 image build timeout (#721)
+- Adjust edge metrics collection config (#736) 
+- Add some fluentbit helm opts (#743)
+- Align CRDs and Operator with the fluentbit loki output (#756)
+- Fluent-bit upgrade to v2.1.4 (#767)
+- build(deps): Bump k8s.io/apimachinery from 0.26.3 to 0.27.1 (#701)
+- build(deps): Bump helm/chart-testing-action from 2.1.0 to 2.4.0 (#710)
+- build(deps): Bump k8s.io/klog/v2 from 2.90.1 to 2.100.1 (#712)
+- build(deps): Bump golang from 1.20.3-alpine3.17 to 1.20.4-alpine3.17 in /cmd/fluent-manager (#713)
+- build(deps): Bump golang from 1.20.3-alpine3.16 to 1.20.4-alpine3.16 in /cmd/fluent-watcher/fluentbit (#714)
+- build(deps): Bump golang from 1.20.2 to 1.20.4 in /docs/best-practice/forwarding-logs-via-http (#715) 
+- build(deps): Bump golang from 1.19.2-alpine3.16 to 1.20.4-alpine3.16 in /cmd/fluent-watcher/fluentd (#717)
+- build(deps): Bump arm64v8/ruby from 3.1-slim-bullseye to 3.2-slim-bullseye in /cmd/fluent-watcher/fluentd (#718)
+- build(deps): Bump alpine from 3.16 to 3.17 in /cmd/fluent-watcher/fluentd (#719)
+- build(deps): Bump github.com/onsi/gomega from 1.27.6 to 1.27.7 (#748)
+- build(deps): Bump k8s.io/apimachinery from 0.27.1 to 0.27.2 (#751)
+- build(deps): Bump helm/kind-action from 1.5.0 to 1.7.0 (#765)
+
+### BUGFIX 
+- Fix: Fix missing log level  (#691) 
+- Fix: Fix rewrite_tag match rule and trim start of string pattern (#692) 
+- Fix(docs): Update cluster outputs docs link (#724)
+- Fix: dereference pointers in parser filter plugin for fluentd (#745)
+- Fix: fluentbit namespace-logging: only generate rewrite tag config once (#746)
+- Fix: minor typo fix for firehose (#764)
+- Fix: fix typo for estimate_current_event in fluentd (#769) 
+
 ## 2.2.0 / 2023-04-07
 ### Features
 - Feat: Adding Fluentd cloudwatch plugin (#586)

Makefile

@@ -1,7 +1,7 @@
 VERSION?=$(shell cat VERSION | tr -d " \t\n\r")
 # Image URL to use all building/pushing image targets
-FB_IMG ?= kubesphere/fluent-bit:v2.0.11
-FB_IMG_DEBUG ?= kubesphere/fluent-bit:v2.0.11-debug
+FB_IMG ?= kubesphere/fluent-bit:v2.1.6
+FB_IMG_DEBUG ?= kubesphere/fluent-bit:v2.1.6-debug
 FD_IMG ?= kubesphere/fluentd:v1.15.3
 FO_IMG ?= kubesphere/fluent-operator:$(VERSION)
 FD_IMG_BASE ?= kubesphere/fluentd:v1.15.3-arm64-base
@@ -150,6 +150,11 @@ CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: go-deps ## Download controller-gen locally if necessary.
 	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
 
+GINKGO = $(shell pwd)/bin/ginkgo
+ginkgo: go-deps ## Download controller-gen locally if necessary.
+	$(call go-get-tool,$(GINKGO),github.com/onsi/ginkgo/[email protected])
+
+
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: go-deps ## Download kustomize locally if necessary.
 	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
@@ -179,10 +184,10 @@ go-deps: # download go dependencies
 docs-update: # update api docs
 	go run ./cmd/doc-gen/main.go
 
-e2e: # make e2e tests
+e2e: ginkgo # make e2e tests
 	chmod a+x tests/scripts/fluentd_e2e.sh && bash tests/scripts/fluentd_e2e.sh
 
-helm-e2e: # make helm e2e tests
+helm-e2e: ginkgo # make helm e2e tests
 	chmod a+x tests/scripts/fluentd_helm_e2e.sh && bash tests/scripts/fluentd_helm_e2e.sh
 
 update-helm-package: # update helm repo

RELEASE.md

@@ -35,6 +35,7 @@ This page describes the release process and the currently planned schedule for u
 | v2.0.1           | 2023-02-08                                 | Elon Cheng (GitHub: @wenchajun)         |
 | v2.1.0           | 2023-03-13                                 | Elon Cheng (GitHub: @wenchajun)         |
 | v2.2.0           | 2023-04-07                                 | Elon Cheng (GitHub: @wenchajun)         |
+| v2.3.0           | 2023-06-05                                 | Elon Cheng (GitHub: @wenchajun)         |
 
 # How to cut a new release
 

apis/fluentbit/v1alpha2/clusterfilter_types.go

@@ -40,6 +40,8 @@ type FilterSpec struct {
 	// A regular expression to match against the tags of incoming records.
 	// Use this option if you want to use the full regex syntax.
 	MatchRegex string `json:"matchRegex,omitempty"`
+	// +kubebuilder:validation:Enum:=off;error;warning;info;debug;trace
+	LogLevel string `json:"logLevel,omitempty"`
 	// A set of filter plugins in order.
 	FilterItems []FilterItem `json:"filters,omitempty"`
 }
@@ -117,6 +119,9 @@ func (list ClusterFilterList) Load(sl plugins.SecretLoader) (string, error) {
 			if p.Name() != "" {
 				buf.WriteString(fmt.Sprintf("    Name    %s\n", p.Name()))
 			}
+			if item.Spec.LogLevel != "" {
+				buf.WriteString(fmt.Sprintf("    Log_Level    %s\n", item.Spec.LogLevel))
+			}
 			if item.Spec.Match != "" {
 				buf.WriteString(fmt.Sprintf("    Match    %s\n", item.Spec.Match))
 			}

apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go

@@ -46,6 +46,27 @@ type FluentBitConfigSpec struct {
 	Namespace *string `json:"namespace,omitempty"`
 }
 
+type Storage struct {
+	// Select an optional location in the file system to store streams and chunks of data/
+	Path string `json:"path,omitempty"`
+	// Configure the synchronization mode used to store the data into the file system
+	// +kubebuilder:validation:Enum:=normal;full
+	Sync string `json:"sync,omitempty"`
+	// Enable the data integrity check when writing and reading data from the filesystem
+	// +kubebuilder:validation:Enum:=on;off
+	Checksum string `json:"checksum,omitempty"`
+	// This option configure a hint of maximum value of memory to use when processing these records
+	BacklogMemLimit string `json:"backlogMemLimit,omitempty"`
+	// If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory
+	MaxChunksUp *int64 `json:"maxChunksUp,omitempty"`
+	// If http_server option has been enabled in the Service section, this option registers a new endpoint where internal metrics of the storage layer can be consumed
+	// +kubebuilder:validation:Enum:=on;off
+	Metrics string `json:"metrics,omitempty"`
+	// When enabled, irrecoverable chunks will be deleted during runtime, and any other irrecoverable chunk located in the configured storage path directory will be deleted when Fluent-Bit starts.
+	// +kubebuilder:validation:Enum:=on;off
+	DeleteIrrecoverableChunks string `json:"deleteIrrecoverableChunks,omitempty"`
+}
+
 type Service struct {
 	// If true go to background on start
 	Daemon *bool `json:"daemon,omitempty"`
@@ -76,10 +97,12 @@ type Service struct {
 	// File to log diagnostic output
 	LogFile string `json:"logFile,omitempty"`
 	// Diagnostic level (error/warning/info/debug/trace)
-	// +kubebuilder:validation:Enum:=error;warning;info;debug;trace
+	// +kubebuilder:validation:Enum:=off;error;warning;info;debug;trace
 	LogLevel string `json:"logLevel,omitempty"`
 	// Optional 'parsers' config file (can be multiple)
 	ParsersFile string `json:"parsersFile,omitempty"`
+	// Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount separately for this storage. The hostPath type should be used for that Volume in Fluentbit daemon set.
+	Storage *Storage `json:"storage,omitempty"`
 }
 
 // +kubebuilder:object:root=true
@@ -149,6 +172,29 @@ func (s *Service) Params() *params.KVs {
 	if s.ParsersFile != "" {
 		m.Insert("Parsers_File", s.ParsersFile)
 	}
+	if s.Storage != nil {
+		if s.Storage.Path != "" {
+			m.Insert("storage.path", s.Storage.Path)
+		}
+		if s.Storage.Sync != "" {
+			m.Insert("storage.sync", s.Storage.Sync)
+		}
+		if s.Storage.Checksum != "" {
+			m.Insert("storage.checksum", s.Storage.Checksum)
+		}
+		if s.Storage.BacklogMemLimit != "" {
+			m.Insert("storage.backlog.mem_limit", s.Storage.BacklogMemLimit)
+		}
+		if s.Storage.Metrics != "" {
+			m.Insert("storage.metrics", s.Storage.Metrics)
+		}
+		if s.Storage.MaxChunksUp != nil {
+			m.Insert("storage.max_chunks_up", fmt.Sprint(*s.Storage.MaxChunksUp))
+		}
+		if s.Storage.DeleteIrrecoverableChunks != "" {
+			m.Insert("storage.delete_irrecoverable_chunks", s.Storage.DeleteIrrecoverableChunks)
+		}
+	}
 	return m
 }
 

apis/fluentbit/v1alpha2/clusterinput_types.go

@@ -37,6 +37,8 @@ type InputSpec struct {
 	// A user friendly alias name for this input plugin.
 	// Used in metrics for distinction of each configured input.
 	Alias string `json:"alias,omitempty"`
+	// +kubebuilder:validation:Enum:=off;error;warning;info;debug;trace
+	LogLevel string `json:"logLevel,omitempty"`
 	// Dummy defines Dummy Input configuration.
 	Dummy *input.Dummy `json:"dummy,omitempty"`
 	// Tail defines Tail Input configuration.
@@ -101,6 +103,9 @@ func (list ClusterInputList) Load(sl plugins.SecretLoader) (string, error) {
 			if item.Spec.Alias != "" {
 				buf.WriteString(fmt.Sprintf("    Alias    %s\n", item.Spec.Alias))
 			}
+			if item.Spec.LogLevel != "" {
+				buf.WriteString(fmt.Sprintf("    Log_Level    %s\n", item.Spec.LogLevel))
+			}
 			kvs, err := p.Params(sl)
 			if err != nil {
 				return err

apis/fluentbit/v1alpha2/clusteroutput_types.go

@@ -43,6 +43,10 @@ type OutputSpec struct {
 	// A user friendly alias name for this output plugin.
 	// Used in metrics for distinction of each configured output.
 	Alias string `json:"alias,omitempty"`
+	// Set the plugin's logging verbosity level. Allowed values are: off, error, warn, info, debug and trace, Defaults to the SERVICE section's Log_Level
+	// +kubebuilder:validation:Enum:=off;error;warning;info;debug;trace
+	LogLevel string `json:"logLevel,omitempty"`
+
 	// AzureBlob defines AzureBlob Output Configuration
 	AzureBlob *output.AzureBlob `json:"azureBlob,omitempty"`
 	// AzureLogAnalytics defines AzureLogAnalytics Output Configuration
@@ -72,10 +76,14 @@ type OutputSpec struct {
 	Loki *output.Loki `json:"loki,omitempty"`
 	// Syslog defines Syslog Output configuration.
 	Syslog *output.Syslog `json:"syslog,omitempty"`
+	// InfluxDB defines InfluxDB Output configuration.
+	InfluxDB *output.InfluxDB `json:"influxDB,omitempty"`
 	// DataDog defines DataDog Output configuration.
 	DataDog *output.DataDog `json:"datadog,omitempty"`
 	// Firehose defines Firehose Output configuration.
-	Fireose *output.Firehose `json:"firehose,omitempty"`
+	Firehose *output.Firehose `json:"firehose,omitempty"`
+	// Kinesis defines Kinesis Output configuration.
+	Kinesis *output.Kinesis `json:"kinesis,omitempty"`
 	// Stackdriver defines Stackdriver Output Configuration
 	Stackdriver *output.Stackdriver `json:"stackdriver,omitempty"`
 	// Splunk defines Splunk Output Configuration
@@ -86,6 +94,9 @@ type OutputSpec struct {
 	OpenTelemetry *output.OpenTelemetry `json:"opentelemetry,omitempty"`
 	// PrometheusRemoteWrite_types defines Prometheus Remote Write configuration.
 	PrometheusRemoteWrite *output.PrometheusRemoteWrite `json:"prometheusRemoteWrite,omitempty"`
+	// S3 defines S3 Output configuration.
+	S3 *output.S3 `json:"s3,omitempty"`
+
 	// CustomPlugin defines Custom Output configuration.
 	CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"`
 }
@@ -139,6 +150,9 @@ func (list ClusterOutputList) Load(sl plugins.SecretLoader) (string, error) {
 			if item.Spec.Match != "" {
 				buf.WriteString(fmt.Sprintf("    Match    %s\n", item.Spec.Match))
 			}
+			if item.Spec.LogLevel != "" {
+				buf.WriteString(fmt.Sprintf("    Log_Level    %s\n", item.Spec.LogLevel))
+			}
 			if item.Spec.MatchRegex != "" {
 				buf.WriteString(fmt.Sprintf("    Match_Regex    %s\n", item.Spec.MatchRegex))
 			}

apis/fluentbit/v1alpha2/collector_types.go

@@ -77,6 +77,8 @@ type CollectorSpec struct {
 	Ports []corev1.ContainerPort `json:"ports,omitempty"`
 	// Service represents configurations on the fluent-bit service.
 	Service CollectorService `json:"service,omitempty"`
+	// SchedulerName represents the desired scheduler for the Fluentbit collector pods
+	SchedulerName string `json:"schedulerName,omitempty"`
 }
 
 // CollectorService defines the service of the FluentBit

apis/fluentbit/v1alpha2/fluentbit_types.go

@@ -75,6 +75,8 @@ type FluentBitSpec struct {
 	Labels map[string]string `json:"labels,omitempty"`
 	// SecurityContext holds pod-level security attributes and common container settings.
 	SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
+	// ContainerSecurityContext holds container-level security attributes.
+	ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"`
 	// Host networking is requested for this pod. Use the host's network namespace. If this option is set, the ports that will be used must be specified. Default to false.
 	HostNetwork bool `json:"hostNetwork,omitempty"`
 	// EnvVars represent environment variables that can be passed to fluentbit pods.
@@ -96,6 +98,8 @@ type FluentBitSpec struct {
 	MetricsPort int32 `json:"metricsPort,omitempty"`
 	// Service represents configurations on the fluent-bit service.
 	Service FluentBitService `json:"service,omitempty"`
+	// SchedulerName represents the desired scheduler for fluent-bit pods.
+	SchedulerName string `json:"schedulerName,omitempty"`
 }
 
 // FluentBitService defines the service of the FluentBit

apis/fluentbit/v1alpha2/plugins/custom/custom_types.go

@@ -13,7 +13,7 @@ import (
 // +kubebuilder:object:generate:=true
 
 // CustomPlugin is used to support filter plugins that are not implemented yet. <br />
-// **For example usage, refer to https://github.com/jjsiv/fluent-operator/blob/master/docs/best-practice/custom-plugin.md**
+// **For example usage, refer to https://github.com/fluent/fluent-operator/blob/master/docs/best-practice/custom-plugin.md**
 type CustomPlugin struct {
 	Config string `json:"config,omitempty"`
 }

apis/fluentbit/v1alpha2/plugins/input/systemd_types.go

@@ -44,6 +44,12 @@ type Systemd struct {
 	// Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID.
 	// +kubebuilder:validation:Enum:=on;off
 	StripUnderscores string `json:"stripUnderscores,omitempty"`
+	// Specify the buffering mechanism to use. It can be memory or filesystem
+	// +kubebuilder:validation:Enum:=filesystem;memory
+	StorageType string `json:"storageType,omitempty"`
+	// Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached.
+	// +kubebuilder:validation:Enum:=on;off
+	PauseOnChunksOverlimit string `json:"pauseOnChunksOverlimit,omitempty"`
 }
 
 func (_ *Systemd) Name() string {
@@ -85,6 +91,12 @@ func (s *Systemd) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 	if s.StripUnderscores != "" {
 		kvs.Insert("Strip_Underscores", s.StripUnderscores)
 	}
+	if s.StorageType != "" {
+		kvs.Insert("storage.type", s.StorageType)
+	}
+	if s.PauseOnChunksOverlimit != "" {
+		kvs.Insert("storage.pause_on_chunks_overlimit", s.PauseOnChunksOverlimit)
+	}
 
 	return kvs, nil
 }

apis/fluentbit/v1alpha2/plugins/input/tail_types.go

@@ -93,6 +93,12 @@ type Tail struct {
 	// This will help to reassembly multiline messages originally split by Docker or CRI
 	//Specify one or Multiline Parser definition to apply to the content.
 	MultilineParser string `json:"multilineParser,omitempty"`
+	// Specify the buffering mechanism to use. It can be memory or filesystem
+	// +kubebuilder:validation:Enum:=filesystem;memory
+	StorageType string `json:"storageType,omitempty"`
+	// Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached.
+	// +kubebuilder:validation:Enum:=on;off
+	PauseOnChunksOverlimit string `json:"pauseOnChunksOverlimit,omitempty"`
 }
 
 func (_ *Tail) Name() string {
@@ -179,5 +185,11 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 	if t.MultilineParser != "" {
 		kvs.Insert("multiline.parser", t.MultilineParser)
 	}
+	if t.StorageType != "" {
+		kvs.Insert("storage.type", t.StorageType)
+	}
+	if t.PauseOnChunksOverlimit != "" {
+		kvs.Insert("storage.pause_on_chunks_overlimit", t.PauseOnChunksOverlimit)
+	}
 	return kvs, nil
 }