add wasm filter piugin (#1325)

* add wasm filter piugin Signed-off-by: jiuxia211 <[email protected]> * feat: update AccessiblePaths type to string[] and correct Event_Format field Signed-off-by: jiuxia211 <[email protected]> * delete the WASM file mounting in Fluent Bit DaemonSet Signed-off-by: jiuxia211 <[email protected]> --------- Signed-off-by: jiuxia211 <[email protected]>
fluent · Aug 30, 2024 · 9b31cf2 · 9b31cf2
1 parent 51ad7fd
commit 9b31cf2
Show file tree

Hide file tree

Showing 13 changed files with 481 additions and 1 deletion.
diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types.go b/apis/fluentbit/v1alpha2/clusterfilter_types.go
@@ -71,6 +71,8 @@ type FilterItem struct {
 	Multiline *filter.Multiline `json:"multiline,omitempty"`
 	// LogToMetrics defines a Log to Metrics Filter configuration.
 	LogToMetrics *filter.LogToMetrics `json:"logToMetrics,omitempty"`
+	// Wasm defines a Wasm configuration.
+	Wasm *filter.Wasm `json:"wasm,omitempty"`
 	// CustomPlugin defines a Custom plugin configuration.
 	CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"`
 }

diff --git a/apis/fluentbit/v1alpha2/plugins/filter/wasm.go b/apis/fluentbit/v1alpha2/plugins/filter/wasm.go
@@ -0,0 +1,62 @@
+package filter
+
+import (
+	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
+	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params"
+)
+
+// +kubebuilder:object:generate:=true
+
+// Wasm Filter allows you to modify the incoming records using Wasm technology.
+// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/wasm**
+type Wasm struct {
+	plugins.CommonParams `json:",inline"`
+	// Path to the built Wasm program that will be used. This can be a relative path against the main configuration file.
+	WasmPath string `json:"wasmPath,omitempty"`
+	// Define event format to interact with Wasm programs: msgpack or json. Default: json
+	EventFormat string `json:"eventFormat,omitempty"`
+	// Wasm function name that will be triggered to do filtering. It's assumed that the function is built inside the Wasm program specified above.
+	FunctionName string `json:"functionName,omitempty"`
+	// Specify the whitelist of paths to be able to access paths from WASM programs.
+	AccessiblePaths []string `json:"accessiblePaths,omitempty"`
+	// Size of the heap size of Wasm execution. Review unit sizes for allowed values.
+	// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
+	WasmHeapSize string `json:"wasmHeapSize,omitempty"`
+	// Size of the stack size of Wasm execution. Review unit sizes for allowed values.
+	// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
+	WasmStackSize string `json:"wasmStackSize,omitempty"`
+}
+
+// Name is the name of the filter plugin.
+func (*Wasm) Name() string {
+	return "wasm"
+}
+
+// Params represents the config options for the filter plugin.
+func (w *Wasm) Params(_ plugins.SecretLoader) (*params.KVs, error) {
+	kvs := params.NewKVs()
+	err := w.AddCommonParams(kvs)
+	if err != nil {
+		return kvs, err
+	}
+	if w.WasmPath != "" {
+		kvs.Insert("Wasm_Path", w.WasmPath)
+	}
+	if w.EventFormat != "" {
+		kvs.Insert("Event_Format", w.EventFormat)
+	}
+	if w.FunctionName != "" {
+		kvs.Insert("Function_Name", w.FunctionName)
+	}
+	for _, p := range w.AccessiblePaths {
+		kvs.Insert("Accessible_Paths", p)
+	}
+	if w.WasmHeapSize != "" {
+		kvs.Insert("Wasm_Heap_Size", w.WasmHeapSize)
+	}
+	if w.WasmStackSize != "" {
+		kvs.Insert("Wasm_Stack_Size", w.WasmStackSize)
+	}
+
+	return kvs, nil
+}
diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go
diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml
@@ -879,6 +879,53 @@ spec:
                           format: int64
                           type: integer
                       type: object
+                    wasm:
+                      description: Wasm defines a Wasm configuration.
+                      properties:
+                        accessiblePaths:
+                          description: Specify the whitelist of paths to be able to
+                            access paths from WASM programs.
+                          items:
+                            type: string
+                          type: array
+                        alias:
+                          description: Alias for the plugin
+                          type: string
+                        eventFormat:
+                          description: 'Define event format to interact with Wasm
+                            programs: msgpack or json. Default: json'
+                          type: string
+                        functionName:
+                          description: Wasm function name that will be triggered to
+                            do filtering. It's assumed that the function is built
+                            inside the Wasm program specified above.
+                          type: string
+                        retryLimit:
+                          description: 'RetryLimit describes how many times fluent-bit
+                            should retry to send data to a specific output. If set
+                            to false fluent-bit will try indefinetly. If set to any
+                            integer N>0 it will try at most N+1 times. Leading zeros
+                            are not allowed (values such as 007, 0150, 01 do not work).
+                            If this property is not defined fluent-bit will use the
+                            default value: 1.'
+                          pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$
+                          type: string
+                        wasmHeapSize:
+                          description: Size of the heap size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                        wasmPath:
+                          description: Path to the built Wasm program that will be
+                            used. This can be a relative path against the main configuration
+                            file.
+                          type: string
+                        wasmStackSize:
+                          description: Size of the stack size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                      type: object
                   type: object
                 type: array
               logLevel:

diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml
@@ -879,6 +879,53 @@ spec:
                           format: int64
                           type: integer
                       type: object
+                    wasm:
+                      description: Wasm defines a Wasm configuration.
+                      properties:
+                        accessiblePaths:
+                          description: Specify the whitelist of paths to be able to
+                            access paths from WASM programs.
+                          items:
+                            type: string
+                          type: array
+                        alias:
+                          description: Alias for the plugin
+                          type: string
+                        eventFormat:
+                          description: 'Define event format to interact with Wasm
+                            programs: msgpack or json. Default: json'
+                          type: string
+                        functionName:
+                          description: Wasm function name that will be triggered to
+                            do filtering. It's assumed that the function is built
+                            inside the Wasm program specified above.
+                          type: string
+                        retryLimit:
+                          description: 'RetryLimit describes how many times fluent-bit
+                            should retry to send data to a specific output. If set
+                            to false fluent-bit will try indefinetly. If set to any
+                            integer N>0 it will try at most N+1 times. Leading zeros
+                            are not allowed (values such as 007, 0150, 01 do not work).
+                            If this property is not defined fluent-bit will use the
+                            default value: 1.'
+                          pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$
+                          type: string
+                        wasmHeapSize:
+                          description: Size of the heap size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                        wasmPath:
+                          description: Path to the built Wasm program that will be
+                            used. This can be a relative path against the main configuration
+                            file.
+                          type: string
+                        wasmStackSize:
+                          description: Size of the stack size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                      type: object
                   type: object
                 type: array
               logLevel:

diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml
@@ -879,6 +879,53 @@ spec:
                           format: int64
                           type: integer
                       type: object
+                    wasm:
+                      description: Wasm defines a Wasm configuration.
+                      properties:
+                        accessiblePaths:
+                          description: Specify the whitelist of paths to be able to
+                            access paths from WASM programs.
+                          items:
+                            type: string
+                          type: array
+                        alias:
+                          description: Alias for the plugin
+                          type: string
+                        eventFormat:
+                          description: 'Define event format to interact with Wasm
+                            programs: msgpack or json. Default: json'
+                          type: string
+                        functionName:
+                          description: Wasm function name that will be triggered to
+                            do filtering. It's assumed that the function is built
+                            inside the Wasm program specified above.
+                          type: string
+                        retryLimit:
+                          description: 'RetryLimit describes how many times fluent-bit
+                            should retry to send data to a specific output. If set
+                            to false fluent-bit will try indefinetly. If set to any
+                            integer N>0 it will try at most N+1 times. Leading zeros
+                            are not allowed (values such as 007, 0150, 01 do not work).
+                            If this property is not defined fluent-bit will use the
+                            default value: 1.'
+                          pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$
+                          type: string
+                        wasmHeapSize:
+                          description: Size of the heap size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                        wasmPath:
+                          description: Path to the built Wasm program that will be
+                            used. This can be a relative path against the main configuration
+                            file.
+                          type: string
+                        wasmStackSize:
+                          description: Size of the stack size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                      type: object
                   type: object
                 type: array
               logLevel:

diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml
@@ -879,6 +879,53 @@ spec:
                           format: int64
                           type: integer
                       type: object
+                    wasm:
+                      description: Wasm defines a Wasm configuration.
+                      properties:
+                        accessiblePaths:
+                          description: Specify the whitelist of paths to be able to
+                            access paths from WASM programs.
+                          items:
+                            type: string
+                          type: array
+                        alias:
+                          description: Alias for the plugin
+                          type: string
+                        eventFormat:
+                          description: 'Define event format to interact with Wasm
+                            programs: msgpack or json. Default: json'
+                          type: string
+                        functionName:
+                          description: Wasm function name that will be triggered to
+                            do filtering. It's assumed that the function is built
+                            inside the Wasm program specified above.
+                          type: string
+                        retryLimit:
+                          description: 'RetryLimit describes how many times fluent-bit
+                            should retry to send data to a specific output. If set
+                            to false fluent-bit will try indefinetly. If set to any
+                            integer N>0 it will try at most N+1 times. Leading zeros
+                            are not allowed (values such as 007, 0150, 01 do not work).
+                            If this property is not defined fluent-bit will use the
+                            default value: 1.'
+                          pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$
+                          type: string
+                        wasmHeapSize:
+                          description: Size of the heap size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                        wasmPath:
+                          description: Path to the built Wasm program that will be
+                            used. This can be a relative path against the main configuration
+                            file.
+                          type: string
+                        wasmStackSize:
+                          description: Size of the stack size of Wasm execution. Review
+                            unit sizes for allowed values.
+                          pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                          type: string
+                      type: object
                   type: object
                 type: array
               logLevel:

diff --git a/docs/fluentbit.md b/docs/fluentbit.md
@@ -285,6 +285,7 @@ Filter is the Schema for namespace level filter API
 | aws | Aws defines a Aws configuration. | *[filter.AWS](plugins/filter/aws.md) |
 | multiline | Multiline defines a Multiline configuration. | *[filter.Multiline](plugins/filter/multiline.md) |
 | logToMetrics | LogToMetrics defines a Log to Metrics Filter configuration. | *[filter.LogToMetrics](plugins/filter/logtometrics.md) |
+| wasm | Wasm defines a Wasm configuration. | *[filter.Wasm](plugins/filter/wasm.md) |
 | customPlugin | CustomPlugin defines a Custom plugin configuration. | *custom.CustomPlugin |
 
 [Back to TOC](#table-of-contents)

diff --git a/docs/plugins/fluentbit/filter/wasm.md b/docs/plugins/fluentbit/filter/wasm.md
@@ -0,0 +1,14 @@
+# WASM
+
+The Wasm Filter allows you to modify the incoming records using Wasm technology. <br /> **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/wasm**
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| wasmPath | Path to the built Wasm program that will be used. This can be a relative path against the main configuration file. | string |
+| eventFormat | Define event format to interact with Wasm programs: msgpack or json. Default: json | string |
+| functionName | Wasm function name that will be triggered to do filtering. It's assumed that the function is built inside the Wasm program specified above. | string |
+| accessiblePaths | Specify the whitelist of paths to be able to access paths from WASM programs. | string |
+| wasmHeapSize | Size of the heap size of Wasm execution. Review unit sizes for allowed values. | string |
+| wasmStackSize | Size of the stack size of Wasm execution. Review unit sizes for allowed values. | string |
+