Skip to content

Commit

Permalink
add exec wasi input plugin (#1326)
Browse files Browse the repository at this point in the history
* add wasm filter piugin

Signed-off-by: jiuxia211 <[email protected]>

* feat: update AccessiblePaths type to string[] and correct Event_Format field

Signed-off-by: jiuxia211 <[email protected]>

* delete the WASM file mounting in Fluent Bit DaemonSet

Signed-off-by: jiuxia211 <[email protected]>

* add exec wasi input plugin

Signed-off-by: jiuxia211 <[email protected]>

* update docs for wasm filter and exec wasi input plugin

Signed-off-by: jiuxia211 <[email protected]>

---------

Signed-off-by: jiuxia211 <[email protected]>
  • Loading branch information
jiuxia211 authored Sep 2, 2024
1 parent 9b31cf2 commit eb81068
Show file tree
Hide file tree
Showing 11 changed files with 306 additions and 6 deletions.
7 changes: 5 additions & 2 deletions apis/fluentbit/v1alpha2/clusterinput_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,15 @@ package v1alpha2
import (
"bytes"
"fmt"
"reflect"
"sort"

"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/custom"
"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/input"
"github.com/fluent/fluent-operator/v2/pkg/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"reflect"
"sigs.k8s.io/yaml"
"sort"
)

// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
Expand Down Expand Up @@ -75,6 +76,8 @@ type InputSpec struct {
UDP *input.UDP `json:"udp,omitempty"`
// KubernetesEvents defines the KubernetesEvents input plugin configuration
KubernetesEvents *input.KubernetesEvents `json:"kubernetesEvents,omitempty"`
// ExecWasi defines the exec wasi input plugin configuration
ExecWasi *input.ExecWasi `json:"execWasi,omitempty"`
// Processors defines the processors configuration
// +kubebuilder:pruning:PreserveUnknownFields
Processors *plugins.Config `json:"processors,omitempty"`
Expand Down
73 changes: 73 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/input/exec_wasi_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package input

import (
"fmt"

"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params"
)

// +kubebuilder:object:generate:=true

// The exec_wasi input plugin, allows to execute WASM program that is WASI target like as external program and collects event logs from there.
// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/exec-wasi**
type ExecWasi struct {
// The place of a WASM program file.
WASIPath string `json:"wasiPath,omitempty"`
// Specify the name of a parser to interpret the entry as a structured message.
Parser string `json:"parser,omitempty"`
// Specify the whitelist of paths to be able to access paths from WASM programs.
AccessiblePaths []string `json:"accessiblePaths,omitempty"`
// Polling interval (seconds).
IntervalSec *int32 `json:"intervalSec,omitempty"`
// Polling interval (nanoseconds).
IntervalNSec *int64 `json:"intervalNSec,omitempty"`
// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
WasmHeapSize string `json:"wasmHeapSize,omitempty"`
// Size of the stack size of Wasm execution. Review unit sizes for allowed values.
// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
WasmStackSize string `json:"wasmStackSize,omitempty"`
// Size of the buffer (check unit sizes for allowed values)
// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
BufSize string `json:"bufSize,omitempty"`
// Indicates whether to run this input in its own thread. Default: false.
Threaded *bool `json:"threaded,omitempty"`
}

func (_ *ExecWasi) Name() string {
return "exec_wasi"
}

// Params implement Section() method
func (w *ExecWasi) Params(_ plugins.SecretLoader) (*params.KVs, error) {
kvs := params.NewKVs()

if w.WASIPath != "" {
kvs.Insert("WASI_Path", w.WASIPath)
}
if w.Parser != "" {
kvs.Insert("Parser", w.Parser)
}
for _, p := range w.AccessiblePaths {
kvs.Insert("Accessible_Paths", p)
}
if w.IntervalSec != nil {
kvs.Insert("Interval_Sec", fmt.Sprint(*w.IntervalSec))
}
if w.IntervalNSec != nil {
kvs.Insert("Interval_NSec", fmt.Sprint(*w.IntervalNSec))
}
if w.WasmHeapSize != "" {
kvs.Insert("Wasm_Heap_Size", w.WasmHeapSize)
}
if w.WasmStackSize != "" {
kvs.Insert("Wasm_Stack_Size", w.WasmStackSize)
}
if w.BufSize != "" {
kvs.Insert("Buf_Size", w.BufSize)
}
if w.Threaded != nil {
kvs.Insert("Threaded", fmt.Sprint(*w.Threaded))
}
return kvs, nil
}
35 changes: 35 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions apis/fluentbit/v1alpha2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,48 @@ spec:
plugin.
type: string
type: object
execWasi:
description: ExecWasi defines the exec wasi input plugin configuration
properties:
accessiblePaths:
description: Specify the whitelist of paths to be able to access
paths from WASM programs.
items:
type: string
type: array
bufSize:
description: Size of the buffer (check unit sizes for allowed
values)
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
intervalNSec:
description: Polling interval (nanoseconds).
format: int64
type: integer
intervalSec:
description: Polling interval (seconds).
format: int32
type: integer
parser:
description: Specify the name of a parser to interpret the entry
as a structured message.
type: string
threaded:
description: 'Indicates whether to run this input in its own thread.
Default: false.'
type: boolean
wasiPath:
description: The place of a WASM program file.
type: string
wasmHeapSize:
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
wasmStackSize:
description: Size of the stack size of Wasm execution. Review
unit sizes for allowed values.
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
type: object
fluentBitMetrics:
description: FluentBitMetrics defines Fluent Bit Metrics Input configuration.
properties:
Expand Down
42 changes: 42 additions & 0 deletions config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,48 @@ spec:
plugin.
type: string
type: object
execWasi:
description: ExecWasi defines the exec wasi input plugin configuration
properties:
accessiblePaths:
description: Specify the whitelist of paths to be able to access
paths from WASM programs.
items:
type: string
type: array
bufSize:
description: Size of the buffer (check unit sizes for allowed
values)
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
intervalNSec:
description: Polling interval (nanoseconds).
format: int64
type: integer
intervalSec:
description: Polling interval (seconds).
format: int32
type: integer
parser:
description: Specify the name of a parser to interpret the entry
as a structured message.
type: string
threaded:
description: 'Indicates whether to run this input in its own thread.
Default: false.'
type: boolean
wasiPath:
description: The place of a WASM program file.
type: string
wasmHeapSize:
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
wasmStackSize:
description: Size of the stack size of Wasm execution. Review
unit sizes for allowed values.
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
type: object
fluentBitMetrics:
description: FluentBitMetrics defines Fluent Bit Metrics Input configuration.
properties:
Expand Down
1 change: 1 addition & 0 deletions docs/fluentbit.md
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,7 @@ InputSpec defines the desired state of ClusterInput
| tcp | TCP defines the TCP input plugin configuration | *[input.TCP](plugins/input/tcp.md) |
| udp | UDP defines the UDP input plugin configuration | *[input.UDP](plugins/input/udp.md) |
| kubernetesEvents | KubernetesEvents defines the KubernetesEvents input plugin configuration | *[input.KubernetesEvents](plugins/input/kubernetesevents.md) |
| execWasi | ExecWasi defines the exec wasi input plugin configuration | *[input.ExecWasi](plugins/input/execwasi.md) |
| processors | Processors defines the processors configuration | *plugins.Config |

[Back to TOC](#table-of-contents)
Expand Down
7 changes: 3 additions & 4 deletions docs/plugins/fluentbit/filter/wasm.md
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
# WASM
# Wasm

The Wasm Filter allows you to modify the incoming records using Wasm technology. <br /> **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/wasm**
Wasm Filter allows you to modify the incoming records using Wasm technology. **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/wasm**


| Field | Description | Scheme |
| ----- | ----------- | ------ |
| wasmPath | Path to the built Wasm program that will be used. This can be a relative path against the main configuration file. | string |
| eventFormat | Define event format to interact with Wasm programs: msgpack or json. Default: json | string |
| functionName | Wasm function name that will be triggered to do filtering. It's assumed that the function is built inside the Wasm program specified above. | string |
| accessiblePaths | Specify the whitelist of paths to be able to access paths from WASM programs. | string |
| accessiblePaths | Specify the whitelist of paths to be able to access paths from WASM programs. | []string |
| wasmHeapSize | Size of the heap size of Wasm execution. Review unit sizes for allowed values. | string |
| wasmStackSize | Size of the stack size of Wasm execution. Review unit sizes for allowed values. | string |

16 changes: 16 additions & 0 deletions docs/plugins/fluentbit/input/exec_wasi.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# ExecWasi

The exec_wasi input plugin, allows to execute WASM program that is WASI target like as external program and collects event logs from there. **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/exec-wasi**


| Field | Description | Scheme |
| ----- | ----------- | ------ |
| wasiPath | The place of a WASM program file. | string |
| parser | Specify the name of a parser to interpret the entry as a structured message. | string |
| accessiblePaths | Specify the whitelist of paths to be able to access paths from WASM programs. | []string |
| intervalSec | Polling interval (seconds). | *int32 |
| intervalNSec | Polling interval (nanoseconds). | *int64 |
| wasmHeapSize | | string |
| wasmStackSize | Size of the stack size of Wasm execution. Review unit sizes for allowed values. | string |
| bufSize | Size of the buffer (check unit sizes for allowed values) | string |
| threaded | Indicates whether to run this input in its own thread. Default: false. | *bool |
42 changes: 42 additions & 0 deletions manifests/setup/fluent-operator-crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2208,6 +2208,48 @@ spec:
plugin.
type: string
type: object
execWasi:
description: ExecWasi defines the exec wasi input plugin configuration
properties:
accessiblePaths:
description: Specify the whitelist of paths to be able to access
paths from WASM programs.
items:
type: string
type: array
bufSize:
description: Size of the buffer (check unit sizes for allowed
values)
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
intervalNSec:
description: Polling interval (nanoseconds).
format: int64
type: integer
intervalSec:
description: Polling interval (seconds).
format: int32
type: integer
parser:
description: Specify the name of a parser to interpret the entry
as a structured message.
type: string
threaded:
description: 'Indicates whether to run this input in its own thread.
Default: false.'
type: boolean
wasiPath:
description: The place of a WASM program file.
type: string
wasmHeapSize:
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
wasmStackSize:
description: Size of the stack size of Wasm execution. Review
unit sizes for allowed values.
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
type: object
fluentBitMetrics:
description: FluentBitMetrics defines Fluent Bit Metrics Input configuration.
properties:
Expand Down
42 changes: 42 additions & 0 deletions manifests/setup/setup.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2208,6 +2208,48 @@ spec:
plugin.
type: string
type: object
execWasi:
description: ExecWasi defines the exec wasi input plugin configuration
properties:
accessiblePaths:
description: Specify the whitelist of paths to be able to access
paths from WASM programs.
items:
type: string
type: array
bufSize:
description: Size of the buffer (check unit sizes for allowed
values)
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
intervalNSec:
description: Polling interval (nanoseconds).
format: int64
type: integer
intervalSec:
description: Polling interval (seconds).
format: int32
type: integer
parser:
description: Specify the name of a parser to interpret the entry
as a structured message.
type: string
threaded:
description: 'Indicates whether to run this input in its own thread.
Default: false.'
type: boolean
wasiPath:
description: The place of a WASM program file.
type: string
wasmHeapSize:
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
wasmStackSize:
description: Size of the stack size of Wasm execution. Review
unit sizes for allowed values.
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
type: object
fluentBitMetrics:
description: FluentBitMetrics defines Fluent Bit Metrics Input configuration.
properties:
Expand Down

0 comments on commit eb81068

Please sign in to comment.