From 9f220bcfaaffc6078ede0fe15ab83c4ef02b612c Mon Sep 17 00:00:00 2001 From: SjLiu <139106424+sjliu1@users.noreply.github.com> Date: Wed, 30 Aug 2023 15:30:12 +0800 Subject: [PATCH] [summerospp]add fluentbit opentelemetry plugin (#890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [summerospp]add fluentbit opentelemetry plugin Signed-off-by: 刘帅军 * [summerospp]add fluentbit opentelemetry plugin Signed-off-by: 刘帅军 * [summerospp]add fluentbit opentelemetry plugin Signed-off-by: 刘帅军 * Delete .DS_Store Signed-off-by: SjLiu <139106424+sjliu1@users.noreply.github.com> --------- Signed-off-by: 刘帅军 Signed-off-by: 刘帅军 Signed-off-by: Elon Cheng Signed-off-by: SjLiu <139106424+sjliu1@users.noreply.github.com> Co-authored-by: 刘帅军 Co-authored-by: 刘帅军 Co-authored-by: Elon Cheng --- apis/fluentbit/v1alpha2/clusterinput_types.go | 2 + .../plugins/input/open_telemetry_types.go | 65 +++++++++++++++++++ .../plugins/input/zz_generated.deepcopy.go | 30 +++++++++ .../v1alpha2/zz_generated.deepcopy.go | 5 ++ .../fluentbit.fluent.io_clusterinputs.yaml | 36 ++++++++++ .../fluentbit.fluent.io_clusterinputs.yaml | 36 ++++++++++ docs/fluentbit.md | 2 + docs/fluentd.md | 9 ++- docs/plugins/fluentbit/input/forward.md | 16 +++++ .../plugins/fluentbit/input/open_telemetry.md | 14 ++++ docs/plugins/fluentbit/input/systemd.md | 2 + docs/plugins/fluentbit/input/tail.md | 2 + docs/plugins/fluentbit/output/open_search.md | 1 + .../fluentbit/output/prometheus_exporter.md | 10 +++ docs/plugins/fluentbit/output/s3.md | 52 +++++++-------- docs/plugins/fluentd/common/parse.md | 24 +++---- docs/plugins/fluentd/output/s3.md | 2 +- manifests/setup/fluent-operator-crd.yaml | 36 ++++++++++ manifests/setup/setup.yaml | 36 ++++++++++ 19 files changed, 338 insertions(+), 42 deletions(-) create mode 100644 apis/fluentbit/v1alpha2/plugins/input/open_telemetry_types.go create mode 100644 docs/plugins/fluentbit/input/forward.md create mode 100644 docs/plugins/fluentbit/input/open_telemetry.md create mode 100644 docs/plugins/fluentbit/output/prometheus_exporter.md diff --git a/apis/fluentbit/v1alpha2/clusterinput_types.go b/apis/fluentbit/v1alpha2/clusterinput_types.go index 57bf38b12..405eb12e7 100644 --- a/apis/fluentbit/v1alpha2/clusterinput_types.go +++ b/apis/fluentbit/v1alpha2/clusterinput_types.go @@ -55,6 +55,8 @@ type InputSpec struct { CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"` // Forward defines forward input plugin configuration Forward *input.Forward `json:"forward,omitempty"` + // OpenTelemetry defines forward input plugin configuration + OpenTelemetry *input.OpenTelemetry `json:"openTelemetry,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/fluentbit/v1alpha2/plugins/input/open_telemetry_types.go b/apis/fluentbit/v1alpha2/plugins/input/open_telemetry_types.go new file mode 100644 index 000000000..2c5cf16d3 --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/input/open_telemetry_types.go @@ -0,0 +1,65 @@ +package input + +import ( + "fmt" + + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" +) + +// +kubebuilder:object:generate:=true + +// The OpenTelemetry plugin allows you to ingest telemetry data as per the OTLP specification,
+// from various OpenTelemetry exporters, the OpenTelemetry Collector, or Fluent Bit's OpenTelemetry output plugin.
+// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/opentelemetry** +type OpenTelemetry struct { + // The address to listen on,default 0.0.0.0 + Listen string `json:"listen,omitempty"` + // The port for Fluent Bit to listen on.default 4318. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=65535 + Port *int32 `json:"port,omitempty"` + // Specify the key name to overwrite a tag. If set, the tag will be overwritten by a value of the key. + Tagkey string `json:"tagKey,omitempty"` + // Route trace data as a log message(default false). + RawTraces *bool `json:"rawTraces,omitempty"` + // Specify the maximum buffer size in KB to receive a JSON message(default 4M). + // +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$" + BufferMaxSize string `json:"bufferMaxSize,omitempty"` + // This sets the chunk size for incoming incoming JSON messages. These chunks are then stored/managed in the space available by buffer_max_size(default 512K). + // +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$" + BufferChunkSize string `json:"bufferChunkSize,omitempty"` + //It allows to set successful response code. 200, 201 and 204 are supported(default 201). + SuccessfulResponseCode *int32 `json:"successfulResponseCode,omitempty"` +} + +func (_ *OpenTelemetry) Name() string { + return "opentelemetry" +} + +// implement Section() method +func (ot *OpenTelemetry) Params(_ plugins.SecretLoader) (*params.KVs, error) { + kvs := params.NewKVs() + if ot.Listen != "" { + kvs.Insert("listen", ot.Listen) + } + if ot.Port != nil { + kvs.Insert("port", fmt.Sprint(*ot.Port)) + } + if ot.Tagkey != "" { + kvs.Insert("tag_key", ot.Tagkey) + } + if ot.RawTraces != nil { + kvs.Insert("raw_traces", fmt.Sprint(*ot.RawTraces)) + } + if ot.BufferMaxSize != "" { + kvs.Insert("buffer_max_size", ot.BufferMaxSize) + } + if ot.BufferChunkSize != "" { + kvs.Insert("buffer_chunk_size", ot.BufferChunkSize) + } + if ot.SuccessfulResponseCode != nil { + kvs.Insert("successful_response_code", fmt.Sprint(*ot.SuccessfulResponseCode)) + } + return kvs, nil +} diff --git a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go index 689a6ca83..6652b1d30 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go @@ -108,6 +108,36 @@ func (in *NodeExporterMetrics) DeepCopy() *NodeExporterMetrics { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenTelemetry) DeepCopyInto(out *OpenTelemetry) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } + if in.RawTraces != nil { + in, out := &in.RawTraces, &out.RawTraces + *out = new(bool) + **out = **in + } + if in.SuccessfulResponseCode != nil { + in, out := &in.SuccessfulResponseCode, &out.SuccessfulResponseCode + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetry. +func (in *OpenTelemetry) DeepCopy() *OpenTelemetry { + if in == nil { + return nil + } + out := new(OpenTelemetry) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PrometheusScrapeMetrics) DeepCopyInto(out *PrometheusScrapeMetrics) { *out = *in diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index 4614cc3df..f333b604d 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1078,6 +1078,11 @@ func (in *InputSpec) DeepCopyInto(out *InputSpec) { *out = new(input.Forward) (*in).DeepCopyInto(*out) } + if in.OpenTelemetry != nil { + in, out := &in.OpenTelemetry, &out.OpenTelemetry + *out = new(input.OpenTelemetry) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InputSpec. diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index 5767962e8..cc45a50b3 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -156,6 +156,42 @@ spec: plugin. type: string type: object + openTelemetry: + description: OpenTelemetry defines forward input plugin configuration + properties: + bufferChunkSize: + description: This sets the chunk size for incoming incoming JSON + messages. These chunks are then stored/managed in the space + available by buffer_max_size(default 512K). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + bufferMaxSize: + description: Specify the maximum buffer size in KB to receive + a JSON message(default 4M). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + listen: + description: The address to listen on,default 0.0.0.0 + type: string + port: + description: The port for Fluent Bit to listen on.default 4318. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + rawTraces: + description: Route trace data as a log message(default false). + type: boolean + successfulResponseCode: + description: It allows to set successful response code. 200, 201 + and 204 are supported(default 201). + format: int32 + type: integer + tagKey: + description: Specify the key name to overwrite a tag. If set, + the tag will be overwritten by a value of the key. + type: string + type: object prometheusScrapeMetrics: description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics Input configuration. diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml index 5767962e8..cc45a50b3 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -156,6 +156,42 @@ spec: plugin. type: string type: object + openTelemetry: + description: OpenTelemetry defines forward input plugin configuration + properties: + bufferChunkSize: + description: This sets the chunk size for incoming incoming JSON + messages. These chunks are then stored/managed in the space + available by buffer_max_size(default 512K). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + bufferMaxSize: + description: Specify the maximum buffer size in KB to receive + a JSON message(default 4M). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + listen: + description: The address to listen on,default 0.0.0.0 + type: string + port: + description: The port for Fluent Bit to listen on.default 4318. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + rawTraces: + description: Route trace data as a log message(default false). + type: boolean + successfulResponseCode: + description: It allows to set successful response code. 200, 201 + and 204 are supported(default 201). + format: int32 + type: integer + tagKey: + description: Specify the key name to overwrite a tag. If set, + the tag will be overwritten by a value of the key. + type: string + type: object prometheusScrapeMetrics: description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics Input configuration. diff --git a/docs/fluentbit.md b/docs/fluentbit.md index 0603583cf..24b340521 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -419,6 +419,8 @@ InputSpec defines the desired state of ClusterInput | fluentBitMetrics | FluentBitMetrics defines Fluent Bit Metrics Input configuration. | *[input.FluentbitMetrics](plugins/input/fluentbitmetrics.md) | | customPlugin | CustomPlugin defines Custom Input configuration. | *custom.CustomPlugin | | forward | Forward defines forward input plugin configuration | *[input.Forward](plugins/input/forward.md) | +| openTelemetry | OpenTelemetry defines forward input plugin configuration | *[input.OpenTelemetry](plugins/input/opentelemetry.md) | + [Back to TOC](#table-of-contents) # NamespacedFluentBitCfgSpec diff --git a/docs/fluentd.md b/docs/fluentd.md index 298825f92..bf6e8fd8f 100644 --- a/docs/fluentd.md +++ b/docs/fluentd.md @@ -287,9 +287,9 @@ FluentdSpec defines the desired state of Fluentd | ----- | ----------- | ------ | | globalInputs | Fluentd global inputs. | [][[input.Input](plugins/input/input.md)](plugins/[input/input](plugins/input/input/md).md) | | defaultFilterSelector | Select cluster filter plugins used to filter for the default cluster output | *[metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) | -| defaultOutputSelector | Select cluster output plugins used to send all logs that did not match a route to the matching outputs | *[metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) | +| defaultOutputSelector | Select cluster output plugins used to send all logs that did not match any route to the matching outputs | *[metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) | | disableService | By default will build the related service according to the globalinputs definition. | bool | -| replicas | Numbers of the Fluentd instance | *int32 | +| replicas | Numbers of the Fluentd instance Applicable when the mode is \"collector\", and will be ignored when the mode is \"agent\" | *int32 | | workers | Numbers of the workers in Fluentd instance | *int32 | | logLevel | Global logging verbosity | string | | image | Fluentd image. | string | @@ -310,10 +310,13 @@ FluentdSpec defines the desired state of Fluentd | rbacRules | RBACRules represents additional rbac rules which will be applied to the fluentd clusterrole. | []rbacv1.PolicyRule | | volumes | List of volumes that can be mounted by containers belonging to the pod. | []corev1.Volume | | volumeMounts | Pod volumes to mount into the container's filesystem. Cannot be updated. | []corev1.VolumeMount | -| volumeClaimTemplates | volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. | []corev1.PersistentVolumeClaim | +| volumeClaimTemplates | volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. Applicable when the mode is \"collector\", and will be ignored when the mode is \"agent\" | []corev1.PersistentVolumeClaim | | service | Service represents configurations on the fluentd service. | FluentDService | | securityContext | PodSecurityContext represents the security context for the fluentd pods. | *corev1.PodSecurityContext | | schedulerName | SchedulerName represents the desired scheduler for fluentd pods. | string | +| mode | Mode to determine whether to run Fluentd as collector or agent. | string | +| containerSecurityContext | ContainerSecurityContext represents the security context for the fluentd container. | *corev1.SecurityContext | +| positionDB | Storage for position db. You will use it if tail input is enabled. Applicable when the mode is \"agent\", and will be ignored when the mode is \"collector\" | [corev1.VolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#volume-v1-core) | [Back to TOC](#table-of-contents) # FluentdStatus diff --git a/docs/plugins/fluentbit/input/forward.md b/docs/plugins/fluentbit/input/forward.md new file mode 100644 index 000000000..03b70f5a3 --- /dev/null +++ b/docs/plugins/fluentbit/input/forward.md @@ -0,0 +1,16 @@ +# Forward + +Forward defines the in_forward Input plugin that listens to TCP socket to recieve the event stream. **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/forward** + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| port | Port for forward plugin instance. | *int32 | +| listen | Listener network interface. | string | +| tag | in_forward uses the tag value for incoming logs. If not set it uses tag from incoming log. | string | +| tagPrefix | Adds the prefix to incoming event's tag | string | +| unixPath | Specify the path to unix socket to recieve a forward message. If set, Listen and port are ignnored. | string | +| unixPerm | Set the permission of unix socket file. | string | +| bufferMaxSize | Specify maximum buffer memory size used to recieve a forward message. The value must be according to the Unit Size specification. | string | +| bufferchunkSize | Set the initial buffer size to store incoming data. This value is used too to increase buffer size as required. The value must be according to the Unit Size specification. | string | +| threaded | Threaded mechanism allows input plugin to run in a separate thread which helps to desaturate the main pipeline. | string | diff --git a/docs/plugins/fluentbit/input/open_telemetry.md b/docs/plugins/fluentbit/input/open_telemetry.md new file mode 100644 index 000000000..04c5b40ae --- /dev/null +++ b/docs/plugins/fluentbit/input/open_telemetry.md @@ -0,0 +1,14 @@ +# OpenTelemetry + +The OpenTelemetry plugin allows you to ingest telemetry data as per the OTLP specification,
from various OpenTelemetry exporters, the OpenTelemetry Collector, or Fluent Bit's OpenTelemetry output plugin.
**For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/opentelemetry** + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| listen | The address to listen on,default 0.0.0.0 | string | +| port | The port for Fluent Bit to listen on.default 4318. | *int32 | +| tagKey | Specify the key name to overwrite a tag. If set, the tag will be overwritten by a value of the key. | string | +| rawTraces | Route trace data as a log message(default false). | *bool | +| bufferMaxSize | Specify the maximum buffer size in KB to receive a JSON message(default 4M). | string | +| bufferChunkSize | This sets the chunk size for incoming incoming JSON messages. These chunks are then stored/managed in the space available by buffer_max_size(default 512K). | string | +| successfulResponseCode | It allows to set successful response code. 200, 201 and 204 are supported(default 201). | *int32 | diff --git a/docs/plugins/fluentbit/input/systemd.md b/docs/plugins/fluentbit/input/systemd.md index 3c6dd404d..df8ef4011 100644 --- a/docs/plugins/fluentbit/input/systemd.md +++ b/docs/plugins/fluentbit/input/systemd.md @@ -15,3 +15,5 @@ The Systemd input plugin allows to collect log messages from the Journald daemon | systemdFilterType | Define the filter type when Systemd_Filter is specified multiple times. Allowed values are And and Or. With And a record is matched only when all of the Systemd_Filter have a match. With Or a record is matched when any of the Systemd_Filter has a match. | string | | readFromTail | Start reading new entries. Skip entries already stored in Journald. | string | | stripUnderscores | Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. | string | +| storageType | Specify the buffering mechanism to use. It can be memory or filesystem | string | +| pauseOnChunksOverlimit | Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. | string | diff --git a/docs/plugins/fluentbit/input/tail.md b/docs/plugins/fluentbit/input/tail.md index 9894b1cc5..41f7cf0f7 100644 --- a/docs/plugins/fluentbit/input/tail.md +++ b/docs/plugins/fluentbit/input/tail.md @@ -31,3 +31,5 @@ The Tail input plugin allows to monitor one or several text files.
It has | dockerModeParser | Specify an optional parser for the first line of the docker multiline mode. The parser name to be specified must be registered in the parsers.conf file. | string | | disableInotifyWatcher | DisableInotifyWatcher will disable inotify and use the file stat watcher instead. | *bool | | multilineParser | This will help to reassembly multiline messages originally split by Docker or CRI Specify one or Multiline Parser definition to apply to the content. | string | +| storageType | Specify the buffering mechanism to use. It can be memory or filesystem | string | +| pauseOnChunksOverlimit | Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. | string | diff --git a/docs/plugins/fluentbit/output/open_search.md b/docs/plugins/fluentbit/output/open_search.md index c45b36a5a..3f6dc09ec 100644 --- a/docs/plugins/fluentbit/output/open_search.md +++ b/docs/plugins/fluentbit/output/open_search.md @@ -38,3 +38,4 @@ OpenSearch is the opensearch output plugin, allows to ingest your records into a | suppressTypeName | When enabled, mapping types is removed and Type option is ignored. Types are deprecated in APIs in v7.0. This options is for v7.0 or later. | *bool | | Workers | Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. | *int32 | | tls | | *[plugins.TLS](../tls.md) | +| totalLimitSize | Limit the maximum number of Chunks in the filesystem for the current output logical destination. | string | diff --git a/docs/plugins/fluentbit/output/prometheus_exporter.md b/docs/plugins/fluentbit/output/prometheus_exporter.md new file mode 100644 index 000000000..3be32afdd --- /dev/null +++ b/docs/plugins/fluentbit/output/prometheus_exporter.md @@ -0,0 +1,10 @@ +# PrometheusExporter + +PrometheusExporter An output plugin to expose Prometheus Metrics.
The prometheus exporter allows you to take metrics from Fluent Bit and expose them such that a Prometheus instance can scrape them.
**Important Note: The prometheus exporter only works with metric plugins, such as Node Exporter Metrics**
**For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/outputs/prometheus-exporter** + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| host | IP address or hostname of the target HTTP Server, default: 0.0.0.0 | string | +| port | This is the port Fluent Bit will bind to when hosting prometheus metrics. | *int32 | +| addLabels | This allows you to add custom labels to all metrics exposed through the prometheus exporter. You may have multiple of these fields | map[string]string | diff --git a/docs/plugins/fluentbit/output/s3.md b/docs/plugins/fluentbit/output/s3.md index c331eae14..127e9c16d 100644 --- a/docs/plugins/fluentbit/output/s3.md +++ b/docs/plugins/fluentbit/output/s3.md @@ -5,29 +5,29 @@ The S3 output plugin, allows to flush your records into a S3 time series databas | Field | Description | Scheme | | ----- | ----------- | ------ | -| region | The AWS region of your S3 bucket | string | -| bucket | S3 Bucket name | string | -| json_date_key | Specify the name of the time key in the output record. To disable the time key just set the value to false. | string | -| json_date_format | Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681) | string | -| total_file_size | Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. | string | -| upload_chunk_size | The size of each 'part' for multipart uploads. Max: 50M | string | -| upload_timeout | Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. | string | -| store_dir | Directory to locally buffer data before sending. | string | -| store_dir_limit_size | The size of the limitation for disk usage in S3. | string | -| s3_key_format | Format string for keys in S3. | string | -| s3_key_format_tag_delimiters | A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. | string | -| static_file_path | Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. | *bool | -| use_put_object | Use the S3 PutObject API, instead of the multipart upload API. | *bool | -| role_arn | ARN of an IAM role to assume | string | -| endpoint | Custom endpoint for the S3 API. | string | -| sts_endpoint | Custom endpoint for the STS API. | string | -| canned_acl | Predefined Canned ACL Policy for S3 objects. | string | -| compression | Compression type for S3 objects. | string | -| content_type | A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. | string | -| send_content_md5 | Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. | *bool | -| auto_retry_requests | Immediately retry failed requests to AWS services once. | *bool | -| log_key | By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. | string | -| preserve_data_ordering | Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. | *bool | -| storage_class | Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. | string | -| retry_limit | Integer value to set the maximum number of retries allowed. | *int32 | -| external_id | Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. | string | +| Region | The AWS region of your S3 bucket | string | +| Bucket | S3 Bucket name | string | +| JsonDateKey | Specify the name of the time key in the output record. To disable the time key just set the value to false. | string | +| JsonDateFormat | Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681) | string | +| TotalFileSize | Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. | string | +| UploadChunkSize | The size of each 'part' for multipart uploads. Max: 50M | string | +| UploadTimeout | Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. | string | +| StoreDir | Directory to locally buffer data before sending. | string | +| StoreDirLimitSize | The size of the limitation for disk usage in S3. | string | +| S3KeyFormat | Format string for keys in S3. | string | +| S3KeyFormatTagDelimiters | A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. | string | +| StaticFilePath | Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. | *bool | +| UsePutObject | Use the S3 PutObject API, instead of the multipart upload API. | *bool | +| RoleArn | ARN of an IAM role to assume | string | +| Endpoint | Custom endpoint for the S3 API. | string | +| StsEndpoint | Custom endpoint for the STS API. | string | +| CannedAcl | Predefined Canned ACL Policy for S3 objects. | string | +| Compression | Compression type for S3 objects. | string | +| ContentType | A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. | string | +| SendContentMd5 | Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. | *bool | +| AutoRetryRequests | Immediately retry failed requests to AWS services once. | *bool | +| LogKey | By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. | string | +| PreserveDataOrdering | Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. | *bool | +| StorageClass | Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. | string | +| RetryLimit | Integer value to set the maximum number of retries allowed. | *int32 | +| ExternalId | Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. | string | diff --git a/docs/plugins/fluentd/common/parse.md b/docs/plugins/fluentd/common/parse.md index b6ccf1bbe..8193006dc 100644 --- a/docs/plugins/fluentd/common/parse.md +++ b/docs/plugins/fluentd/common/parse.md @@ -24,19 +24,19 @@ Parse defines various parameters for the parse plugin | grokPattern | The pattern of grok. | *string | | customPatternPath | Path to the file that includes custom grok patterns. | *string | | grokFailureKey | The key has grok failure reason. | *string | -| multiLineStartRegexp | The regexp to match beginning of multiline. This is only for "multiline_grok". | *string | +| multiLineStartRegexp | The regexp to match beginning of multiline. This is only for \"multiline_grok\". | *string | | grokPatternSeries | Specify grok pattern series set. | *string | -| grok | Grok Sections. | []Grok | - +| grok | Grok Sections | []Grok | # Grok -grok defines multiple grok expressions. -| Field | Description | Scheme | -| ----------- | ------------------------------------------------------------ | ------- | -| name | The name of this grok section. | *string | -| pattern | The pattern of grok. Required parameter. | *string | -| keepTimeKey | If true, keep time field in the record. | *bool | -| timeKey | Specify time field for event time. If the event doesn't have this field, current time is used. | *string | -| timeFormat | Process value using specified format. This is available only when time_type is string | *string | -| timeZone | Use specified timezone. one can parse/format the time value in the specified timezone. | *string | \ No newline at end of file + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| name | The name of this grok section. | *string | +| pattern | The pattern of grok. Required parameter. | *string | +| keepTimeKey | If true, keep time field in the record. | *bool | +| timeKey | Specify time field for event time. If the event doesn't have this field, current time is used. | *string | +| timeFormat | Process value using specified format. This is available only when time_type is string | *string | +| timeZone | Use specified timezone. one can parse/format the time value in the specified timezone. | *string | diff --git a/docs/plugins/fluentd/output/s3.md b/docs/plugins/fluentd/output/s3.md index e0d1ac40d..1c52b111a 100644 --- a/docs/plugins/fluentd/output/s3.md +++ b/docs/plugins/fluentd/output/s3.md @@ -9,7 +9,7 @@ S3 defines the parameters for out_s3 output plugin | awsSecKey | The AWS secret key. | *string | | s3Bucket | The Amazon S3 bucket name. | *string | | s3Region | The Amazon S3 region name | *string | -| s3Endpoint | The endpoint URL (like "http://localhost:9000/") | *string | +| s3Endpoint | The endpoint URL (like \"http://localhost:9000/\") | *string | | forcePathStyle | This prevents AWS SDK from breaking endpoint URL | *bool | | timeSliceFormat | This timestamp is added to each file name | *string | | path | The path prefix of the files on S3. | *string | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 602855412..e305ecf88 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -1905,6 +1905,42 @@ spec: plugin. type: string type: object + openTelemetry: + description: OpenTelemetry defines forward input plugin configuration + properties: + bufferChunkSize: + description: This sets the chunk size for incoming incoming JSON + messages. These chunks are then stored/managed in the space + available by buffer_max_size(default 512K). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + bufferMaxSize: + description: Specify the maximum buffer size in KB to receive + a JSON message(default 4M). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + listen: + description: The address to listen on,default 0.0.0.0 + type: string + port: + description: The port for Fluent Bit to listen on.default 4318. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + rawTraces: + description: Route trace data as a log message(default false). + type: boolean + successfulResponseCode: + description: It allows to set successful response code. 200, 201 + and 204 are supported(default 201). + format: int32 + type: integer + tagKey: + description: Specify the key name to overwrite a tag. If set, + the tag will be overwritten by a value of the key. + type: string + type: object prometheusScrapeMetrics: description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics Input configuration. diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 5ae048dae..c954fc22f 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -1905,6 +1905,42 @@ spec: plugin. type: string type: object + openTelemetry: + description: OpenTelemetry defines forward input plugin configuration + properties: + bufferChunkSize: + description: This sets the chunk size for incoming incoming JSON + messages. These chunks are then stored/managed in the space + available by buffer_max_size(default 512K). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + bufferMaxSize: + description: Specify the maximum buffer size in KB to receive + a JSON message(default 4M). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + listen: + description: The address to listen on,default 0.0.0.0 + type: string + port: + description: The port for Fluent Bit to listen on.default 4318. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + rawTraces: + description: Route trace data as a log message(default false). + type: boolean + successfulResponseCode: + description: It allows to set successful response code. 200, 201 + and 204 are supported(default 201). + format: int32 + type: integer + tagKey: + description: Specify the key name to overwrite a tag. If set, + the tag will be overwritten by a value of the key. + type: string + type: object prometheusScrapeMetrics: description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics Input configuration.