From 52ca3b464d1b3ddba6bdacaca9d02989d54c454a Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Thu, 31 Aug 2023 21:17:58 +0200 Subject: [PATCH 1/2] add compress in fluentbit output es Signed-off-by: Husni Alhamdani --- .../v1alpha2/plugins/output/elasticsearch_types.go | 6 ++++++ .../crds/fluentbit.fluent.io_clusteroutputs.yaml | 6 ++++++ .../crds/fluentbit.fluent.io_outputs.yaml | 6 ++++++ .../bases/fluentbit.fluent.io_clusteroutputs.yaml | 6 ++++++ config/crd/bases/fluentbit.fluent.io_outputs.yaml | 6 ++++++ manifests/setup/fluent-operator-crd.yaml | 12 ++++++++++++ manifests/setup/setup.yaml | 12 ++++++++++++ 7 files changed, 54 insertions(+) diff --git a/apis/fluentbit/v1alpha2/plugins/output/elasticsearch_types.go b/apis/fluentbit/v1alpha2/plugins/output/elasticsearch_types.go index 8e52c3624..091976479 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/elasticsearch_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/elasticsearch_types.go @@ -23,6 +23,9 @@ type Elasticsearch struct { // This option defines such path on the fluent-bit side. // It simply adds a path prefix in the indexing HTTP POST URI. Path string `json:"path,omitempty"` + // Set payload compression mechanism. Option available is 'gzip' + // +kubebuilder:validation:Enum=gzip + Compress string `json:"compress,omitempty"` // Specify the buffer size used to read the response from the Elasticsearch HTTP service. // This option is useful for debugging purposes where is required to read full responses, // note that response size grows depending of the number of records inserted. @@ -114,6 +117,9 @@ func (es *Elasticsearch) Params(sl plugins.SecretLoader) (*params.KVs, error) { if es.Path != "" { kvs.Insert("Path", es.Path) } + if es.Compress != "" { + kvs.Insert("Compress", es.Compress) + } if es.BufferSize != "" { kvs.Insert("Buffer_Size", es.BufferSize) } diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index 4f008f850..a05a53050 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -435,6 +435,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index 1f788e513..383ce7085 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -435,6 +435,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index 4f008f850..a05a53050 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -435,6 +435,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index 1f788e513..383ce7085 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -435,6 +435,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index e305ecf88..300d42f11 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -2656,6 +2656,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record @@ -25372,6 +25378,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index c954fc22f..5a229ea7a 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -2656,6 +2656,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record @@ -25372,6 +25378,12 @@ spec: description: If you are using Elastic's Elasticsearch Service you can specify the cloud_id of the cluster running. type: string + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + enum: + - gzip + type: string currentTimeIndex: description: Use current time for index generation instead of message record From 91fb05257029ee8909b9aedf4af29e4483ca7226 Mon Sep 17 00:00:00 2001 From: Husni Alhamdani Date: Thu, 31 Aug 2023 21:24:27 +0200 Subject: [PATCH 2/2] docs related to es output changes Signed-off-by: Husni Alhamdani --- docs/fluentbit.md | 1 - docs/plugins/fluentbit/output/elasticsearch.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fluentbit.md b/docs/fluentbit.md index 24b340521..70dd2e563 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -421,7 +421,6 @@ InputSpec defines the desired state of ClusterInput | forward | Forward defines forward input plugin configuration | *[input.Forward](plugins/input/forward.md) | | openTelemetry | OpenTelemetry defines forward input plugin configuration | *[input.OpenTelemetry](plugins/input/opentelemetry.md) | - [Back to TOC](#table-of-contents) # NamespacedFluentBitCfgSpec diff --git a/docs/plugins/fluentbit/output/elasticsearch.md b/docs/plugins/fluentbit/output/elasticsearch.md index f53c5f315..c5d6e38a9 100644 --- a/docs/plugins/fluentbit/output/elasticsearch.md +++ b/docs/plugins/fluentbit/output/elasticsearch.md @@ -8,6 +8,7 @@ Elasticsearch is the es output plugin, allows to ingest your records into an Ela | host | IP address or hostname of the target Elasticsearch instance | string | | port | TCP port of the target Elasticsearch instance | *int32 | | path | Elasticsearch accepts new data on HTTP query path \"/_bulk\". But it is also possible to serve Elasticsearch behind a reverse proxy on a subpath. This option defines such path on the fluent-bit side. It simply adds a path prefix in the indexing HTTP POST URI. | string | +| compress | Set payload compression mechanism. Option available is 'gzip' | string | | bufferSize | Specify the buffer size used to read the response from the Elasticsearch HTTP service. This option is useful for debugging purposes where is required to read full responses, note that response size grows depending of the number of records inserted. To set an unlimited amount of memory set this value to False, otherwise the value must be according to the Unit Size specification. | string | | pipeline | Newer versions of Elasticsearch allows setting up filters called pipelines. This option allows defining which pipeline the database should use. For performance reasons is strongly suggested parsing and filtering on Fluent Bit side, avoid pipelines. | string | | awsAuth | Enable AWS Sigv4 Authentication for Amazon ElasticSearch Service. | string |