Merge pull request #899 from husnialhamdani/add-compress-fluentbit-ou…

…tput-es

add compress in fluentbit output es

apis/fluentbit/v1alpha2/plugins/output/elasticsearch_types.go

@@ -23,6 +23,9 @@ type Elasticsearch struct {
 	// This option defines such path on the fluent-bit side.
 	// It simply adds a path prefix in the indexing HTTP POST URI.
 	Path string `json:"path,omitempty"`
+	// Set payload compression mechanism. Option available is 'gzip'
+	// +kubebuilder:validation:Enum=gzip
+	Compress string `json:"compress,omitempty"`
 	// Specify the buffer size used to read the response from the Elasticsearch HTTP service.
 	// This option is useful for debugging purposes where is required to read full responses,
 	// note that response size grows depending of the number of records inserted.
@@ -114,6 +117,9 @@ func (es *Elasticsearch) Params(sl plugins.SecretLoader) (*params.KVs, error) {
 	if es.Path != "" {
 		kvs.Insert("Path", es.Path)
 	}
+	if es.Compress != "" {
+		kvs.Insert("Compress", es.Compress)
+	}
 	if es.BufferSize != "" {
 		kvs.Insert("Buffer_Size", es.BufferSize)
 	}

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml

@@ -435,6 +435,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml

@@ -435,6 +435,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record

config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml

@@ -435,6 +435,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record

config/crd/bases/fluentbit.fluent.io_outputs.yaml

@@ -435,6 +435,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record

docs/fluentbit.md

@@ -421,7 +421,6 @@ InputSpec defines the desired state of ClusterInput
 | forward | Forward defines forward  input plugin configuration | *[input.Forward](plugins/input/forward.md) |
 | openTelemetry | OpenTelemetry defines forward  input plugin configuration | *[input.OpenTelemetry](plugins/input/opentelemetry.md) |
 
-
 [Back to TOC](#table-of-contents)
 # NamespacedFluentBitCfgSpec
 

docs/plugins/fluentbit/output/elasticsearch.md

@@ -8,6 +8,7 @@ Elasticsearch is the es output plugin, allows to ingest your records into an Ela
 | host | IP address or hostname of the target Elasticsearch instance | string |
 | port | TCP port of the target Elasticsearch instance | *int32 |
 | path | Elasticsearch accepts new data on HTTP query path \"/_bulk\". But it is also possible to serve Elasticsearch behind a reverse proxy on a subpath. This option defines such path on the fluent-bit side. It simply adds a path prefix in the indexing HTTP POST URI. | string |
+| compress | Set payload compression mechanism. Option available is 'gzip' | string |
 | bufferSize | Specify the buffer size used to read the response from the Elasticsearch HTTP service. This option is useful for debugging purposes where is required to read full responses, note that response size grows depending of the number of records inserted. To set an unlimited amount of memory set this value to False, otherwise the value must be according to the Unit Size specification. | string |
 | pipeline | Newer versions of Elasticsearch allows setting up filters called pipelines. This option allows defining which pipeline the database should use. For performance reasons is strongly suggested parsing and filtering on Fluent Bit side, avoid pipelines. | string |
 | awsAuth | Enable AWS Sigv4 Authentication for Amazon ElasticSearch Service. | string |

manifests/setup/fluent-operator-crd.yaml

@@ -2656,6 +2656,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record
@@ -25372,6 +25378,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record

manifests/setup/setup.yaml

@@ -2656,6 +2656,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record
@@ -25372,6 +25378,12 @@ spec:
                     description: If you are using Elastic's Elasticsearch Service
                       you can specify the cloud_id of the cluster running.
                     type: string
+                  compress:
+                    description: Set payload compression mechanism. Option available
+                      is 'gzip'
+                    enum:
+                    - gzip
+                    type: string
                   currentTimeIndex:
                     description: Use current time for index generation instead of
                       message record