From 7564fa6f55823e33e9d36ebdcc673a9ae54036f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gre=CC=81gory=20Cuellar?= Date: Fri, 30 Jun 2023 11:59:50 +0200 Subject: [PATCH] add S3 output plugin for Fluent Bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Grégory Cuellar --- .../fluentbit/v1alpha2/clusteroutput_types.go | 2 + .../v1alpha2/plugins/output/s3_types.go | 157 ++++++++++++ .../v1alpha2/plugins/output/s3_types_test.go | 77 ++++++ .../plugins/output/zz_generated.deepcopy.go | 45 ++++ .../v1alpha2/zz_generated.deepcopy.go | 5 + .../fluentbit.fluent.io_clusteroutputs.yaml | 114 +++++++++ .../crds/fluentbit.fluent.io_outputs.yaml | 114 +++++++++ .../fluentbit.fluent.io_clusteroutputs.yaml | 114 +++++++++ .../bases/fluentbit.fluent.io_outputs.yaml | 114 +++++++++ docs/fluentbit.md | 1 + docs/plugins/fluentbit/output/s3.md | 33 +++ manifests/setup/fluent-operator-crd.yaml | 228 ++++++++++++++++++ manifests/setup/setup.yaml | 228 ++++++++++++++++++ 13 files changed, 1232 insertions(+) create mode 100644 apis/fluentbit/v1alpha2/plugins/output/s3_types.go create mode 100644 apis/fluentbit/v1alpha2/plugins/output/s3_types_test.go create mode 100644 docs/plugins/fluentbit/output/s3.md diff --git a/apis/fluentbit/v1alpha2/clusteroutput_types.go b/apis/fluentbit/v1alpha2/clusteroutput_types.go index efb97292b..b1e78f448 100644 --- a/apis/fluentbit/v1alpha2/clusteroutput_types.go +++ b/apis/fluentbit/v1alpha2/clusteroutput_types.go @@ -94,6 +94,8 @@ type OutputSpec struct { OpenTelemetry *output.OpenTelemetry `json:"opentelemetry,omitempty"` // PrometheusRemoteWrite_types defines Prometheus Remote Write configuration. PrometheusRemoteWrite *output.PrometheusRemoteWrite `json:"prometheusRemoteWrite,omitempty"` + // S3 defines S3 Output configuration. + S3 *output.S3 `json:"s3,omitempty"` // CustomPlugin defines Custom Output configuration. CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"` diff --git a/apis/fluentbit/v1alpha2/plugins/output/s3_types.go b/apis/fluentbit/v1alpha2/plugins/output/s3_types.go new file mode 100644 index 000000000..063499b30 --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/output/s3_types.go @@ -0,0 +1,157 @@ +package output + +import ( + "fmt" + + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" +) + +// +kubebuilder:object:generate:=true + +// The S3 output plugin, allows to flush your records into a S3 time series database.
+// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/outputs/s3** +type S3 struct { + // The AWS region of your S3 bucket + Region string `json:"Region"` + // S3 Bucket name + Bucket string `json:"Bucket"` + // Specify the name of the time key in the output record. To disable the time key just set the value to false. + JsonDateKey string `json:"JsonDateKey,omitempty"` + // Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681) + JsonDateFormat string `json:"JsonDateFormat,omitempty"` + // Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. + TotalFileSize string `json:"TotalFileSize,omitempty"` + // The size of each 'part' for multipart uploads. Max: 50M + UploadChunkSize string `json:"UploadChunkSize,omitempty"` + // Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. + UploadTimeout string `json:"UploadTimeout,omitempty"` + // Directory to locally buffer data before sending. + StoreDir string `json:"StoreDir,omitempty"` + // The size of the limitation for disk usage in S3. + StoreDirLimitSize string `json:"StoreDirLimitSize,omitempty"` + // Format string for keys in S3. + S3KeyFormat string `json:"S3KeyFormat,omitempty"` + // A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. + S3KeyFormatTagDelimiters string `json:"S3KeyFormatTagDelimiters,omitempty"` + // Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. + StaticFilePath *bool `json:"StaticFilePath,omitempty"` + // Use the S3 PutObject API, instead of the multipart upload API. + UsePutObject *bool `json:"UsePutObject,omitempty"` + // ARN of an IAM role to assume + RoleArn string `json:"RoleArn,omitempty"` + // Custom endpoint for the S3 API. + Endpoint string `json:"Endpoint,omitempty"` + // Custom endpoint for the STS API. + StsEndpoint string `json:"StsEndpoint,omitempty"` + // Predefined Canned ACL Policy for S3 objects. + CannedAcl string `json:"CannedAcl,omitempty"` + // Compression type for S3 objects. + Compression string `json:"Compression,omitempty"` + // A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. + ContentType string `json:"ContentType,omitempty"` + // Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. + SendContentMd5 *bool `json:"SendContentMd5,omitempty"` + // Immediately retry failed requests to AWS services once. + AutoRetryRequests *bool `json:"AutoRetryRequests,omitempty"` + // By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. + LogKey string `json:"LogKey,omitempty"` + // Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. + PreserveDataOrdering *bool `json:"PreserveDataOrdering,omitempty"` + // Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. + StorageClass string `json:"StorageClass,omitempty"` + // Integer value to set the maximum number of retries allowed. + RetryLimit *int32 `json:"RetryLimit,omitempty"` + // Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. + ExternalId string `json:"ExternalId,omitempty"` +} + +// Name implement Section() method +func (_ *S3) Name() string { + return "s3" +} + +func (o *S3) Params(sl plugins.SecretLoader) (*params.KVs, error) { + kvs := params.NewKVs() + // S3 Validation + + if o.Region != "" { + kvs.Insert("region", o.Region) + } + if o.Bucket != "" { + kvs.Insert("bucket", o.Bucket) + } + if o.JsonDateKey != "" { + kvs.Insert("json_date_key", o.JsonDateKey) + } + if o.JsonDateFormat != "" { + kvs.Insert("json_date_format", o.JsonDateFormat) + } + if o.TotalFileSize != "" { + kvs.Insert("total_file_size", o.TotalFileSize) + } + if o.UploadChunkSize != "" { + kvs.Insert("upload_chunk_size", o.UploadChunkSize) + } + if o.UploadTimeout != "" { + kvs.Insert("upload_timeout", o.UploadTimeout) + } + if o.StoreDir != "" { + kvs.Insert("store_dir", o.StoreDir) + } + if o.StoreDirLimitSize != "" { + kvs.Insert("store_dir_limit_size", o.StoreDirLimitSize) + } + if o.S3KeyFormat != "" { + kvs.Insert("s3_key_format", o.S3KeyFormat) + } + if o.S3KeyFormatTagDelimiters != "" { + kvs.Insert("s3_key_format_tag_delimiters", o.S3KeyFormatTagDelimiters) + } + if o.StaticFilePath != nil { + kvs.Insert("static_file_path", fmt.Sprint(*o.StaticFilePath)) + } + if o.UsePutObject != nil { + kvs.Insert("use_put_object", fmt.Sprint(*o.UsePutObject)) + } + if o.RoleArn != "" { + kvs.Insert("role_arn", o.RoleArn) + } + if o.Endpoint != "" { + kvs.Insert("endpoint", o.Endpoint) + } + if o.StsEndpoint != "" { + kvs.Insert("sts_endpoint", o.StsEndpoint) + } + if o.CannedAcl != "" { + kvs.Insert("canned_acl", o.CannedAcl) + } + if o.Compression != "" { + kvs.Insert("compression", o.Compression) + } + if o.ContentType != "" { + kvs.Insert("content_type", o.ContentType) + } + if o.SendContentMd5 != nil { + kvs.Insert("send_content_md5", fmt.Sprint(*o.SendContentMd5)) + } + if o.AutoRetryRequests != nil { + kvs.Insert("auto_retry_requests", fmt.Sprint(*o.AutoRetryRequests)) + } + if o.LogKey != "" { + kvs.Insert("log_key", o.LogKey) + } + if o.PreserveDataOrdering != nil { + kvs.Insert("preserve_data_ordering", fmt.Sprint(*o.PreserveDataOrdering)) + } + if o.StorageClass != "" { + kvs.Insert("storage_class", o.StorageClass) + } + if o.RetryLimit != nil { + kvs.Insert("retry_limit", fmt.Sprint(*o.RetryLimit)) + } + if o.ExternalId != "" { + kvs.Insert("external_id", o.ExternalId) + } + return kvs, nil +} diff --git a/apis/fluentbit/v1alpha2/plugins/output/s3_types_test.go b/apis/fluentbit/v1alpha2/plugins/output/s3_types_test.go new file mode 100644 index 000000000..39b622d7d --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/output/s3_types_test.go @@ -0,0 +1,77 @@ +package output + +import ( + "testing" + + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" + . "github.com/onsi/gomega" +) + +func TestOutput_S3_Params(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + + s3 := S3{ + Region: "us-east-1", + Bucket: "fluentbit", + JsonDateKey: "2018-05-30T09:39:52.000681Z", + JsonDateFormat: "iso8601", + TotalFileSize: "100M", + UploadChunkSize: "50M", + UploadTimeout: "10m", + StoreDir: "/tmp/fluent-bit/s3", + StoreDirLimitSize: "0", + S3KeyFormat: "/fluent-bit-logs/$TAG/%Y/%m/%d/%H/%M/%S", + S3KeyFormatTagDelimiters: ".", + StaticFilePath: ptrAny(false), + UsePutObject: ptrAny(false), + RoleArn: "role", + Endpoint: "endpoint", + StsEndpoint: "sts_endpoint", + CannedAcl: "canned_acl", + Compression: "gzip", + ContentType: "text/plain", + SendContentMd5: ptrAny(false), + AutoRetryRequests: ptrAny(true), + LogKey: "log_key", + PreserveDataOrdering: ptrAny(true), + StorageClass: "storage_class", + RetryLimit: ptrAny(int32(1)), + ExternalId: "external_id", + } + + expected := params.NewKVs() + expected.Insert("region", "us-east-1") + expected.Insert("bucket", "fluentbit") + expected.Insert("json_date_key", "2018-05-30T09:39:52.000681Z") + expected.Insert("json_date_format", "iso8601") + expected.Insert("total_file_size", "100M") + expected.Insert("upload_chunk_size", "50M") + expected.Insert("upload_timeout", "10m") + expected.Insert("store_dir", "/tmp/fluent-bit/s3") + expected.Insert("store_dir_limit_size", "0") + expected.Insert("s3_key_format", "/fluent-bit-logs/$TAG/%Y/%m/%d/%H/%M/%S") + expected.Insert("s3_key_format_tag_delimiters", ".") + expected.Insert("static_file_path", "false") + expected.Insert("use_put_object", "false") + expected.Insert("role_arn", "role") + expected.Insert("endpoint", "endpoint") + expected.Insert("sts_endpoint", "sts_endpoint") + expected.Insert("canned_acl", "canned_acl") + expected.Insert("compression", "gzip") + expected.Insert("content_type", "text/plain") + expected.Insert("send_content_md5", "false") + expected.Insert("auto_retry_requests", "true") + expected.Insert("log_key", "log_key") + expected.Insert("preserve_data_ordering", "true") + expected.Insert("storage_class", "storage_class") + expected.Insert("retry_limit", "1") + expected.Insert("external_id", "external_id") + + kvs, err := s3.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) + +} diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go index 784e4bca0..574f4bf02 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go @@ -767,6 +767,51 @@ func (in *PrometheusRemoteWrite) DeepCopy() *PrometheusRemoteWrite { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3) DeepCopyInto(out *S3) { + *out = *in + if in.StaticFilePath != nil { + in, out := &in.StaticFilePath, &out.StaticFilePath + *out = new(bool) + **out = **in + } + if in.UsePutObject != nil { + in, out := &in.UsePutObject, &out.UsePutObject + *out = new(bool) + **out = **in + } + if in.SendContentMd5 != nil { + in, out := &in.SendContentMd5, &out.SendContentMd5 + *out = new(bool) + **out = **in + } + if in.AutoRetryRequests != nil { + in, out := &in.AutoRetryRequests, &out.AutoRetryRequests + *out = new(bool) + **out = **in + } + if in.PreserveDataOrdering != nil { + in, out := &in.PreserveDataOrdering, &out.PreserveDataOrdering + *out = new(bool) + **out = **in + } + if in.RetryLimit != nil { + in, out := &in.RetryLimit, &out.RetryLimit + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3. +func (in *S3) DeepCopy() *S3 { + if in == nil { + return nil + } + out := new(S3) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Splunk) DeepCopyInto(out *Splunk) { *out = *in diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index 6754d69c9..4ba987718 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1333,6 +1333,11 @@ func (in *OutputSpec) DeepCopyInto(out *OutputSpec) { *out = new(output.PrometheusRemoteWrite) (*in).DeepCopyInto(*out) } + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(output.S3) + (*in).DeepCopyInto(*out) + } if in.CustomPlugin != nil { in, out := &in.CustomPlugin, &out.CustomPlugin *out = new(custom.CustomPlugin) diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index addb6cd28..ee10b9518 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -2211,6 +2211,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index f431dc7f4..9b52e800c 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -2211,6 +2211,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index addb6cd28..ee10b9518 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -2211,6 +2211,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index f431dc7f4..9b52e800c 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -2211,6 +2211,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: diff --git a/docs/fluentbit.md b/docs/fluentbit.md index 7f75ee0e3..77addfd31 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -487,6 +487,7 @@ OutputSpec defines the desired state of ClusterOutput | opensearch | OpenSearch defines OpenSearch Output configuration. | *[output.OpenSearch](plugins/output/opensearch.md) | | opentelemetry | OpenTelemetry defines OpenTelemetry Output configuration. | *[output.OpenTelemetry](plugins/output/opentelemetry.md) | | prometheusRemoteWrite | PrometheusRemoteWrite_types defines Prometheus Remote Write configuration. | *[output.PrometheusRemoteWrite](plugins/output/prometheusremotewrite.md) | +| s3 | S3 defines S3 Output configuration. | *[output.S3](plugins/output/s3.md) | | customPlugin | CustomPlugin defines Custom Output configuration. | *custom.CustomPlugin | [Back to TOC](#table-of-contents) diff --git a/docs/plugins/fluentbit/output/s3.md b/docs/plugins/fluentbit/output/s3.md new file mode 100644 index 000000000..c331eae14 --- /dev/null +++ b/docs/plugins/fluentbit/output/s3.md @@ -0,0 +1,33 @@ +# S3 + +The S3 output plugin, allows to flush your records into a S3 time series database.
**For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/outputs/s3** + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| region | The AWS region of your S3 bucket | string | +| bucket | S3 Bucket name | string | +| json_date_key | Specify the name of the time key in the output record. To disable the time key just set the value to false. | string | +| json_date_format | Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681) | string | +| total_file_size | Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. | string | +| upload_chunk_size | The size of each 'part' for multipart uploads. Max: 50M | string | +| upload_timeout | Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. | string | +| store_dir | Directory to locally buffer data before sending. | string | +| store_dir_limit_size | The size of the limitation for disk usage in S3. | string | +| s3_key_format | Format string for keys in S3. | string | +| s3_key_format_tag_delimiters | A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. | string | +| static_file_path | Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. | *bool | +| use_put_object | Use the S3 PutObject API, instead of the multipart upload API. | *bool | +| role_arn | ARN of an IAM role to assume | string | +| endpoint | Custom endpoint for the S3 API. | string | +| sts_endpoint | Custom endpoint for the STS API. | string | +| canned_acl | Predefined Canned ACL Policy for S3 objects. | string | +| compression | Compression type for S3 objects. | string | +| content_type | A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. | string | +| send_content_md5 | Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. | *bool | +| auto_retry_requests | Immediately retry failed requests to AWS services once. | *bool | +| log_key | By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. | string | +| preserve_data_ordering | Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. | *bool | +| storage_class | Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. | string | +| retry_limit | Integer value to set the maximum number of retries allowed. | *int32 | +| external_id | Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. | string | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index e81dec557..b681a0519 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -4224,6 +4224,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: @@ -24739,6 +24853,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index b9a5ae3b5..a51e4e9a8 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -4224,6 +4224,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: @@ -24739,6 +24853,120 @@ spec: allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. type: string + s3: + description: S3 defines S3 Output configuration. + properties: + auto_retry_requests: + description: Immediately retry failed requests to AWS services + once. + type: boolean + bucket: + description: S3 Bucket name + type: string + canned_acl: + description: Predefined Canned ACL Policy for S3 objects. + type: string + compression: + description: Compression type for S3 objects. + type: string + content_type: + description: A standard MIME type for the S3 object; this will + be set as the Content-Type HTTP header. + type: string + endpoint: + description: Custom endpoint for the S3 API. + type: string + external_id: + description: Specify an external ID for the STS API, can be used + with the role_arn parameter if your role requires an external + ID. + type: string + json_date_format: + description: 'Specify the format of the date. Supported formats + are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) + and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' + type: string + json_date_key: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + log_key: + description: By default, the whole log record will be sent to + S3. If you specify a key name with this option, then only the + value of that key will be sent to S3. + type: string + preserve_data_ordering: + description: Normally, when an upload request fails, there is + a high chance for the last received chunk to be swapped with + a later chunk, resulting in data shuffling. This feature prevents + this shuffling by using a queue logic for uploads. + type: boolean + region: + description: The AWS region of your S3 bucket + type: string + retry_limit: + description: Integer value to set the maximum number of retries + allowed. + format: int32 + type: integer + role_arn: + description: ARN of an IAM role to assume + type: string + s3_key_format: + description: Format string for keys in S3. + type: string + s3_key_format_tag_delimiters: + description: A series of characters which will be used to split + the tag into 'parts' for use with the s3_key_format option. + type: string + send_content_md5: + description: Send the Content-MD5 header with PutObject and UploadPart + requests, as is required when Object Lock is enabled. + type: boolean + static_file_path: + description: Disables behavior where UUID string is automatically + appended to end of S3 key name when $UUID is not provided in + s3_key_format. $UUID, time formatters, $TAG, and other dynamic + key formatters all work as expected while this feature is set + to true. + type: boolean + storage_class: + description: Specify the storage class for S3 objects. If this + option is not specified, objects will be stored with the default + 'STANDARD' storage class. + type: string + store_dir: + description: Directory to locally buffer data before sending. + type: string + store_dir_limit_size: + description: The size of the limitation for disk usage in S3. + type: string + sts_endpoint: + description: Custom endpoint for the STS API. + type: string + total_file_size: + description: Specifies the size of files in S3. Minimum size is + 1M. With use_put_object On the maximum size is 1G. With multipart + upload mode, the maximum size is 50G. + type: string + upload_chunk_size: + description: 'The size of each ''part'' for multipart uploads. + Max: 50M' + type: string + upload_timeout: + description: Whenever this amount of time has elapsed, Fluent + Bit will complete an upload and create a new file in S3. For + example, set this value to 60m and you will get a new file every + hour. + type: string + use_put_object: + description: Use the S3 PutObject API, instead of the multipart + upload API. + type: boolean + required: + - bucket + - region + type: object splunk: description: Splunk defines Splunk Output Configuration properties: