Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[summerospp]add fluentbit opentelemetry plugin #890

Merged
merged 5 commits into from
Aug 30, 2023
Merged

[summerospp]add fluentbit opentelemetry plugin #890

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c0e20b9
[summerospp]add fluentbit opentelemetry plugin
Aug 26, 2023
8c07d09
[summerospp]add fluentbit opentelemetry plugin
Aug 30, 2023
40ec334
[summerospp]add fluentbit opentelemetry plugin
Aug 30, 2023
4ef6513
Merge branch 'master' into opentelemetry
wenchajun Aug 30, 2023
cd92293
Delete .DS_Store
sjliu1 Aug 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apis/fluentbit/v1alpha2/clusterinput_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ type InputSpec struct {
CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"`
// Forward defines forward input plugin configuration
Forward *input.Forward `json:"forward,omitempty"`
// OpenTelemetry defines forward input plugin configuration
OpenTelemetry *input.OpenTelemetry `json:"opentelemetry,omitempty"`
sjliu1 marked this conversation as resolved.
Show resolved Hide resolved
}

// +kubebuilder:object:root=true
Expand Down
65 changes: 65 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/input/open_telemetry_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package input

import (
"fmt"

"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params"
)

// +kubebuilder:object:generate:=true

// The OpenTelemetry plugin allows you to ingest telemetry data as per the OTLP specification, <br />
// from various OpenTelemetry exporters, the OpenTelemetry Collector, or Fluent Bit's OpenTelemetry output plugin. <br />
// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/opentelemetry**
type OpenTelemetry struct {
// The address to listen on,default 0.0.0.0
Listen string `json:"listen,omitempty"`
// The port for Fluent Bit to listen on.default 4318.
// +kubebuilder:validation:Minimum:=1
// +kubebuilder:validation:Maximum:=65535
Port *int32 `json:"port,omitempty"`
// Specify the key name to overwrite a tag. If set, the tag will be overwritten by a value of the key.
Tagkey string `json:"tagKey,omitempty"`
// Route trace data as a log message(default false).
RawTraces *bool `json:"rawTraces,omitempty"`
// Specify the maximum buffer size in KB to receive a JSON message(default 4M).
// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
bufferMaxSize string `json:"bufferMaxSize,omitempty"`

Check failure on line 28 in apis/fluentbit/v1alpha2/plugins/input/open_telemetry_types.go

View workflow job for this annotation

GitHub Actions / Basic test and verify 

struct field bufferMaxSize has json tag but is not exported
sjliu1 marked this conversation as resolved.
Show resolved Hide resolved
// This sets the chunk size for incoming incoming JSON messages. These chunks are then stored/managed in the space available by buffer_max_size(default 512K).
// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
BufferChunkSize string `json:"bufferChunkSize,omitempty"`
//It allows to set successful response code. 200, 201 and 204 are supported(default 201).
SuccessfulResponseCode *int32 `json:"successfulResponseCode,omitempty"`
}

func (_ *OpenTelemetry) Name() string {
return "opentelemetry"
}

// implement Section() method
func (ot *OpenTelemetry) Params(_ plugins.SecretLoader) (*params.KVs, error) {
kvs := params.NewKVs()
if ot.Listen != "" {
kvs.Insert("listen", ot.Listen)
}
if ot.Port != nil {
kvs.Insert("port", fmt.Sprint(*ot.Port))
}
if ot.Tagkey != "" {
kvs.Insert("tag_key", ot.Tagkey)
}
if ot.RawTraces != nil {
kvs.Insert("raw_traces", fmt.Sprint(*ot.RawTraces))
}
if ot.bufferMaxSize != "" {
sjliu1 marked this conversation as resolved.
Show resolved Hide resolved
kvs.Insert("buffer_max_size", ot.bufferMaxSize)
sjliu1 marked this conversation as resolved.
Show resolved Hide resolved
}
if ot.BufferChunkSize != "" {
kvs.Insert("buffer_chunk_size", ot.BufferChunkSize)
}
if ot.SuccessfulResponseCode != nil {
kvs.Insert("successful_response_code", fmt.Sprint(*ot.SuccessfulResponseCode))
}
return kvs, nil
}
30 changes: 30 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions apis/fluentbit/v1alpha2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 36 additions & 0 deletions charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,42 @@ spec:
plugin.
type: string
type: object
opentelemetry:
description: OpenTelemetry defines forward input plugin configuration
properties:
bufferChunkSize:
description: This sets the chunk size for incoming incoming JSON
messages. These chunks are then stored/managed in the space
available by buffer_max_size(default 512K).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
bufferMaxSize:
description: Specify the maximum buffer size in KB to receive
a JSON message(default 4M).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
listen:
description: The address to listen on,default 0.0.0.0
type: string
port:
description: The port for Fluent Bit to listen on.default 4318.
format: int32
maximum: 65535
minimum: 1
type: integer
rawTraces:
description: Route trace data as a log message(default false).
type: boolean
successfulResponseCode:
description: It allows to set successful response code. 200, 201
and 204 are supported(default 201).
format: int32
type: integer
tagKey:
description: Specify the key name to overwrite a tag. If set,
the tag will be overwritten by a value of the key.
type: string
type: object
prometheusScrapeMetrics:
description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics
Input configuration.
Expand Down
36 changes: 36 additions & 0 deletions config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,42 @@ spec:
plugin.
type: string
type: object
opentelemetry:
Copy link
Member

@benjaminhuo benjaminhuo Aug 30, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sjliu1 Still opentelemetry , You'll need to regenerate the yaml by make test
And the opentelemetry in the helm chart needs to be changed to openTelemetry

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I have done it.

description: OpenTelemetry defines forward input plugin configuration
properties:
bufferChunkSize:
description: This sets the chunk size for incoming incoming JSON
messages. These chunks are then stored/managed in the space
available by buffer_max_size(default 512K).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
bufferMaxSize:
description: Specify the maximum buffer size in KB to receive
a JSON message(default 4M).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
listen:
description: The address to listen on,default 0.0.0.0
type: string
port:
description: The port for Fluent Bit to listen on.default 4318.
format: int32
maximum: 65535
minimum: 1
type: integer
rawTraces:
description: Route trace data as a log message(default false).
type: boolean
successfulResponseCode:
description: It allows to set successful response code. 200, 201
and 204 are supported(default 201).
format: int32
type: integer
tagKey:
description: Specify the key name to overwrite a tag. If set,
the tag will be overwritten by a value of the key.
type: string
type: object
prometheusScrapeMetrics:
description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics
Input configuration.
Expand Down
4 changes: 4 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,14 @@ require (
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.9.3 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.1 // indirect
Expand All @@ -72,7 +74,9 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.26.1 // indirect
k8s.io/code-generator v0.26.1 // indirect
k8s.io/component-base v0.26.1 // indirect
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
Expand Down
11 changes: 11 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
Expand Down Expand Up @@ -349,6 +350,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -503,6 +506,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
Expand All @@ -513,6 +517,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down Expand Up @@ -640,8 +645,13 @@ k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs=
k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s=
k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ=
k8s.io/code-generator v0.26.1 h1:dusFDsnNSKlMFYhzIM0jAO1OlnTN5WYwQQ+Ai12IIlo=
k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I=
k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU=
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08=
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
Expand All @@ -657,5 +667,6 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
36 changes: 36 additions & 0 deletions manifests/setup/fluent-operator-crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1905,6 +1905,42 @@ spec:
plugin.
type: string
type: object
opentelemetry:
description: OpenTelemetry defines forward input plugin configuration
properties:
bufferChunkSize:
description: This sets the chunk size for incoming incoming JSON
messages. These chunks are then stored/managed in the space
available by buffer_max_size(default 512K).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
bufferMaxSize:
description: Specify the maximum buffer size in KB to receive
a JSON message(default 4M).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
listen:
description: The address to listen on,default 0.0.0.0
type: string
port:
description: The port for Fluent Bit to listen on.default 4318.
format: int32
maximum: 65535
minimum: 1
type: integer
rawTraces:
description: Route trace data as a log message(default false).
type: boolean
successfulResponseCode:
description: It allows to set successful response code. 200, 201
and 204 are supported(default 201).
format: int32
type: integer
tagKey:
description: Specify the key name to overwrite a tag. If set,
the tag will be overwritten by a value of the key.
type: string
type: object
prometheusScrapeMetrics:
description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics
Input configuration.
Expand Down
36 changes: 36 additions & 0 deletions manifests/setup/setup.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1905,6 +1905,42 @@ spec:
plugin.
type: string
type: object
opentelemetry:
description: OpenTelemetry defines forward input plugin configuration
properties:
bufferChunkSize:
description: This sets the chunk size for incoming incoming JSON
messages. These chunks are then stored/managed in the space
available by buffer_max_size(default 512K).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
bufferMaxSize:
description: Specify the maximum buffer size in KB to receive
a JSON message(default 4M).
pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
type: string
listen:
description: The address to listen on,default 0.0.0.0
type: string
port:
description: The port for Fluent Bit to listen on.default 4318.
format: int32
maximum: 65535
minimum: 1
type: integer
rawTraces:
description: Route trace data as a log message(default false).
type: boolean
successfulResponseCode:
description: It allows to set successful response code. 200, 201
and 204 are supported(default 201).
format: int32
type: integer
tagKey:
description: Specify the key name to overwrite a tag. If set,
the tag will be overwritten by a value of the key.
type: string
type: object
prometheusScrapeMetrics:
description: PrometheusScrapeMetrics defines Prometheus Scrape Metrics
Input configuration.
Expand Down
Loading