Skip to content

Commit

Permalink
feat(operator): Add support for Loki OTLP limits config (#13446)
Browse files Browse the repository at this point in the history
Co-authored-by: Robert Jacob <[email protected]>
  • Loading branch information
periklis and xperimental authored Sep 6, 2024
1 parent 1b3ba53 commit d02f435
Show file tree
Hide file tree
Showing 19 changed files with 3,374 additions and 3 deletions.
163 changes: 163 additions & 0 deletions operator/apis/loki/v1/lokistack_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,140 @@ type IngestionLimitSpec struct {
PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"`
}

// OTLPAttributeAction defines the action to executed when indexing
// OTLP resource attributes. Resource attributes can be either added
// to the index, the chunk structured metadata or entirely dropped.
type OTLPAttributeAction string

const (
// OTLPAttributeActionIndexLabel stores a resource attribute as a label, which is part of the index identifying streams.
OTLPAttributeActionIndexLabel OTLPAttributeAction = "indexLabel"
// OTLPAttributeActionStructuredMetadata stores an attribute as structured metadata with each log entry.
OTLPAttributeActionStructuredMetadata OTLPAttributeAction = "structuredMetadata"
// OTLPAttributeActionDrop removes the matching attributes from the log entry.
OTLPAttributeActionDrop OTLPAttributeAction = "drop"
)

// OTLPAttributesSpec contains the configuration for a set of attributes
// to store them as index labels or structured metadata or drop them altogether.
type OTLPAttributesSpec struct {
// Action defines the indexing action for the selected attributes. They
// can be either added to structured metadata or drop altogether.
//
// +required
// +kubebuilder:validation:Required
// +kubebuilder:validation:Enum=structured_metadata;drop
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Action"
Action OTLPAttributeAction `json:"action"`

// Attributes allows choosing the attributes by listing their names.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attribute Names"
Attributes []string `json:"attributes,omitempty"`

// Regex allows choosing the attributes by matching a regular expression.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Regular Expression"
Regex string `json:"regex,omitempty"`
}

// OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes
// to store them as index labels or structured metadata or drop them altogether.
type OTLPResourceAttributesConfigSpec struct {
// Action defines the indexing action for the selected resoure attributes. They
// can be either indexed as labels, added to structured metadata or drop altogether.
//
// +required
// +kubebuilder:validation:Required
// +kubebuilder:validation:Enum=index_label;structured_metadata;drop
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Action"
Action OTLPAttributeAction `json:"action,omitempty"`

// Attributes is the list of attributes to configure indexing or drop them
// altogether.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attribute Names"
Attributes []string `json:"attributes,omitempty"`

// Regex allows choosing the attributes by matching a regular expression.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Regular Expression"
Regex string `json:"regex,omitempty"`
}

// OTLPResourceAttributesSpec contains the configuration for resource attributes
// to store them as index labels or structured metadata or drop them altogether.
type OTLPResourceAttributesSpec struct {
// IgnoreDefaults controls whether to ignore the global configuration for resource attributes
// indexed as labels.
//
// If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch",displayName="Ignore Global Defaults"
IgnoreDefaults bool `json:"ignoreDefaults,omitempty"`

// Attributes contains the configuration for resource attributes
// to store them as index labels or structured metadata or drop them altogether.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attributes"
Attributes []OTLPResourceAttributesConfigSpec `json:"attributes,omitempty"`
}

// GlobalOTLPSpec defines which resource, scope and log attributes to
// be stored as index or structured metadata or drop altogether for all
// tenants.
type GlobalOTLPSpec struct {
// IndexedResourceAttributes contains the global configuration for resource attributes
// to store them as index labels or structured metadata or drop them altogether.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Indexed Resource Attributes"
IndexedResourceAttributes []string `json:"indexedResourceAttributes,omitempty"`

OTLPSpec `json:",omitempty"`
}

// OTLPSpec defines which resource, scope and log attributes to
// be stored as index or structured metadata or drop altogether
type OTLPSpec struct {
// ResourceAttributes contains the configuration for resource attributes
// to store them as index labels or structured metadata or drop them altogether.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resource Attributes"
ResourceAttributes *OTLPResourceAttributesSpec `json:"resourceAttributes,omitempty"`

// ScopeAttributes contains the configuration for scope attributes
// to store them as index labels or structured metadata or drop them altogether.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Scope Attributes"
ScopeAttributes []OTLPAttributesSpec `json:"scopeAttributes,omitempty"`

// LogAttributes contains the configuration for log attributes
// to store them as index labels or structured metadata or drop them altogether.
//
// +optional
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Log Attributes"
LogAttributes []OTLPAttributesSpec `json:"logAttributes,omitempty"`
}

// RetentionStreamSpec defines a log stream with separate retention time.
type RetentionStreamSpec struct {
// Days contains the number of days logs are kept.
Expand Down Expand Up @@ -844,6 +978,14 @@ type LimitsTemplateSpec struct {
// +kubebuilder:validation:Optional
QueryLimits *QueryLimitSpec `json:"queries,omitempty"`

// OTLP to configure which resource, scope and log attributes
// to store as labels or structured metadata or drop them altogether
// for all tenants.
//
// +optional
// +kubebuilder:validation:Optional
OTLP *GlobalOTLPSpec `json:"otlp,omitempty"`

// Retention defines how long logs are kept in storage.
//
// +optional
Expand All @@ -865,6 +1007,14 @@ type PerTenantLimitsTemplateSpec struct {
// +kubebuilder:validation:Optional
QueryLimits *PerTenantQueryLimitSpec `json:"queries,omitempty"`

// OTLP to configure which resource, scope and log attributes
// to store as labels or structured metadata or drop them altogether
// for a single tenants.
//
// +optional
// +kubebuilder:validation:Optional
OTLP *OTLPSpec `json:"otlp,omitempty"`

// Retention defines how long logs are kept in storage.
//
// +optional
Expand Down Expand Up @@ -1313,3 +1463,16 @@ func (t BlockedQueryTypes) String() string {

return strings.Join(res, ",")
}

func (a OTLPAttributeAction) Value() string {
switch a {
case OTLPAttributeActionIndexLabel:
return "index_label"
case OTLPAttributeActionStructuredMetadata:
return "structured_metadata"
case OTLPAttributeActionDrop:
return "drop"
default:
return string(a)
}
}
7 changes: 7 additions & 0 deletions operator/apis/loki/v1/v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@ var (
// ErrIPv6InstanceAddrTypeNotAllowed when the default InstanceAddrType is used with enableIPv6.
ErrIPv6InstanceAddrTypeNotAllowed = errors.New(`instanceAddrType "default" cannot be used with enableIPv6 at the same time`)

// ErrOTLPResourceAttributesEmptyNotAllowed when the OTLP ResourceAttributes are empty even though ignoreDefaults is enabled.
ErrOTLPResourceAttributesEmptyNotAllowed = errors.New(`resourceAttributes cannot be empty when ignoreDefaults is true`)
// ErrOTLPResourceAttributesIndexLabelActionMissing when OTLP ResourceAttributes does not contain at least one index label when ignoreDefaults is enabled.
ErrOTLPResourceAttributesIndexLabelActionMissing = errors.New(`resourceAttributes does not contain at least one attributed mapped to "index_label"`)
// ErrOTLPAttributesSpecInvalid when the OTLPAttributesSpec attibutes and regex fields are both empty.
ErrOTLPAttributesSpecInvalid = errors.New(`attributes and regex cannot be empty at the same time`)

// ErrRuleMustMatchNamespace indicates that an expression used in an alerting or recording rule is missing
// matchers for a namespace.
ErrRuleMustMatchNamespace = errors.New("rule needs to have a matcher for the namespace")
Expand Down
127 changes: 127 additions & 0 deletions operator/apis/loki/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit d02f435

Please sign in to comment.