diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index bd561e1116b9..e0425597ec9b 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -3265,7 +3265,7 @@ shard_streams: # Allow user to send structured metadata in push payload. # CLI flag: -validation.allow-structured-metadata -[allow_structured_metadata: | default = false] +[allow_structured_metadata: | default = true] # Maximum size accepted for structured metadata per log line. # CLI flag: -limits.max-structured-metadata-size @@ -4792,7 +4792,7 @@ The `period_config` block configures what index schemas should be used for from # gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc. [object_store: | default = ""] -# The schema version to use, current recommended schema is v12. +# The schema version to use, current recommended schema is v13. [schema: | default = ""] # Configures how the index is updated and stored. diff --git a/docs/sources/configure/examples/configuration-examples.md b/docs/sources/configure/examples/configuration-examples.md index eaaf659049de..76b5ffc7a719 100644 --- a/docs/sources/configure/examples/configuration-examples.md +++ b/docs/sources/configure/examples/configuration-examples.md @@ -404,61 +404,3 @@ memberlist: ``` - -## 16-(Deprecated)-Cassandra-Snippet.yaml - -```yaml - -# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage -# WARNING - DEPRECATED: The Cassandra index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: cassandra - object_store: filesystem - schema: v12 - index: - prefix: cassandra_table - period: 168h - -storage_config: - cassandra: - username: cassandra - password: cassandra - addresses: 127.0.0.1 - auth: true - keyspace: lokiindex - - filesystem: - directory: /tmp/loki/chunks - - -``` - - -## 17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml - -```yaml - -# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage -# WARNING - DEPRECATED: The DynamoDB index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: aws - object_store: s3 - schema: v12 - index: - prefix: loki_ - -storage_config: - aws: - s3: s3://access_key:secret_access_key@region/bucket_name - dynamodb: - dynamodb_url: dynamodb://access_key:secret_access_key@region - - -``` - diff --git a/docs/sources/configure/examples/yaml/16-(Deprecated)-Cassandra-Snippet.yaml b/docs/sources/configure/examples/yaml/16-(Deprecated)-Cassandra-Snippet.yaml deleted file mode 100644 index 71cacb574368..000000000000 --- a/docs/sources/configure/examples/yaml/16-(Deprecated)-Cassandra-Snippet.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage -# WARNING - DEPRECATED: The Cassandra index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: cassandra - object_store: filesystem - schema: v12 - index: - prefix: cassandra_table - period: 168h - -storage_config: - cassandra: - username: cassandra - password: cassandra - addresses: 127.0.0.1 - auth: true - keyspace: lokiindex - - filesystem: - directory: /tmp/loki/chunks - \ No newline at end of file diff --git a/docs/sources/configure/examples/yaml/17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml b/docs/sources/configure/examples/yaml/17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml deleted file mode 100644 index 0b297b02650b..000000000000 --- a/docs/sources/configure/examples/yaml/17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage -# WARNING - DEPRECATED: The DynamoDB index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: aws - object_store: s3 - schema: v12 - index: - prefix: loki_ - -storage_config: - aws: - s3: s3://access_key:secret_access_key@region/bucket_name - dynamodb: - dynamodb_url: dynamodb://access_key:secret_access_key@region - \ No newline at end of file diff --git a/docs/sources/setup/upgrade/_index.md b/docs/sources/setup/upgrade/_index.md index 481cf14182d7..133d9493fc53 100644 --- a/docs/sources/setup/upgrade/_index.md +++ b/docs/sources/setup/upgrade/_index.md @@ -174,6 +174,14 @@ This new metric will provide a more clear signal that there is an issue with ing | `legacy-read-mode` | false | true | Deprecated. It will be removed in the next minor release. | {{% /responsive-table %}} +#### Structured Metadata, Open Telemetry, Schemas and Indexes + +A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{< relref "../../get-started/labels/structured-metadata" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. + +Structured Metadata is enabled by default in Loki 3.0, however, it requires your active schema be using both the `TSDB` index type AND the `v13` storage schema. If you are not using both of these you have two options: + * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../operations/storage/schema#changing-the-schema" >}}). + * Disable Structured Metadata (and therefor OTLP support) and upgrade to 3.0 and perform the schema migration after. This can be done by setting `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false`. + #### Automatic stream sharding is enabled by default Automatic stream sharding helps keep the write load of high volume streams balanced across ingesters and helps to avoid hot-spotting. Check out the [operations page](https://grafana.com/docs/loki/latest/operations/automatic-stream-sharding/) for more information diff --git a/integration/loki_micro_services_delete_test.go b/integration/loki_micro_services_delete_test.go index ce83cdb4d9f5..068d10ceb68b 100644 --- a/integration/loki_micro_services_delete_test.go +++ b/integration/loki_micro_services_delete_test.go @@ -29,7 +29,7 @@ type pushRequest struct { } func TestMicroServicesDeleteRequest(t *testing.T) { - clu := cluster.New(nil, cluster.SchemaWithBoltDBAndBoltDB, func(c *cluster.Cluster) { + clu := cluster.New(nil, cluster.SchemaWithTSDBAndTSDB, func(c *cluster.Cluster) { c.SetSchemaVer("v13") }) defer func() { diff --git a/integration/loki_micro_services_test.go b/integration/loki_micro_services_test.go index 611fafb15ab7..33f28296bc06 100644 --- a/integration/loki_micro_services_test.go +++ b/integration/loki_micro_services_test.go @@ -30,7 +30,9 @@ import ( ) func TestMicroServicesIngestQuery(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDBAndTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() @@ -227,10 +229,12 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) { "compactor", "-target=compactor", "-compactor.compaction-interval=1s", + "-validation.allow-structured-metadata=false", ) tDistributor = clu.AddComponent( "distributor", "-target=distributor", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -241,11 +245,13 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) { "ingester", "-target=ingester", "-ingester.flush-on-shutdown=true", + "-validation.allow-structured-metadata=false", ) tQueryScheduler = clu.AddComponent( "query-scheduler", "-target=query-scheduler", "-query-scheduler.use-scheduler-ring=false", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -258,12 +264,14 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) { "-frontend.scheduler-address="+tQueryScheduler.GRPCURL(), "-frontend.default-validity=0s", "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) tQuerier = clu.AddComponent( "querier", "-target=querier", "-querier.scheduler-address="+tQueryScheduler.GRPCURL(), "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -420,10 +428,12 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) "compactor", "-target=compactor", "-compactor.compaction-interval=1s", + "-validation.allow-structured-metadata=false", ) tDistributor = clu.AddComponent( "distributor", "-target=distributor", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -434,11 +444,13 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) "ingester", "-target=ingester", "-ingester.flush-on-shutdown=true", + "-validation.allow-structured-metadata=false", ) tQueryScheduler = clu.AddComponent( "query-scheduler", "-target=query-scheduler", "-query-scheduler.use-scheduler-ring=false", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -451,12 +463,14 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) "-frontend.scheduler-address="+tQueryScheduler.GRPCURL(), "-frontend.default-validity=0s", "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) tQuerier = clu.AddComponent( "querier", "-target=querier", "-querier.scheduler-address="+tQueryScheduler.GRPCURL(), "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -472,12 +486,12 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) cliQueryFrontend.Now = now t.Run("ingest-logs", func(t *testing.T) { - require.NoError(t, cliDistributor.PushLogLine("lineA", time.Now().Add(-48*time.Hour), map[string]string{"traceID": "123"}, map[string]string{"job": "fake"})) - require.NoError(t, cliDistributor.PushLogLine("lineB", time.Now().Add(-36*time.Hour), map[string]string{"traceID": "456"}, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineA", time.Now().Add(-48*time.Hour), nil, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineB", time.Now().Add(-36*time.Hour), nil, map[string]string{"job": "fake"})) // ingest logs to the current period - require.NoError(t, cliDistributor.PushLogLine("lineC", now, map[string]string{"traceID": "789"}, map[string]string{"job": "fake"})) - require.NoError(t, cliDistributor.PushLogLine("lineD", now, map[string]string{"traceID": "123"}, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineC", now, nil, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineD", now, nil, map[string]string{"job": "fake"})) }) @@ -527,7 +541,9 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) } func TestSchedulerRing(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() @@ -645,7 +661,7 @@ func TestSchedulerRing(t *testing.T) { } func TestOTLPLogsIngestQuery(t *testing.T) { - clu := cluster.New(nil, func(c *cluster.Cluster) { + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { c.SetSchemaVer("v13") }) defer func() { diff --git a/integration/loki_rule_eval_test.go b/integration/loki_rule_eval_test.go index 00caeef8883c..e0898c50829a 100644 --- a/integration/loki_rule_eval_test.go +++ b/integration/loki_rule_eval_test.go @@ -36,7 +36,9 @@ func TestRemoteRuleEval(t *testing.T) { // In this test we stub out a remote-write receiver and check that the expected data is sent to it. // Both the local and the remote rule evaluation modes should produce the same result. func testRuleEval(t *testing.T, mode string) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) t.Cleanup(func() { assert.NoError(t, clu.Cleanup()) }) diff --git a/integration/loki_simple_scalable_test.go b/integration/loki_simple_scalable_test.go index 070d3f918a14..aec97b116fbf 100644 --- a/integration/loki_simple_scalable_test.go +++ b/integration/loki_simple_scalable_test.go @@ -15,7 +15,9 @@ import ( ) func TestSimpleScalable_IngestQuery(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() diff --git a/integration/loki_single_binary_test.go b/integration/loki_single_binary_test.go index 6aaf64f5b415..693c946c2864 100644 --- a/integration/loki_single_binary_test.go +++ b/integration/loki_single_binary_test.go @@ -15,7 +15,9 @@ import ( ) func TestSingleBinaryIngestQuery(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() diff --git a/integration/per_request_limits_test.go b/integration/per_request_limits_test.go index 482ff0e93fcf..935593a04841 100644 --- a/integration/per_request_limits_test.go +++ b/integration/per_request_limits_test.go @@ -17,7 +17,9 @@ import ( ) func TestPerRequestLimits(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 2144591e0f3a..b64fe6b93e20 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,6 +1,7 @@ ## Main - [12469](https://github.com/grafana/loki/pull/12469) **btaani**: Configure Loki to use virtual-host-style URLs for S3 AWS endpoints +- [12370](https://github.com/grafana/loki/pull/12370) **periklis**: Update Loki operand to v2.9.6 - [12333](https://github.com/grafana/loki/pull/12333) **periklis**: Bump max OpenShift version to next release ## 0.6.0 (2024-03-19) diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index 0fe4276c4b72..36639683bc0d 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.0 - createdAt: "2024-03-19T14:29:49Z" + createdAt: "2024-03-27T08:04:23Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. features.operators.openshift.io/disconnected: "true" @@ -1714,7 +1714,7 @@ spec: - /manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:2.9.4 + value: docker.io/grafana/loki:2.9.6 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA @@ -1838,7 +1838,7 @@ spec: provider: name: Grafana Loki SIG Operator relatedImages: - - image: docker.io/grafana/loki:2.9.4 + - image: docker.io/grafana/loki:2.9.6 name: loki - image: quay.io/observatorium/api:latest name: gateway diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index 5179090de036..0e73ba8ac395 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.0 - createdAt: "2024-03-19T14:29:48Z" + createdAt: "2024-03-27T08:04:21Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -1694,7 +1694,7 @@ spec: - /manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:2.9.4 + value: docker.io/grafana/loki:2.9.6 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA @@ -1806,7 +1806,7 @@ spec: provider: name: Grafana Loki SIG Operator relatedImages: - - image: docker.io/grafana/loki:2.9.4 + - image: docker.io/grafana/loki:2.9.6 name: loki - image: quay.io/observatorium/api:latest name: gateway diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index f492a055d8fb..280feb060463 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:0.1.0 - createdAt: "2024-03-19T14:29:51Z" + createdAt: "2024-03-27T08:04:24Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -1699,7 +1699,7 @@ spec: - /manager env: - name: RELATED_IMAGE_LOKI - value: quay.io/openshift-logging/loki:v2.9.4 + value: quay.io/openshift-logging/loki:v2.9.6 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA @@ -1823,7 +1823,7 @@ spec: provider: name: Red Hat relatedImages: - - image: quay.io/openshift-logging/loki:v2.9.4 + - image: quay.io/openshift-logging/loki:v2.9.6 name: loki - image: quay.io/observatorium/api:latest name: gateway diff --git a/operator/config/overlays/community-openshift/manager_related_image_patch.yaml b/operator/config/overlays/community-openshift/manager_related_image_patch.yaml index 0b11adb57bbe..68084a440490 100644 --- a/operator/config/overlays/community-openshift/manager_related_image_patch.yaml +++ b/operator/config/overlays/community-openshift/manager_related_image_patch.yaml @@ -9,7 +9,7 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:2.9.4 + value: docker.io/grafana/loki:2.9.6 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA diff --git a/operator/config/overlays/community/manager_related_image_patch.yaml b/operator/config/overlays/community/manager_related_image_patch.yaml index 0b11adb57bbe..68084a440490 100644 --- a/operator/config/overlays/community/manager_related_image_patch.yaml +++ b/operator/config/overlays/community/manager_related_image_patch.yaml @@ -9,7 +9,7 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:2.9.4 + value: docker.io/grafana/loki:2.9.6 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA diff --git a/operator/config/overlays/development/manager_related_image_patch.yaml b/operator/config/overlays/development/manager_related_image_patch.yaml index a9c7cf22f1d1..65b258b1c41c 100644 --- a/operator/config/overlays/development/manager_related_image_patch.yaml +++ b/operator/config/overlays/development/manager_related_image_patch.yaml @@ -9,6 +9,6 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:2.9.4 + value: docker.io/grafana/loki:2.9.6 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest diff --git a/operator/config/overlays/openshift/manager_related_image_patch.yaml b/operator/config/overlays/openshift/manager_related_image_patch.yaml index 28c00163194a..7b10a721a0b8 100644 --- a/operator/config/overlays/openshift/manager_related_image_patch.yaml +++ b/operator/config/overlays/openshift/manager_related_image_patch.yaml @@ -9,7 +9,7 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: quay.io/openshift-logging/loki:v2.9.4 + value: quay.io/openshift-logging/loki:v2.9.6 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA diff --git a/operator/docs/operator/compatibility.md b/operator/docs/operator/compatibility.md index 2fc1e79662d1..30f86e0c272e 100644 --- a/operator/docs/operator/compatibility.md +++ b/operator/docs/operator/compatibility.md @@ -38,3 +38,4 @@ The versions of Loki compatible to be run with the Loki Operator are: * v2.9.2 * v2.9.3 * v2.9.4 +* v2.9.6 diff --git a/operator/hack/addons_dev.yaml b/operator/hack/addons_dev.yaml index f2538cd9f5af..9021682211a8 100644 --- a/operator/hack/addons_dev.yaml +++ b/operator/hack/addons_dev.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: logcli - image: docker.io/grafana/logcli:2.9.4-amd64 + image: docker.io/grafana/logcli:2.9.6-amd64 imagePullPolicy: IfNotPresent command: - /bin/sh @@ -73,7 +73,7 @@ spec: spec: containers: - name: promtail - image: docker.io/grafana/promtail:2.9.4 + image: docker.io/grafana/promtail:2.9.6 args: - -config.file=/etc/promtail/promtail.yaml - -log.level=info diff --git a/operator/hack/addons_ocp.yaml b/operator/hack/addons_ocp.yaml index 5a09cbc0e6aa..47faa521b5fe 100644 --- a/operator/hack/addons_ocp.yaml +++ b/operator/hack/addons_ocp.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: logcli - image: docker.io/grafana/logcli:2.9.4-amd64 + image: docker.io/grafana/logcli:2.9.6-amd64 imagePullPolicy: IfNotPresent command: - /bin/sh @@ -70,7 +70,7 @@ spec: spec: containers: - name: promtail - image: docker.io/grafana/promtail:2.9.4 + image: docker.io/grafana/promtail:2.9.6 args: - -config.file=/etc/promtail/promtail.yaml - -log.level=info diff --git a/operator/internal/manifests/var.go b/operator/internal/manifests/var.go index c81636fe5a55..bfdef5d0db59 100644 --- a/operator/internal/manifests/var.go +++ b/operator/internal/manifests/var.go @@ -59,7 +59,7 @@ const ( EnvRelatedImageGateway = "RELATED_IMAGE_GATEWAY" // DefaultContainerImage declares the default fallback for loki image. - DefaultContainerImage = "docker.io/grafana/loki:2.9.4" + DefaultContainerImage = "docker.io/grafana/loki:2.9.6" // DefaultLokiStackGatewayImage declares the default image for lokiStack-gateway. DefaultLokiStackGatewayImage = "quay.io/observatorium/api:latest" diff --git a/pkg/loki/config_test.go b/pkg/loki/config_test.go index 7a29f80bf02b..0fa36ea2ba9d 100644 --- a/pkg/loki/config_test.go +++ b/pkg/loki/config_test.go @@ -67,6 +67,8 @@ func TestCrossComponentValidation(t *testing.T) { }, } { tc.base.RegisterFlags(flag.NewFlagSet(tc.desc, 0)) + // This test predates the newer schema required for structured metadata + tc.base.LimitsConfig.AllowStructuredMetadata = false err := tc.base.Validate() if tc.err { require.NotNil(t, err) diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 536ec5728b05..c070cbe1025b 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -3,6 +3,7 @@ package loki import ( "bytes" "context" + stdlib_errors "errors" "flag" "fmt" "net/http" @@ -273,6 +274,29 @@ func (c *Config) Validate() error { } } + var errs []error + + // Schema version 13 is required to use structured metadata + p := config.ActivePeriodConfig(c.SchemaConfig.Configs) + version, err := c.SchemaConfig.Configs[p].VersionAsInt() + if err != nil { + return err + } + if c.LimitsConfig.AllowStructuredMetadata && version < 13 { + errs = append(errs, fmt.Errorf("CONFIG ERROR: schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is %s. Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].Schema)) + } + // TSDB index is required to use structured metadata + if c.LimitsConfig.AllowStructuredMetadata && c.SchemaConfig.Configs[p].IndexType != config.TSDBType { + errs = append(errs, fmt.Errorf("CONFIG ERROR: `tsdb` index type is required to store Structured Metadata and use native OTLP ingestion, your index type is `%s` (defined in the `store` parameter of the schema_config). Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update the schema to use index type `tsdb` before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].IndexType)) + } + + if len(errs) > 1 { + errs = append([]error{fmt.Errorf("MULTIPLE CONFIG ERRORS FOUND, PLEASE READ CAREFULLY")}, errs...) + return stdlib_errors.Join(errs...) + } else if len(errs) == 1 { + return errs[0] + } + return nil } diff --git a/pkg/storage/config/schema_config.go b/pkg/storage/config/schema_config.go index c7e72886b738..9ff620490177 100644 --- a/pkg/storage/config/schema_config.go +++ b/pkg/storage/config/schema_config.go @@ -161,7 +161,7 @@ type PeriodConfig struct { IndexType string `yaml:"store" doc:"description=store and object_store below affect which key is used. Which index to use. Either tsdb or boltdb-shipper. Following stores are deprecated: aws, aws-dynamo, gcp, gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc."` // type of object client to use. ObjectType string `yaml:"object_store" doc:"description=Which store to use for the chunks. Either aws (alias s3), azure, gcs, alibabacloud, bos, cos, swift, filesystem, or a named_store (refer to named_stores_config). Following stores are deprecated: aws-dynamo, gcp, gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc."` - Schema string `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v12."` + Schema string `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v13."` IndexTables IndexPeriodicTableConfig `yaml:"index" doc:"description=Configures how the index is updated and stored."` ChunkTables PeriodicTableConfig `yaml:"chunks" doc:"description=Configured how the chunks are updated and stored."` RowShards uint32 `yaml:"row_shards" doc:"default=16|description=How many shards will be created. Only used if schema is v10 or greater."` diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index ed8f50844768..a304fcfd362f 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -382,7 +382,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.VolumeMaxSeries, "limits.volume-max-series", 1000, "The default number of aggregated series or labels that can be returned from a log-volume endpoint") - f.BoolVar(&l.AllowStructuredMetadata, "validation.allow-structured-metadata", false, "Allow user to send structured metadata (non-indexed labels) in push payload.") + f.BoolVar(&l.AllowStructuredMetadata, "validation.allow-structured-metadata", true, "Allow user to send structured metadata (non-indexed labels) in push payload.") _ = l.MaxStructuredMetadataSize.Set(defaultMaxStructuredMetadataSize) f.Var(&l.MaxStructuredMetadataSize, "limits.max-structured-metadata-size", "Maximum size accepted for structured metadata per entry. Default: 64 kb. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.") f.IntVar(&l.MaxStructuredMetadataEntriesCount, "limits.max-structured-metadata-entries-count", defaultMaxStructuredMetadataCount, "Maximum number of structured metadata entries per log line. Default: 128. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.")