Merge branch 'main' into virtual-host-style-s3

docs/sources/configure/_index.md

@@ -3265,7 +3265,7 @@ shard_streams:
 
 # Allow user to send structured metadata in push payload.
 # CLI flag: -validation.allow-structured-metadata
-[allow_structured_metadata: <boolean> | default = false]
+[allow_structured_metadata: <boolean> | default = true]
 
 # Maximum size accepted for structured metadata per log line.
 # CLI flag: -limits.max-structured-metadata-size
@@ -4792,7 +4792,7 @@ The `period_config` block configures what index schemas should be used for from
 # gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc.
 [object_store: <string> | default = ""]
 
-# The schema version to use, current recommended schema is v12.
+# The schema version to use, current recommended schema is v13.
 [schema: <string> | default = ""]
 
 # Configures how the index is updated and stored.

docs/sources/configure/examples/configuration-examples.md

@@ -404,61 +404,3 @@ memberlist:
 
 ```
 
-
-## 16-(Deprecated)-Cassandra-Snippet.yaml
-
-```yaml
-
-# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage
-# WARNING - DEPRECATED: The Cassandra index store is deprecated and will be removed in a future release.
-
-schema_config:
-  configs:
-  - from: 2020-05-15
-    store: cassandra
-    object_store: filesystem
-    schema: v12
-    index:
-      prefix: cassandra_table
-      period: 168h
-
-storage_config:
-  cassandra:
-    username: cassandra
-    password: cassandra
-    addresses: 127.0.0.1
-    auth: true
-    keyspace: lokiindex
-
-  filesystem:
-    directory: /tmp/loki/chunks
-
-
-```
-
-
-## 17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml
-
-```yaml
-
-# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage
-# WARNING - DEPRECATED: The DynamoDB index store is deprecated and will be removed in a future release.
-
-schema_config:
-  configs:
-  - from: 2020-05-15
-    store: aws
-    object_store: s3
-    schema: v12
-    index:
-      prefix: loki_
-
-storage_config:
-  aws:
-    s3: s3://access_key:secret_access_key@region/bucket_name
-    dynamodb:
-      dynamodb_url: dynamodb://access_key:secret_access_key@region
-
-
-```
-

docs/sources/setup/upgrade/_index.md

@@ -174,6 +174,14 @@ This new metric will provide a more clear signal that there is an issue with ing
 | `legacy-read-mode`                                     | false       | true        | Deprecated. It will be removed in the next minor release. |
 {{% /responsive-table %}}
 
+#### Structured Metadata, Open Telemetry, Schemas and Indexes
+
+A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{< relref "../../get-started/labels/structured-metadata" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line.
+
+Structured Metadata is enabled by default in Loki 3.0, however, it requires your active schema be using both the `TSDB` index type AND the `v13` storage schema.  If you are not using both of these you have two options:
+  * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../operations/storage/schema#changing-the-schema" >}}). 
+  * Disable Structured Metadata (and therefor OTLP support) and upgrade to 3.0 and perform the schema migration after. This can be done by setting `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false`.
+
 #### Automatic stream sharding is enabled by default
 
 Automatic stream sharding helps keep the write load of high volume streams balanced across ingesters and helps to avoid hot-spotting. Check out the [operations page](https://grafana.com/docs/loki/latest/operations/automatic-stream-sharding/) for more information

integration/loki_micro_services_delete_test.go

@@ -29,7 +29,7 @@ type pushRequest struct {
 }
 
 func TestMicroServicesDeleteRequest(t *testing.T) {
-	clu := cluster.New(nil, cluster.SchemaWithBoltDBAndBoltDB, func(c *cluster.Cluster) {
+	clu := cluster.New(nil, cluster.SchemaWithTSDBAndTSDB, func(c *cluster.Cluster) {
 		c.SetSchemaVer("v13")
 	})
 	defer func() {

integration/loki_micro_services_test.go

@@ -30,7 +30,9 @@ import (
 )
 
 func TestMicroServicesIngestQuery(t *testing.T) {
-	clu := cluster.New(nil)
+	clu := cluster.New(nil, cluster.SchemaWithTSDBAndTSDB, func(c *cluster.Cluster) {
+		c.SetSchemaVer("v13")
+	})
 	defer func() {
 		assert.NoError(t, clu.Cleanup())
 	}()
@@ -227,10 +229,12 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) {
 			"compactor",
 			"-target=compactor",
 			"-compactor.compaction-interval=1s",
+			"-validation.allow-structured-metadata=false",
 		)
 		tDistributor = clu.AddComponent(
 			"distributor",
 			"-target=distributor",
+			"-validation.allow-structured-metadata=false",
 		)
 	)
 	require.NoError(t, clu.Run())
@@ -241,11 +245,13 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) {
 			"ingester",
 			"-target=ingester",
 			"-ingester.flush-on-shutdown=true",
+			"-validation.allow-structured-metadata=false",
 		)
 		tQueryScheduler = clu.AddComponent(
 			"query-scheduler",
 			"-target=query-scheduler",
 			"-query-scheduler.use-scheduler-ring=false",
+			"-validation.allow-structured-metadata=false",
 		)
 	)
 	require.NoError(t, clu.Run())
@@ -258,12 +264,14 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) {
 			"-frontend.scheduler-address="+tQueryScheduler.GRPCURL(),
 			"-frontend.default-validity=0s",
 			"-common.compactor-address="+tCompactor.HTTPURL(),
+			"-validation.allow-structured-metadata=false",
 		)
 		tQuerier = clu.AddComponent(
 			"querier",
 			"-target=querier",
 			"-querier.scheduler-address="+tQueryScheduler.GRPCURL(),
 			"-common.compactor-address="+tCompactor.HTTPURL(),
+			"-validation.allow-structured-metadata=false",
 		)
 	)
 	require.NoError(t, clu.Run())
@@ -420,10 +428,12 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T)
 					"compactor",
 					"-target=compactor",
 					"-compactor.compaction-interval=1s",
+					"-validation.allow-structured-metadata=false",
 				)
 				tDistributor = clu.AddComponent(
 					"distributor",
 					"-target=distributor",
+					"-validation.allow-structured-metadata=false",
 				)
 			)
 			require.NoError(t, clu.Run())
@@ -434,11 +444,13 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T)
 					"ingester",
 					"-target=ingester",
 					"-ingester.flush-on-shutdown=true",
+					"-validation.allow-structured-metadata=false",
 				)
 				tQueryScheduler = clu.AddComponent(
 					"query-scheduler",
 					"-target=query-scheduler",
 					"-query-scheduler.use-scheduler-ring=false",
+					"-validation.allow-structured-metadata=false",
 				)
 			)
 			require.NoError(t, clu.Run())
@@ -451,12 +463,14 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T)
 					"-frontend.scheduler-address="+tQueryScheduler.GRPCURL(),
 					"-frontend.default-validity=0s",
 					"-common.compactor-address="+tCompactor.HTTPURL(),
+					"-validation.allow-structured-metadata=false",
 				)
 				tQuerier = clu.AddComponent(
 					"querier",
 					"-target=querier",
 					"-querier.scheduler-address="+tQueryScheduler.GRPCURL(),
 					"-common.compactor-address="+tCompactor.HTTPURL(),
+					"-validation.allow-structured-metadata=false",
 				)
 			)
 			require.NoError(t, clu.Run())
@@ -472,12 +486,12 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T)
 			cliQueryFrontend.Now = now
 
 			t.Run("ingest-logs", func(t *testing.T) {
-				require.NoError(t, cliDistributor.PushLogLine("lineA", time.Now().Add(-48*time.Hour), map[string]string{"traceID": "123"}, map[string]string{"job": "fake"}))
-				require.NoError(t, cliDistributor.PushLogLine("lineB", time.Now().Add(-36*time.Hour), map[string]string{"traceID": "456"}, map[string]string{"job": "fake"}))
+				require.NoError(t, cliDistributor.PushLogLine("lineA", time.Now().Add(-48*time.Hour), nil, map[string]string{"job": "fake"}))
+				require.NoError(t, cliDistributor.PushLogLine("lineB", time.Now().Add(-36*time.Hour), nil, map[string]string{"job": "fake"}))
 
 				// ingest logs to the current period
-				require.NoError(t, cliDistributor.PushLogLine("lineC", now, map[string]string{"traceID": "789"}, map[string]string{"job": "fake"}))
-				require.NoError(t, cliDistributor.PushLogLine("lineD", now, map[string]string{"traceID": "123"}, map[string]string{"job": "fake"}))
+				require.NoError(t, cliDistributor.PushLogLine("lineC", now, nil, map[string]string{"job": "fake"}))
+				require.NoError(t, cliDistributor.PushLogLine("lineD", now, nil, map[string]string{"job": "fake"}))
 
 			})
 
@@ -527,7 +541,9 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T)
 }
 
 func TestSchedulerRing(t *testing.T) {
-	clu := cluster.New(nil)
+	clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) {
+		c.SetSchemaVer("v13")
+	})
 	defer func() {
 		assert.NoError(t, clu.Cleanup())
 	}()
@@ -645,7 +661,7 @@ func TestSchedulerRing(t *testing.T) {
 }
 
 func TestOTLPLogsIngestQuery(t *testing.T) {
-	clu := cluster.New(nil, func(c *cluster.Cluster) {
+	clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) {
 		c.SetSchemaVer("v13")
 	})
 	defer func() {

integration/loki_rule_eval_test.go

@@ -36,7 +36,9 @@ func TestRemoteRuleEval(t *testing.T) {
 // In this test we stub out a remote-write receiver and check that the expected data is sent to it.
 // Both the local and the remote rule evaluation modes should produce the same result.
 func testRuleEval(t *testing.T, mode string) {
-	clu := cluster.New(nil)
+	clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) {
+		c.SetSchemaVer("v13")
+	})
 	t.Cleanup(func() {
 		assert.NoError(t, clu.Cleanup())
 	})

integration/loki_simple_scalable_test.go

@@ -15,7 +15,9 @@ import (
 )
 
 func TestSimpleScalable_IngestQuery(t *testing.T) {
-	clu := cluster.New(nil)
+	clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) {
+		c.SetSchemaVer("v13")
+	})
 	defer func() {
 		assert.NoError(t, clu.Cleanup())
 	}()

integration/loki_single_binary_test.go

@@ -15,7 +15,9 @@ import (
 )
 
 func TestSingleBinaryIngestQuery(t *testing.T) {
-	clu := cluster.New(nil)
+	clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) {
+		c.SetSchemaVer("v13")
+	})
 	defer func() {
 		assert.NoError(t, clu.Cleanup())
 	}()

integration/per_request_limits_test.go

@@ -17,7 +17,9 @@ import (
 )
 
 func TestPerRequestLimits(t *testing.T) {
-	clu := cluster.New(nil)
+	clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) {
+		c.SetSchemaVer("v13")
+	})
 	defer func() {
 		assert.NoError(t, clu.Cleanup())
 	}()

operator/CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Main
 
 - [12469](https://github.com/grafana/loki/pull/12469) **btaani**: Configure Loki to use virtual-host-style URLs for S3 AWS endpoints
+- [12370](https://github.com/grafana/loki/pull/12370) **periklis**: Update Loki operand to v2.9.6
 - [12333](https://github.com/grafana/loki/pull/12333) **periklis**: Bump max OpenShift version to next release
 
 ## 0.6.0 (2024-03-19)

operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.6.0
-    createdAt: "2024-03-19T14:29:49Z"
+    createdAt: "2024-03-27T08:04:23Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     features.operators.openshift.io/disconnected: "true"
@@ -1714,7 +1714,7 @@ spec:
                 - /manager
                 env:
                 - name: RELATED_IMAGE_LOKI
-                  value: docker.io/grafana/loki:2.9.4
+                  value: docker.io/grafana/loki:2.9.6
                 - name: RELATED_IMAGE_GATEWAY
                   value: quay.io/observatorium/api:latest
                 - name: RELATED_IMAGE_OPA
@@ -1838,7 +1838,7 @@ spec:
   provider:
     name: Grafana Loki SIG Operator
   relatedImages:
-  - image: docker.io/grafana/loki:2.9.4
+  - image: docker.io/grafana/loki:2.9.6
     name: loki
   - image: quay.io/observatorium/api:latest
     name: gateway

operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.6.0
-    createdAt: "2024-03-19T14:29:48Z"
+    createdAt: "2024-03-27T08:04:21Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown
@@ -1694,7 +1694,7 @@ spec:
                 - /manager
                 env:
                 - name: RELATED_IMAGE_LOKI
-                  value: docker.io/grafana/loki:2.9.4
+                  value: docker.io/grafana/loki:2.9.6
                 - name: RELATED_IMAGE_GATEWAY
                   value: quay.io/observatorium/api:latest
                 - name: RELATED_IMAGE_OPA
@@ -1806,7 +1806,7 @@ spec:
   provider:
     name: Grafana Loki SIG Operator
   relatedImages:
-  - image: docker.io/grafana/loki:2.9.4
+  - image: docker.io/grafana/loki:2.9.6
     name: loki
   - image: quay.io/observatorium/api:latest
     name: gateway