Merge branch 'main' into explore-logs-fallback-query-path

cmd/chunks-inspect/go.mod

@@ -7,6 +7,7 @@ toolchain go1.22.2
 require (
 	github.com/golang/snappy v0.0.4
 	github.com/grafana/loki/v3 v3.0.0
+	github.com/pkg/errors v0.9.1
 )
 
 require (
@@ -93,7 +94,6 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.19.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
@@ -126,7 +126,7 @@ require (
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
 	golang.org/x/mod v0.16.0 // indirect
-	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect

cmd/chunks-inspect/go.sum

@@ -709,8 +709,8 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

docs/sources/release-notes/v3-1.md

@@ -22,6 +22,8 @@ Key features in Loki 3.1.0 include the following:
 
 - **lokitool:** Add `lokitool` to replace `cortextool`. ([#12166](https://github.com/grafana/loki/issues/12166)) ([7b7d3d4](https://github.com/grafana/loki/commit/7b7d3d4cd2c979c778d3741156f0d765a9e531b2)). Introduce `index audit` to `lokitool` ([#13008](https://github.com/grafana/loki/issues/13008)) ([47f0236](https://github.com/grafana/loki/commit/47f0236ea8f33a67a0a1abf6e6d6b3582661c4ba)).
 
+- **Explore Logs:** Explore Logs, which lets you explore your Loki data without writing LogQL queries, is now available in public preview. If you are a Grafana Cloud user, you can access Explore Logs in the Grafana Cloud main navigation menu. If you are not a Grafana Cloud user, you can install the [Explore Logs plugin](https://grafana.com/docs/grafana-cloud/visualizations/simplified-exploration/logs/access/).  For more information, refer to the [Explore Logs documentation](https://grafana.com/docs/grafana-cloud/visualizations/simplified-exploration/logs/).
+
 - **Docs:** Added a video to the Getting Started demo and updated for Grafana Alloy. Added an interactive sandbox to the Loki Quickstart tutorial. Updated the documentation for the SSD and microservices deployment modes using the Helm charts.  Documented the new meta-monitoring Helm chart.
 
 Other improvements include the following:

docs/sources/setup/install/helm/reference.md

@@ -2640,7 +2640,7 @@ null
     "tolerations": []
   },
   "useExternalLicense": false,
-  "version": "3.0.1"
+  "version": "3.1.0"
 }
 </pre>
 </td>

docs/sources/shared/configuration.md

@@ -316,6 +316,25 @@ ingester_rf1:
   # CLI flag: -ingester-rf1.flush-op-timeout
   [flush_op_timeout: <duration> | default = 10m]
 
+  # The maximum age of a segment before it should be flushed. Increasing this
+  # value allows more time for a segment to grow to max-segment-size, but may
+  # increase latency if the write volume is too small.
+  # CLI flag: -ingester-rf1.max-segment-age
+  [max_segment_age: <duration> | default = 500ms]
+
+  # The maximum size of a segment before it should be flushed. It is not a
+  # strict limit, and segments can exceed the maximum size when individual
+  # appends are larger than the remaining capacity.
+  # CLI flag: -ingester-rf1.max-segment-size
+  [max_segment_size: <int> | default = 8388608]
+
+  # The maximum number of segments to buffer in-memory. Increasing this value
+  # allows for large bursts of writes to be buffered in memory, but may increase
+  # latency if the write volume exceeds the rate at which segments can be
+  # flushed.
+  # CLI flag: -ingester-rf1.max-segments
+  [max_segments: <int> | default = 10]
+
   # How long chunks should be retained in-memory after they've been flushed.
   # CLI flag: -ingester-rf1.chunks-retain-period
   [chunk_retain_period: <duration> | default = 0s]

operator/internal/handlers/internal/storage/storage.go

@@ -45,8 +45,9 @@ func BuildOptions(ctx context.Context, k k8s.Client, stack *lokiv1.LokiStack, fg
 		}
 	}
 
+	now := time.Now().UTC()
 	storageSchemas, err := storage.BuildSchemaConfig(
-		time.Now().UTC(),
+		now,
 		stack.Spec.Storage,
 		stack.Status.Storage,
 	)
@@ -59,6 +60,7 @@ func BuildOptions(ctx context.Context, k k8s.Client, stack *lokiv1.LokiStack, fg
 	}
 
 	objStore.Schemas = storageSchemas
+	objStore.AllowStructuredMetadata = allowStructuredMetadata(storageSchemas, now)
 
 	if stack.Spec.Storage.TLS == nil {
 		return objStore, nil
@@ -98,3 +100,16 @@ func BuildOptions(ctx context.Context, k k8s.Client, stack *lokiv1.LokiStack, fg
 
 	return objStore, nil
 }
+
+func allowStructuredMetadata(schemas []lokiv1.ObjectStorageSchema, now time.Time) bool {
+	activeVersion := lokiv1.ObjectStorageSchemaV11
+	for _, s := range schemas {
+		time, _ := s.EffectiveDate.UTCTime()
+		if time.Before(now) {
+			activeVersion = s.Version
+		}
+	}
+
+	return activeVersion != lokiv1.ObjectStorageSchemaV11 &&
+		activeVersion != lokiv1.ObjectStorageSchemaV12
+}

operator/internal/handlers/internal/storage/storage_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -560,3 +561,124 @@ func TestBuildOptions_WhenInvalidCAConfigMap_SetDegraded(t *testing.T) {
 	require.Error(t, err)
 	require.Equal(t, degradedErr, err)
 }
+
+func TestAllowStructuredMetadata(t *testing.T) {
+	testTime := time.Date(2024, 7, 1, 1, 0, 0, 0, time.UTC)
+	tt := []struct {
+		desc      string
+		schemas   []lokiv1.ObjectStorageSchema
+		wantAllow bool
+	}{
+		{
+			desc:      "disallow - no schemas",
+			schemas:   []lokiv1.ObjectStorageSchema{},
+			wantAllow: false,
+		},
+		{
+			desc: "disallow - only v12",
+			schemas: []lokiv1.ObjectStorageSchema{
+				{
+					Version:       lokiv1.ObjectStorageSchemaV12,
+					EffectiveDate: "2024-07-01",
+				},
+			},
+			wantAllow: false,
+		},
+		{
+			desc: "allow - only v13",
+			schemas: []lokiv1.ObjectStorageSchema{
+				{
+					Version:       lokiv1.ObjectStorageSchemaV13,
+					EffectiveDate: "2024-07-01",
+				},
+			},
+			wantAllow: true,
+		},
+		{
+			desc: "disallow - v13 in future",
+			schemas: []lokiv1.ObjectStorageSchema{
+				{
+					Version:       lokiv1.ObjectStorageSchemaV12,
+					EffectiveDate: "2024-07-01",
+				},
+				{
+					Version:       lokiv1.ObjectStorageSchemaV13,
+					EffectiveDate: "2024-07-02",
+				},
+			},
+			wantAllow: false,
+		},
+		{
+			desc: "disallow - v13 in past",
+			schemas: []lokiv1.ObjectStorageSchema{
+				{
+					Version:       lokiv1.ObjectStorageSchemaV13,
+					EffectiveDate: "2024-06-01",
+				},
+				{
+					Version:       lokiv1.ObjectStorageSchemaV12,
+					EffectiveDate: "2024-07-01",
+				},
+			},
+			wantAllow: false,
+		},
+		{
+			desc: "disallow - v13 in past and future",
+			schemas: []lokiv1.ObjectStorageSchema{
+				{
+					Version:       lokiv1.ObjectStorageSchemaV13,
+					EffectiveDate: "2024-06-01",
+				},
+				{
+					Version:       lokiv1.ObjectStorageSchemaV12,
+					EffectiveDate: "2024-07-01",
+				},
+				{
+					Version:       lokiv1.ObjectStorageSchemaV13,
+					EffectiveDate: "2024-07-02",
+				},
+			},
+			wantAllow: false,
+		},
+		{
+			desc: "allow - v13 active",
+			schemas: []lokiv1.ObjectStorageSchema{
+				{
+					Version:       lokiv1.ObjectStorageSchemaV12,
+					EffectiveDate: "2024-06-01",
+				},
+				{
+					Version:       lokiv1.ObjectStorageSchemaV13,
+					EffectiveDate: "2024-07-01",
+				},
+			},
+			wantAllow: true,
+		},
+		{
+			desc: "allow - v13 active, v12 in future",
+			schemas: []lokiv1.ObjectStorageSchema{
+				{
+					Version:       lokiv1.ObjectStorageSchemaV13,
+					EffectiveDate: "2024-07-01",
+				},
+				{
+					Version:       lokiv1.ObjectStorageSchemaV12,
+					EffectiveDate: "2024-08-01",
+				},
+			},
+			wantAllow: true,
+		},
+	}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
+
+			allow := allowStructuredMetadata(tc.schemas, testTime)
+			if allow != tc.wantAllow {
+				t.Errorf("got %v, want %v", allow, tc.wantAllow)
+			}
+		})
+	}
+}