Merge branch 'main' into s3-secret-validation

.drone/drone.jsonnet

@@ -400,7 +400,7 @@ local manifest_ecr(apps, archs) = pipeline('manifest-ecr') {
   ],
 };
 
-local build_image_tag = '0.33.0';
+local build_image_tag = '0.33.1';
 [
   pipeline('loki-build-image-' + arch) {
     workspace: {

.drone/drone.yml

@@ -17,7 +17,7 @@ steps:
       from_secret: docker_password
     repo: grafana/loki-build-image
     tags:
-    - 0.33.0-amd64
+    - 0.33.1-amd64
     username:
       from_secret: docker_username
   when:
@@ -54,7 +54,7 @@ steps:
       from_secret: docker_password
     repo: grafana/loki-build-image
     tags:
-    - 0.33.0-arm64
+    - 0.33.1-arm64
     username:
       from_secret: docker_username
   when:
@@ -86,7 +86,7 @@ steps:
     password:
       from_secret: docker_password
     spec: .drone/docker-manifest-build-image.tmpl
-    target: loki-build-image:0.33.0
+    target: loki-build-image:0.33.1
     username:
       from_secret: docker_username
   when:
@@ -1362,6 +1362,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: 32b44aecaad0258ed9494225595e1016a56bea960bcd0b15b2db3449bed957e0
+hmac: 452ccacf982174ae96c64fc2d0868859d26dbf6944e49d9170d780ff40a81eb8
 
 ...

CHANGELOG.md

@@ -88,6 +88,7 @@
 * [11121](https://github.com/grafana/loki/pull/11121) **periklis** Ensure all lifecycler cfgs ref a valid IPv6 addr and port combination
 * [10650](https://github.com/grafana/loki/pull/10650) **matthewpi** Ensure the frontend uses a valid IPv6 addr and port combination
 * [11665](https://github.com/grafana/loki/pull/11665) **salvacorts** Deprecate and flip `-legacy-read-mode` flag to `false` by default.
+* [12448](https://github.com/grafana/loki/pull/12448/files) **slim-bean** BREAKING CHANGE: refactor how we do defaults for runtime overrides
 
 #### Promtail
 

Makefile

@@ -803,6 +803,8 @@ check-format: format
 doc: ## Generates the config file documentation
 	go run ./tools/doc-generator $(DOC_FLAGS_TEMPLATE) > $(DOC_FLAGS)
 
+docs: doc
+
 check-doc: ## Check the documentation files are up to date
 check-doc: doc
 	@find . -name "*.md" | xargs git diff --exit-code -- \

cmd/loki/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/prometheus/common/version"
 
 	"github.com/grafana/loki/v3/pkg/loki"
+	loki_runtime "github.com/grafana/loki/v3/pkg/runtime"
 	"github.com/grafana/loki/v3/pkg/util"
 	_ "github.com/grafana/loki/v3/pkg/util/build"
 	"github.com/grafana/loki/v3/pkg/util/cfg"
@@ -49,6 +50,7 @@ func main() {
 	// call it atleast once, the defaults are set to an empty struct.
 	// We call it with the flag values so that the config file unmarshalling only overrides the values set in the config.
 	validation.SetDefaultLimitsForYAMLUnmarshalling(config.LimitsConfig)
+	loki_runtime.SetDefaultLimitsForYAMLUnmarshalling(config.OperationalConfig)
 
 	// Init the logger which will honor the log level set in config.Server
 	if reflect.DeepEqual(&config.Server.LogLevel, &log.Level{}) {

docs/sources/configure/_index.md

@@ -186,7 +186,9 @@ Pass the `-config.expand-env` flag at the command line to enable this way of set
 # shards for performance.
 [compactor: <compactor>]
 
-# The limits_config block configures global and per-tenant limits in Loki.
+# The limits_config block configures global and per-tenant limits in Loki. The
+# values here can be overridden in the `overrides` section of the runtime_config
+# file
 [limits_config: <limits_config>]
 
 # The frontend_worker configures the worker - running within the Loki querier -
@@ -208,6 +210,11 @@ Pass the `-config.expand-env` flag at the command line to enable this way of set
 # configuration file.
 [runtime_config: <runtime_config>]
 
+# These are values which allow you to control aspects of Loki's operation, most
+# commonly used for controlling types of higher verbosity logging, the values
+# here can be overridden in the `configs` section of the `runtime_config` file.
+[operational_config: <operational_config>]
+
 # Configuration for tracing.
 [tracing: <tracing>]
 
@@ -2747,7 +2754,7 @@ retention:
 
 ### limits_config
 
-The `limits_config` block configures global and per-tenant limits in Loki.
+The `limits_config` block configures global and per-tenant limits in Loki. The values here can be overridden in the `overrides` section of the runtime_config file
 
 ```yaml
 # Whether the ingestion rate limit should be applied individually to each
@@ -3675,6 +3682,32 @@ Configuration for 'runtime config' module, responsible for reloading runtime con
 [file: <string> | default = ""]
 ```
 
+### operational_config
+
+These are values which allow you to control aspects of Loki's operation, most commonly used for controlling types of higher verbosity logging, the values here can be overridden in the `configs` section of the `runtime_config` file.
+
+```yaml
+# Log every new stream created by a push request (very verbose, recommend to
+# enable via runtime config only).
+# CLI flag: -operation-config.log-stream-creation
+[log_stream_creation: <boolean> | default = false]
+
+# Log every push request (very verbose, recommend to enable via runtime config
+# only).
+# CLI flag: -operation-config.log-push-request
+[log_push_request: <boolean> | default = false]
+
+# Log every stream in a push request (very verbose, recommend to enable via
+# runtime config only).
+# CLI flag: -operation-config.log-push-request-streams
+[log_push_request_streams: <boolean> | default = false]
+
+# Log push errors with a rate limited logger, will show client push errors
+# without overly spamming logs.
+# CLI flag: -operation-config.limited-log-push-errors
+[limited_log_push_errors: <boolean> | default = true]
+```
+
 ### tracing
 
 Configuration for `tracing`.

docs/sources/operations/storage/schema/_index.md

@@ -14,6 +14,32 @@ Loki uses the defined schemas to determine which format to use when storing and
 
 Use of a schema allows Loki to iterate over the storage layer without requiring migration of existing data.
 
+## New Loki installs
+For a new Loki install with no previous data, here is an example schema configuration with recommended values
+
+```
+schema_config:
+  configs:
+    - from: 2024-04-01
+      object_store: s3
+      store: tsdb
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+```
+
+
+| Property     | Description                                                                                                                                            |
+|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
+| from         | for a new install, this must be a date in the past, use a recent date. Format is YYYY-MM-DD.                                                           |
+| object_store | s3, azure, gcs, alibabacloud, bos, cos, swift, filesystem, or a named_store (see [StorageConfig]({{< relref "../../../configure#storage_config" >}})). |
+| store        | `tsdb` is the current and only recommended value for store.                                                                                            |
+| schema       | `v13` is the most recent schema and recommended value.                                                                                                 |
+| prefix:      | any value without spaces is acceptable.                                                                                                                |
+| period:      | must be `24h`.                                                                                                                                         |
+
+
 ## Changing the schema
 
 Here are items to consider when changing the schema; if schema changes are not done properly, a scenario can be created which prevents data from being read.
@@ -33,19 +59,19 @@ Here are items to consider when changing the schema; if schema changes are not d
 
 ```
 schema_config:
-    configs:
-        - from: "2020-07-31"
-          index:
-            period: 24h
-            prefix: loki_ops_index_
-          object_store: gcs
-          schema: v11
-          store: tsdb
-        - from: "2022-01-20"
-          index:
-            period: 24h
-            prefix: loki_ops_index_
-          object_store: gcs
-          schema: v13
-          store: tsdb
+  configs:
+    - from: "2020-07-31"
+      index:
+        period: 24h
+        prefix: loki_ops_index_
+      object_store: gcs
+      schema: v11
+      store: tsdb
+    - from: "2022-01-20"
+      index:
+        period: 24h
+        prefix: loki_ops_index_
+      object_store: gcs
+      schema: v13
+      store: tsdb
 ```

docs/sources/setup/upgrade/_index.md

@@ -38,6 +38,12 @@ The output is incredibly verbose as it shows the entire internal config struct u
 
 ### Loki
 
+#### Removed the `default` section of the runtime overrides config file.
+
+This was introduced in 2.9 and likely not widely used.  This only affects you if you run Loki with a runtime config file AND you had populated the new `default` block added in 2.9.
+
+The `default` block was removed and instead a top level config now exists in the standard Loki config called `operational_config`, you can set default values here for runtime configs.
+
 #### Removed `shared_store` and `shared_store_key_prefix` from shipper configuration
 
 The following CLI flags and the corresponding YAML settings to configure shared store for TSDB and BoltDB shippers are now removed:
@@ -168,6 +174,10 @@ This new metric will provide a more clear signal that there is an issue with ing
 | `legacy-read-mode`                                     | false       | true        | Deprecated. It will be removed in the next minor release. |
 {{% /responsive-table %}}
 
+#### Automatic stream sharding is enabled by default
+
+Automatic stream sharding helps keep the write load of high volume streams balanced across ingesters and helps to avoid hot-spotting. Check out the [operations page](https://grafana.com/docs/loki/latest/operations/automatic-stream-sharding/) for more information
+
 #### Write dedupe cache is deprecated
 Write dedupe cache is deprecated because it not required by the newer single store indexes ([TSDB]({{< relref "../../operations/storage/tsdb" >}}) and [boltdb-shipper]({{< relref "../../operations/storage/boltdb-shipper" >}})).
 If you using a [legacy index type]({{< relref "../../storage#index-storage" >}}), consider migrating to TSDB (recommended).

loki-build-image/Dockerfile

@@ -2,11 +2,11 @@
 # pipelines.
 # If you make changes to this Dockerfile you also need to update the
 # tag of the Docker image in `../.drone/drone.jsonnet` and run `make drone`.
-# See ../docs/sources/maintaining/release-loki-build-image.md for instructions
+# See ../docs/sources/community/maintaining/release-loki-build-image.md for instructions
 # on how to publish a new build image.
 
 # Install helm (https://helm.sh/) and helm-docs (https://github.com/norwoodj/helm-docs) for generating Helm Chart reference.
-FROM golang:1.21.3-bullseye as helm
+FROM golang:1.21.9-bullseye as helm
 ARG TARGETARCH
 ARG HELM_VER="v3.2.3"
 RUN curl -L "https://get.helm.sh/helm-${HELM_VER}-linux-$TARGETARCH.tar.gz" | tar zx && \
@@ -15,7 +15,7 @@ RUN BIN=$([ "$TARGETARCH" = "arm64" ] && echo "helm-docs_Linux_arm64" || echo "h
     curl -L "https://github.com/norwoodj/helm-docs/releases/download/v1.11.2/$BIN.tar.gz" | tar zx && \
     install -t /usr/local/bin helm-docs
 
-FROM alpine:3.18.5 as lychee
+FROM alpine:3.18.6 as lychee
 ARG TARGETARCH
 ARG LYCHEE_VER="0.7.0"
 RUN apk add --no-cache curl && \
@@ -24,21 +24,21 @@ RUN apk add --no-cache curl && \
     mv /tmp/lychee /usr/bin/lychee && \
     rm -rf "/tmp/linux-$TARGETARCH" /tmp/lychee-$LYCHEE_VER.tgz
 
-FROM alpine:3.18.5 as golangci
+FROM alpine:3.18.6 as golangci
 RUN apk add --no-cache curl && \
     cd / && \
     curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.55.1
 
-FROM alpine:3.18.5 as buf
+FROM alpine:3.18.6 as buf
 ARG TARGETOS
 RUN apk add --no-cache curl && \
     curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.4.0/buf-$TARGETOS-$(uname -m)" -o "/usr/bin/buf" && \
     chmod +x "/usr/bin/buf"
 
-FROM alpine:3.18.5 as docker
+FROM alpine:3.18.6 as docker
 RUN apk add --no-cache docker-cli docker-cli-buildx
 
-FROM golang:1.21.3-bullseye as drone
+FROM golang:1.21.9-bullseye as drone
 ARG TARGETARCH
 RUN curl -L "https://github.com/drone/drone-cli/releases/download/v1.7.0/drone_linux_$TARGETARCH".tar.gz | tar zx && \
     install -t /usr/local/bin drone
@@ -48,35 +48,35 @@ RUN curl -L "https://github.com/drone/drone-cli/releases/download/v1.7.0/drone_l
 # Error:
 # github.com/fatih/[email protected] requires golang.org/x/[email protected]
 #   (not golang.org/x/[email protected] from golang.org/x/tools/cmd/goyacc@58d531046acdc757f177387bc1725bfa79895d69)
-FROM golang:1.21.3-bullseye as faillint
+FROM golang:1.21.9-bullseye as faillint
 RUN GO111MODULE=on go install github.com/fatih/[email protected]
 RUN GO111MODULE=on go install golang.org/x/tools/cmd/[email protected]
 
-FROM golang:1.21.3-bullseye as delve
+FROM golang:1.21.9-bullseye as delve
 RUN GO111MODULE=on go install github.com/go-delve/delve/cmd/dlv@latest
 
 # Install ghr used to push binaries and template the release
 # This collides with the version of go tools used in the base image, thus we install it in its own image and copy it over.
-FROM golang:1.21.3-bullseye as ghr
+FROM golang:1.21.9-bullseye as ghr
 RUN GO111MODULE=on go install github.com/tcnksm/ghr@9349474
 
 # Install nfpm (https://nfpm.goreleaser.com) for creating .deb and .rpm packages.
-FROM golang:1.21.3-bullseye as nfpm
+FROM golang:1.21.9-bullseye as nfpm
 RUN GO111MODULE=on go install github.com/goreleaser/nfpm/v2/cmd/[email protected]
 
 # Install gotestsum
-FROM golang:1.21.3-bullseye as gotestsum
+FROM golang:1.21.9-bullseye as gotestsum
 RUN GO111MODULE=on go install gotest.tools/[email protected]
 
 # Install tools used to compile jsonnet.
-FROM golang:1.21.3-bullseye as jsonnet
+FROM golang:1.21.9-bullseye as jsonnet
 RUN GO111MODULE=on go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
 RUN GO111MODULE=on go install github.com/monitoring-mixins/mixtool/cmd/mixtool@bca3066
 RUN GO111MODULE=on go install github.com/google/go-jsonnet/cmd/[email protected]
 
 FROM aquasec/trivy as trivy
 
-FROM golang:1.21.3-bullseye
+FROM golang:1.21.9-bullseye
 RUN apt-get update && \
     apt-get install -qy \
     musl gnupg ragel \

pkg/bloomgateway/client.go

@@ -144,11 +144,12 @@ type Client interface {
 }
 
 type GatewayClient struct {
-	cfg    ClientConfig
-	limits Limits
-	logger log.Logger
-	pool   *ringclient.Pool
-	ring   ring.ReadRing
+	cfg     ClientConfig
+	limits  Limits
+	logger  log.Logger
+	metrics *clientMetrics
+	pool    *ringclient.Pool
+	ring    ring.ReadRing
 }
 
 func NewClient(
@@ -206,11 +207,12 @@ func NewClient(
 	}
 
 	return &GatewayClient{
-		cfg:    cfg,
-		logger: logger,
-		limits: limits,
-		pool:   clientpool.NewPool("bloom-gateway", cfg.PoolConfig, cfg.Ring, ringclient.PoolAddrFunc(poolFactory), logger, metricsNamespace),
-		ring:   readRing,
+		cfg:     cfg,
+		logger:  logger,
+		limits:  limits,
+		metrics: newClientMetrics(registerer),
+		pool:    clientpool.NewPool("bloom-gateway", cfg.PoolConfig, cfg.Ring, ringclient.PoolAddrFunc(poolFactory), logger, metricsNamespace),
+		ring:    readRing,
 	}, nil
 }
 
@@ -231,7 +233,7 @@ func shuffleAddrs(addrs []string) []string {
 
 // FilterChunkRefs implements Client
 func (c *GatewayClient) FilterChunks(ctx context.Context, tenant string, from, through model.Time, groups []*logproto.GroupedChunkRefs, plan plan.QueryPlan) ([]*logproto.GroupedChunkRefs, error) {
-	if !c.limits.BloomGatewayEnabled(tenant) {
+	if !c.limits.BloomGatewayEnabled(tenant) || len(groups) == 0 {
 		return groups, nil
 	}
 
@@ -246,7 +248,20 @@ func (c *GatewayClient) FilterChunks(ctx context.Context, tenant string, from, t
 	if err != nil {
 		return nil, errors.Wrap(err, "bloom gateway get replication sets")
 	}
+
 	servers = partitionByReplicationSet(groups, servers)
+	if len(servers) > 0 {
+		// cache locality score (higher is better):
+		// `% keyspace / % instances`. Ideally converges to 1 (querying x% of keyspace requires x% of instances),
+		// but can be less if the keyspace is not evenly distributed across instances. Ideal operation will see the range of
+		// `1-2/num_instances` -> `1`, where the former represents slight
+		// overlap on instances to the left and right of the range.
+		firstFp, lastFp := groups[0].Fingerprint, groups[len(groups)-1].Fingerprint
+		pctKeyspace := float64(lastFp-firstFp) / float64(math.MaxUint64)
+		pctInstances := float64(len(servers)) / float64(len(rs.Instances))
+		cacheLocalityScore := pctKeyspace / pctInstances
+		c.metrics.cacheLocalityScore.Observe(cacheLocalityScore)
+	}
 
 	results := make([][]*logproto.GroupedChunkRefs, len(servers))
 	count := 0