Merge branch 'main' of github.com:grafana/loki into dannykopping/max-…

…ingester-splits

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "image": "grafana/loki-build-image:0.29.0",
+  "image": "grafana/loki-build-image:0.33.0",
   "containerEnv": {
     "BUILD_IN_CONTAINER": "false"
   },

.drone/drone.jsonnet

@@ -496,7 +496,7 @@ local manifest_ecr(apps, archs) = pipeline('manifest-ecr') {
   ],
 };
 
-local build_image_tag = '0.32.0';
+local build_image_tag = '0.33.0';
 [
   pipeline('loki-build-image-' + arch) {
     workspace: {
@@ -640,6 +640,7 @@ local build_image_tag = '0.32.0';
         'GIT_TARGET_BRANCH="$DRONE_TARGET_BRANCH"',
       ]) { depends_on: ['loki'], when: onPRs },
       make('validate-example-configs', container=false) { depends_on: ['loki'] },
+      make('validate-dev-cluster-config', container=false) { depends_on: ['loki'] },
       make('check-example-config-doc', container=false) { depends_on: ['clone'] },
       {
         name: 'build-docs-website',

.drone/drone.yml

@@ -15,7 +15,7 @@ steps:
     dry_run: true
     repo: grafana/loki-build-image
     tags:
-    - 0.32.0-amd64
+    - 0.33.0-amd64
   when:
     event:
     - pull_request
@@ -33,7 +33,7 @@ steps:
       from_secret: docker_password
     repo: grafana/loki-build-image
     tags:
-    - 0.32.0-amd64
+    - 0.33.0-amd64
     username:
       from_secret: docker_username
   when:
@@ -68,7 +68,7 @@ steps:
     dry_run: true
     repo: grafana/loki-build-image
     tags:
-    - 0.32.0-arm64
+    - 0.33.0-arm64
   when:
     event:
     - pull_request
@@ -86,7 +86,7 @@ steps:
       from_secret: docker_password
     repo: grafana/loki-build-image
     tags:
-    - 0.32.0-arm64
+    - 0.33.0-arm64
     username:
       from_secret: docker_username
   when:
@@ -118,7 +118,7 @@ steps:
     password:
       from_secret: docker_password
     spec: .drone/docker-manifest-build-image.tmpl
-    target: loki-build-image:0.32.0
+    target: loki-build-image:0.33.0
     username:
       from_secret: docker_username
   when:
@@ -182,14 +182,14 @@ steps:
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: check-drone-drift
 - commands:
   - make BUILD_IN_CONTAINER=false check-generated-files
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: check-generated-files
 - commands:
   - cd ..
@@ -199,7 +199,7 @@ steps:
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: clone-target-branch
   when:
     event:
@@ -210,14 +210,14 @@ steps:
   - clone-target-branch
   - check-generated-files
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: test
 - commands:
   - cd ../loki-target-branch && BUILD_IN_CONTAINER=false make test
   depends_on:
   - clone-target-branch
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: test-target-branch
   when:
     event:
@@ -230,7 +230,7 @@ steps:
   - test
   - test-target-branch
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: compare-coverage
   when:
     event:
@@ -248,7 +248,7 @@ steps:
     TOKEN:
       from_secret: github_token
     USER: grafanabot
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: report-coverage
   when:
     event:
@@ -258,15 +258,15 @@ steps:
   depends_on:
   - check-generated-files
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: lint
 - commands:
   - make BUILD_IN_CONTAINER=false check-mod
   depends_on:
   - test
   - lint
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: check-mod
 - commands:
   - apk add make bash && make lint-scripts
@@ -277,21 +277,21 @@ steps:
   depends_on:
   - check-generated-files
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: loki
 - commands:
   - make BUILD_IN_CONTAINER=false check-doc
   depends_on:
   - loki
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: check-doc
 - commands:
   - make BUILD_IN_CONTAINER=false check-format GIT_TARGET_BRANCH="$DRONE_TARGET_BRANCH"
   depends_on:
   - loki
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: check-format
   when:
     event:
@@ -301,14 +301,21 @@ steps:
   depends_on:
   - loki
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: validate-example-configs
+- commands:
+  - make BUILD_IN_CONTAINER=false validate-dev-cluster-config
+  depends_on:
+  - loki
+  environment: {}
+  image: grafana/loki-build-image:0.33.0
+  name: validate-dev-cluster-config
 - commands:
   - make BUILD_IN_CONTAINER=false check-example-config-doc
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: check-example-config-doc
 - commands:
   - mkdir -p /hugo/content/docs/loki/latest
@@ -341,7 +348,7 @@ steps:
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: loki-mixin-check
   when:
     event:
@@ -366,7 +373,7 @@ steps:
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: documentation-helm-reference-check
 trigger:
   ref:
@@ -1772,15 +1779,15 @@ steps:
     NFPM_SIGNING_KEY:
       from_secret: gpg_private_key
     NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: write-key
 - commands:
   - make BUILD_IN_CONTAINER=false packages
   environment:
     NFPM_PASSPHRASE:
       from_secret: gpg_passphrase
     NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: test packaging
 - commands:
   - ./tools/packaging/verify-deb-install.sh
@@ -1806,7 +1813,7 @@ steps:
     NFPM_PASSPHRASE:
       from_secret: gpg_passphrase
     NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: publish
   when:
     event:
@@ -1841,7 +1848,7 @@ steps:
       from_secret: docker_password
     DOCKER_USERNAME:
       from_secret: docker_username
-  image: grafana/loki-build-image:0.31.2
+  image: grafana/loki-build-image:0.33.0
   name: build and push
   privileged: true
   volumes:
@@ -2106,6 +2113,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: 8ae9cff1a379503d0b568f727d9c12bcb486a5e8d1fc3271deea32f07939baf1
+hmac: fe7669a21410ae5f2d1ad6b6205fdc582af874f65f7bd6a679731a88174e3a1c
 
 ...

.github/workflows/checks.yml

@@ -6,7 +6,7 @@ jobs:
     env:
       BUILD_IN_CONTAINER: false 
     container:
-      image: grafana/loki-build-image:0.32.0
+      image: grafana/loki-build-image:0.33.0
     steps:
       - uses: actions/checkout@v4
       - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

CHANGELOG.md

@@ -47,12 +47,15 @@
 * [11545](https://github.com/grafana/loki/pull/11545) **dannykopping** Force correct memcached timeout when fetching chunks.
 * [11589](https://github.com/grafana/loki/pull/11589) **ashwanthgoli** Results Cache: Adds `query_length_served` cache stat to measure the length of the query served from cache.
 * [11535](https://github.com/grafana/loki/pull/11535) **dannykopping** Query Frontend: Allow customisable splitting of queries which overlap the `query_ingester_within` window to reduce query pressure on ingesters.
+* [11654](https://github.com/grafana/loki/pull/11654) **dannykopping** Cache: atomically check background cache size limit correctly. 
 
 ##### Fixes
 * [11074](https://github.com/grafana/loki/pull/11074) **hainenber** Fix panic in lambda-promtail due to mishandling of empty DROP_LABELS env var.
 * [11195](https://github.com/grafana/loki/pull/11195) **canuteson** Generate tsdb_shipper storage_config even if using_boltdb_shipper is false
+* [9831](https://github.com/grafana/loki/pull/9831) **sijmenhuizenga**: Fix Promtail excludepath not evaluated on newly added files.
 * [11551](https://github.com/grafana/loki/pull/11551) **dannykopping** Do not reflect label names in request metrics' "route" label.
 * [11601](https://github.com/grafana/loki/pull/11601) **dannykopping** Ruler: Fixed a panic that can be caused by concurrent read-write access of tenant configs when there are a large amount of rules.
+* [11606](https://github.com/grafana/loki/pull/11606) **dannykopping** Fixed regression adding newlines to HTTP error response bodies which may break client integrations.
 
 ##### Changes
 
@@ -78,6 +81,7 @@
 
 * [10677](https://github.com/grafana/loki/pull/10677) **chaudum** Remove deprecated `stream_lag_labels` setting from both the `options` and `client` configuration sections.
 * [10689](https://github.com/grafana/loki/pull/10689) **dylanguedes**: Ingester: Make jitter to be 20% of flush check period instead of 1%.
+* [11420](https://github.com/grafana/loki/pull/11420) **zry98**: Show a clearer reason in "disable watchConfig" log message when server is disabled.
 
 ##### Fixes
 

Makefile

@@ -37,7 +37,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
 BUILD_IN_CONTAINER ?= true
 
 # ensure you run `make drone` after changing this
-BUILD_IMAGE_VERSION ?= 0.31.2
+BUILD_IMAGE_VERSION ?= 0.33.0
 
 # Docker image info
 IMAGE_PREFIX ?= grafana
@@ -801,6 +801,9 @@ EXAMPLES_SKIP_VALIDATION_FLAG := "doc-example:skip-validation=true"
 validate-example-configs: loki
 	for f in $$(grep -rL $(EXAMPLES_SKIP_VALIDATION_FLAG) $(EXAMPLES_YAML_PATH)/*.yaml); do echo "Validating provided example config: $$f" && ./cmd/loki/loki -config.file=$$f -verify-config || exit 1; done
 
+validate-dev-cluster-config: loki
+	./cmd/loki/loki -config.file=./tools/dev/loki-boltdb-storage-s3/config/loki.yaml -verify-config
+
 # Dynamically generate ./docs/sources/configure/examples.md using the example configs that we provide.
 # This target should be run if any of our example configs change.
 generate-example-config-doc:
@@ -836,14 +839,16 @@ dev-k3d-down:
 
 # Trivy is used to scan images for vulnerabilities
 .PHONY: trivy
-trivy: loki-image
+trivy: loki-image build-image
 	trivy i $(IMAGE_PREFIX)/loki:$(IMAGE_TAG)
+	trivy i $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG)
 	trivy fs go.mod
 
 # Synk is also used to scan for vulnerabilities, and detects things that trivy might miss
 .PHONY: snyk
-snyk: loki-image
-	snyk container test $(IMAGE_PREFIX)/loki:$(IMAGE_TAG)
+snyk: loki-image build-image
+	snyk container test $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) --file=cmd/loki/Dockerfile
+	snyk container test $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG) --file=loki-build-image/Dockerfile
 	snyk code test
 
 .PHONY: scan-vulnerabilities

clients/cmd/docker-driver/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.31.2
+ARG BUILD_IMAGE=grafana/loki-build-image:0.33.0
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intended to be called from the root like so:
 # docker build -t grafana/loki -f cmd/loki/Dockerfile .
@@ -9,7 +9,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false clients/cmd/docker-driver/docker-driver
 
-FROM alpine:3.18.4
+FROM alpine:3.18.5
 RUN apk add --update --no-cache ca-certificates tzdata
 COPY --from=build /src/loki/clients/cmd/docker-driver/docker-driver /bin/docker-driver
 WORKDIR /bin/

clients/cmd/docker-driver/README.md

@@ -23,4 +23,4 @@ To build and contribute. you will need:
 To build the driver you can use `make docker-driver`, then you can install this driver using `make docker-driver-enable`.
 If you want to uninstall the driver simply run `make docker-driver-clean`.
 
-Make you update the [documentation](../../docs/sources/clients/docker-driver/) accordingly when submitting a new change.
+Make sure you update the [documentation](../../docs/sources/send-data/docker-driver/) accordingly when submitting a new change.

clients/cmd/promtail/Dockerfile.cross

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.29.3
+ARG BUILD_IMAGE=grafana/loki-build-image:0.33.0
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intended to be called from the root like so:
 # docker build -t grafana/promtail -f clients/cmd/promtail/Dockerfile .

clients/cmd/promtail/Dockerfile.debug

@@ -2,14 +2,14 @@
 # This file is intended to be called from the root like so:
 # docker build -t grafana/promtail -f clients/cmd/promtail/Dockerfile.debug .
 
-FROM grafana/loki-build-image:0.29.3 as build
+FROM grafana/loki-build-image:0.33.0 as build
 ARG GOARCH="amd64"
 COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail-debug
 
 
-FROM       alpine:3.18.4
+FROM       alpine:3.18.5
 RUN        apk add --update --no-cache ca-certificates tzdata
 COPY       --from=build /src/loki/clients/cmd/promtail/promtail-debug /usr/bin/promtail-debug
 COPY       --from=build /usr/bin/dlv /usr/bin/dlv