diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index 49f67f06861a..9d0589fe22a8 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -177,16 +177,6 @@ local promtail_win() = pipeline('promtail-windows') { local querytee() = pipeline('querytee-amd64') + arch_image('amd64', 'main') { steps+: [ - // dry run for everything that is not tag or main - docker('amd64', 'querytee') { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - repo: 'grafana/loki-query-tee', - }, - }, - ] + [ // publish for tag or main docker('amd64', 'querytee') { depends_on: ['image-tag'], @@ -196,21 +186,10 @@ local querytee() = pipeline('querytee-amd64') + arch_image('amd64', 'main') { }, }, ], - depends_on: ['check'], }; local fluentbit(arch) = pipeline('fluent-bit-' + arch) + arch_image(arch) { steps+: [ - // dry run for everything that is not tag or main - clients_docker(arch, 'fluent-bit') { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - repo: 'grafana/fluent-bit-plugin-loki', - }, - }, - ] + [ // publish for tag or main clients_docker(arch, 'fluent-bit') { depends_on: ['image-tag'], @@ -220,21 +199,10 @@ local fluentbit(arch) = pipeline('fluent-bit-' + arch) + arch_image(arch) { }, }, ], - depends_on: ['check'], }; local fluentd() = pipeline('fluentd-amd64') + arch_image('amd64', 'main') { steps+: [ - // dry run for everything that is not tag or main - clients_docker('amd64', 'fluentd') { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - repo: 'grafana/fluent-plugin-loki', - }, - }, - ] + [ // publish for tag or main clients_docker('amd64', 'fluentd') { depends_on: ['image-tag'], @@ -244,21 +212,10 @@ local fluentd() = pipeline('fluentd-amd64') + arch_image('amd64', 'main') { }, }, ], - depends_on: ['check'], }; local logstash() = pipeline('logstash-amd64') + arch_image('amd64', 'main') { steps+: [ - // dry run for everything that is not tag or main - clients_docker('amd64', 'logstash') { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - repo: 'grafana/logstash-output-loki', - }, - }, - ] + [ // publish for tag or main clients_docker('amd64', 'logstash') { depends_on: ['image-tag'], @@ -268,20 +225,10 @@ local logstash() = pipeline('logstash-amd64') + arch_image('amd64', 'main') { }, }, ], - depends_on: ['check'], }; local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) { steps+: [ - // dry run for everything that is not tag or main - clients_docker(arch, 'promtail') { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - }, - }, - ] + [ // publish for tag or main clients_docker(arch, 'promtail') { depends_on: ['image-tag'], @@ -289,7 +236,6 @@ local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) { settings+: {}, }, ], - depends_on: ['check'], }; local lambda_promtail(arch) = pipeline('lambda-promtail-' + arch) + arch_image(arch) { @@ -297,15 +243,6 @@ local lambda_promtail(arch) = pipeline('lambda-promtail-' + arch) + arch_image(a steps+: [ skipStep, - // dry run for everything that is not tag or main - lambda_promtail_ecr('lambda-promtail') { - depends_on: ['image-tag', skipStep.name], - when: onPRs, - settings+: { - dry_run: true, - }, - }, - ] + [ // publish for tag or main lambda_promtail_ecr('lambda-promtail') { depends_on: ['image-tag'], @@ -313,20 +250,10 @@ local lambda_promtail(arch) = pipeline('lambda-promtail-' + arch) + arch_image(a settings+: {}, }, ], - depends_on: ['check'], }; local lokioperator(arch) = pipeline('lokioperator-' + arch) + arch_image(arch) { steps+: [ - // dry run for everything that is not tag or main - docker_operator(arch, 'loki-operator') { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - }, - }, - ] + [ // publish for tag or main docker_operator(arch, 'loki-operator') { depends_on: ['image-tag'], @@ -336,21 +263,10 @@ local lokioperator(arch) = pipeline('lokioperator-' + arch) + arch_image(arch) { settings+: {}, }, ], - depends_on: ['check'], }; local logql_analyzer() = pipeline('logql-analyzer') + arch_image('amd64') { steps+: [ - // dry run for everything that is not tag or main - docker('amd64', 'logql-analyzer') { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - repo: 'grafana/logql-analyzer', - }, - }, - ] + [ // publish for tag or main docker('amd64', 'logql-analyzer') { depends_on: ['image-tag'], @@ -360,21 +276,10 @@ local logql_analyzer() = pipeline('logql-analyzer') + arch_image('amd64') { }, }, ], - depends_on: ['check'], }; local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) { steps+: [ - // dry run for everything that is not tag or main - docker(arch, app) { - depends_on: ['image-tag'], - when: onPRs, - settings+: { - dry_run: true, - }, - } - for app in apps - ] + [ // publish for tag or main docker(arch, app) { depends_on: ['image-tag'], @@ -383,7 +288,6 @@ local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) { } for app in apps ], - depends_on: ['check'], }; local manifest(apps) = pipeline('manifest') { @@ -508,21 +412,6 @@ local build_image_tag = '0.33.0'; arch: arch, }, steps: [ - { - name: 'test', - image: 'plugins/docker', - when: onPRs + onPath('loki-build-image/**'), - environment: { - DOCKER_BUILDKIT: 1, - }, - settings: { - repo: 'grafana/loki-build-image', - context: 'loki-build-image', - dockerfile: 'loki-build-image/Dockerfile', - tags: [build_image_tag + '-' + arch], - dry_run: true, - }, - }, { name: 'push', image: 'plugins/docker', @@ -571,16 +460,6 @@ local build_image_tag = '0.33.0'; path: 'loki', }, steps: [ - { - name: 'test-image', - image: 'plugins/docker', - when: onPRs + onPath('production/helm/loki/src/helm-test/**'), - settings: { - repo: 'grafana/loki-helm-test', - dockerfile: 'production/helm/loki/src/helm-test/Dockerfile', - dry_run: true, - }, - }, { name: 'push-image', image: 'plugins/docker', @@ -595,64 +474,6 @@ local build_image_tag = '0.33.0'; }, ], }, - pipeline('check') { - workspace: { - base: '/src', - path: 'loki', - }, - steps: [ - make('check-drone-drift', container=false) { depends_on: ['clone'] }, - make('check-generated-files', container=false) { depends_on: ['clone'] }, - run('clone-target-branch', commands=[ - 'cd ..', - 'echo "cloning "$DRONE_TARGET_BRANCH ', - 'git clone -b $DRONE_TARGET_BRANCH $CI_REPO_REMOTE loki-target-branch', - 'cd -', - ]) { depends_on: ['clone'], when: onPRs }, - make('test', container=false) { depends_on: ['clone-target-branch', 'check-generated-files'] }, - run('test-target-branch', commands=['cd ../loki-target-branch && BUILD_IN_CONTAINER=false make test']) { depends_on: ['clone-target-branch'], when: onPRs }, - make('compare-coverage', container=false, args=[ - 'old=../loki-target-branch/test_results.txt', - 'new=test_results.txt', - 'packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki', - '> diff.txt', - ]) { depends_on: ['test', 'test-target-branch'], when: onPRs }, - run('report-coverage', commands=[ - "total_diff=$(sed 's/%//' diff.txt | awk '{sum+=$3;}END{print sum;}')", - 'if [ $total_diff = 0 ]; then exit 0; fi', - "pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')", - "body=$(jq -Rs '{body: . }' diff.txt)", - 'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments -d "$body" > /dev/null', - ], env={ - USER: 'grafanabot', - TOKEN: { from_secret: github_secret.name }, - }) { depends_on: ['compare-coverage'], when: onPRs }, - make('lint', container=false) { depends_on: ['check-generated-files'] }, - make('check-mod', container=false) { depends_on: ['test', 'lint'] }, - { - name: 'shellcheck', - image: 'koalaman/shellcheck-alpine:stable', - commands: ['apk add make bash && make lint-scripts'], - }, - make('loki', container=false) { depends_on: ['check-generated-files'] }, - make('check-doc', container=false) { depends_on: ['loki'] }, - make('check-format', container=false, args=[ - 'GIT_TARGET_BRANCH="$DRONE_TARGET_BRANCH"', - ]) { depends_on: ['loki'], when: onPRs }, - make('validate-example-configs', container=false) { depends_on: ['loki'] }, - make('validate-dev-cluster-config', container=false) { depends_on: ['validate-example-configs'] }, - make('check-example-config-doc', container=false) { depends_on: ['clone'] }, - { - name: 'build-docs-website', - image: 'grafana/docs-base:e6ef023f8b8', - commands: [ - 'mkdir -p /hugo/content/docs/loki/latest', - 'cp -r docs/sources/* /hugo/content/docs/loki/latest/', - 'cd /hugo && make prod', - ], - }, - ], - }, pipeline('mixins') { workspace: { base: '/src', @@ -788,7 +609,7 @@ local build_image_tag = '0.33.0'; depends_on: ['manifest'], image_pull_secrets: [pull_secret.name], trigger: { - // wee need to run it only on Loki tags that starts with `v`. + // we need to run it only on Loki tags that starts with `v`. ref: ['refs/tags/v*'], }, steps: [ @@ -835,109 +656,6 @@ local build_image_tag = '0.33.0'; }, promtail_win(), logql_analyzer(), - pipeline('release') { - trigger+: { - event: ['pull_request', 'tag'], - }, - depends_on+: ['check'], - image_pull_secrets: [pull_secret.name], - volumes+: [ - { - name: 'cgroup', - host: { - path: '/sys/fs/cgroup', - }, - }, - { - name: 'docker', - host: { - path: '/var/run/docker.sock', - }, - }, - ], - // Launch docker images with systemd - services: [ - { - name: 'systemd-debian', - image: 'jrei/systemd-debian:12', - volumes: [ - { - name: 'cgroup', - path: '/sys/fs/cgroup', - }, - ], - privileged: true, - }, - { - name: 'systemd-centos', - image: 'jrei/systemd-centos:8', - volumes: [ - { - name: 'cgroup', - path: '/sys/fs/cgroup', - }, - ], - privileged: true, - }, - ], - // Package and test the packages - steps: [ - skipMissingSecretPipelineStep(gpg_private_key.name), // Needs GPG keys to run - { - name: 'fetch-tags', - image: 'alpine', - commands: [ - 'apk add --no-cache bash git', - 'git fetch origin --tags', - ], - }, - run('write-key', - commands=['printf "%s" "$NFPM_SIGNING_KEY" > $NFPM_SIGNING_KEY_FILE'], - env={ - NFPM_SIGNING_KEY: { from_secret: gpg_private_key.name }, - NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key', - }), - run('test packaging', - commands=[ - 'make BUILD_IN_CONTAINER=false packages', - ], - env={ - NFPM_PASSPHRASE: { from_secret: gpg_passphrase.name }, - NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key', - }), - { - name: 'test deb package', - image: 'docker', - commands: ['./tools/packaging/verify-deb-install.sh'], - volumes: [ - { - name: 'docker', - path: '/var/run/docker.sock', - }, - ], - privileged: true, - }, - { - name: 'test rpm package', - image: 'docker', - commands: ['./tools/packaging/verify-rpm-install.sh'], - volumes: [ - { - name: 'docker', - path: '/var/run/docker.sock', - }, - ], - privileged: true, - }, - run('publish', - commands=['make BUILD_IN_CONTAINER=false publish'], - env={ - GITHUB_TOKEN: { from_secret: github_secret.name }, - NFPM_PASSPHRASE: { from_secret: gpg_passphrase.name }, - NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key', - }) { when: { event: ['tag'] } }, - ], - }, pipeline('docker-driver') { trigger+: onTagOrMain, steps: [ diff --git a/.drone/drone.yml b/.drone/drone.yml index 7a62b621262a..ccac7a2c6ce5 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -5,22 +5,6 @@ platform: arch: amd64 os: linux steps: -- environment: - DOCKER_BUILDKIT: 1 - image: plugins/docker - name: test - settings: - context: loki-build-image - dockerfile: loki-build-image/Dockerfile - dry_run: true - repo: grafana/loki-build-image - tags: - - 0.33.0-amd64 - when: - event: - - pull_request - paths: - - loki-build-image/** - environment: DOCKER_BUILDKIT: 1 image: plugins/docker @@ -58,22 +42,6 @@ platform: arch: arm64 os: linux steps: -- environment: - DOCKER_BUILDKIT: 1 - image: plugins/docker - name: test - settings: - context: loki-build-image - dockerfile: loki-build-image/Dockerfile - dry_run: true - repo: grafana/loki-build-image - tags: - - 0.33.0-arm64 - when: - event: - - pull_request - paths: - - loki-build-image/** - environment: DOCKER_BUILDKIT: 1 image: plugins/docker @@ -137,17 +105,6 @@ trigger: kind: pipeline name: helm-test-image steps: -- image: plugins/docker - name: test-image - settings: - dockerfile: production/helm/loki/src/helm-test/Dockerfile - dry_run: true - repo: grafana/loki-helm-test - when: - event: - - pull_request - paths: - - production/helm/loki/src/helm-test/** - image: plugins/docker name: push-image settings: @@ -175,165 +132,6 @@ workspace: path: loki --- kind: pipeline -name: check -steps: -- commands: - - make BUILD_IN_CONTAINER=false check-drone-drift - depends_on: - - clone - environment: {} - image: grafana/loki-build-image:0.33.0 - name: check-drone-drift -- commands: - - make BUILD_IN_CONTAINER=false check-generated-files - depends_on: - - clone - environment: {} - image: grafana/loki-build-image:0.33.0 - name: check-generated-files -- commands: - - cd .. - - 'echo "cloning "$DRONE_TARGET_BRANCH ' - - git clone -b $DRONE_TARGET_BRANCH $CI_REPO_REMOTE loki-target-branch - - cd - - depends_on: - - clone - environment: {} - image: grafana/loki-build-image:0.33.0 - name: clone-target-branch - when: - event: - - pull_request -- commands: - - make BUILD_IN_CONTAINER=false test - depends_on: - - clone-target-branch - - check-generated-files - environment: {} - image: grafana/loki-build-image:0.33.0 - name: test -- commands: - - cd ../loki-target-branch && BUILD_IN_CONTAINER=false make test - depends_on: - - clone-target-branch - environment: {} - image: grafana/loki-build-image:0.33.0 - name: test-target-branch - when: - event: - - pull_request -- commands: - - make BUILD_IN_CONTAINER=false compare-coverage old=../loki-target-branch/test_results.txt - new=test_results.txt packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki - > diff.txt - depends_on: - - test - - test-target-branch - environment: {} - image: grafana/loki-build-image:0.33.0 - name: compare-coverage - when: - event: - - pull_request -- commands: - - total_diff=$(sed 's/%//' diff.txt | awk '{sum+=$3;}END{print sum;}') - - if [ $total_diff = 0 ]; then exit 0; fi - - pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}') - - 'body=$(jq -Rs ''{body: . }'' diff.txt)' - - 'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments - -d "$body" > /dev/null' - depends_on: - - compare-coverage - environment: - TOKEN: - from_secret: github_token - USER: grafanabot - image: grafana/loki-build-image:0.33.0 - name: report-coverage - when: - event: - - pull_request -- commands: - - make BUILD_IN_CONTAINER=false lint - depends_on: - - check-generated-files - environment: {} - image: grafana/loki-build-image:0.33.0 - name: lint -- commands: - - make BUILD_IN_CONTAINER=false check-mod - depends_on: - - test - - lint - environment: {} - image: grafana/loki-build-image:0.33.0 - name: check-mod -- commands: - - apk add make bash && make lint-scripts - image: koalaman/shellcheck-alpine:stable - name: shellcheck -- commands: - - make BUILD_IN_CONTAINER=false loki - depends_on: - - check-generated-files - environment: {} - image: grafana/loki-build-image:0.33.0 - name: loki -- commands: - - make BUILD_IN_CONTAINER=false check-doc - depends_on: - - loki - environment: {} - image: grafana/loki-build-image:0.33.0 - name: check-doc -- commands: - - make BUILD_IN_CONTAINER=false check-format GIT_TARGET_BRANCH="$DRONE_TARGET_BRANCH" - depends_on: - - loki - environment: {} - image: grafana/loki-build-image:0.33.0 - name: check-format - when: - event: - - pull_request -- commands: - - make BUILD_IN_CONTAINER=false validate-example-configs - depends_on: - - loki - environment: {} - image: grafana/loki-build-image:0.33.0 - name: validate-example-configs -- commands: - - make BUILD_IN_CONTAINER=false validate-dev-cluster-config - depends_on: - - validate-example-configs - environment: {} - image: grafana/loki-build-image:0.33.0 - name: validate-dev-cluster-config -- commands: - - make BUILD_IN_CONTAINER=false check-example-config-doc - depends_on: - - clone - environment: {} - image: grafana/loki-build-image:0.33.0 - name: check-example-config-doc -- commands: - - mkdir -p /hugo/content/docs/loki/latest - - cp -r docs/sources/* /hugo/content/docs/loki/latest/ - - cd /hugo && make prod - image: grafana/docs-base:e6ef023f8b8 - name: build-docs-website -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -workspace: - base: /src - path: loki ---- -kind: pipeline name: mixins steps: - commands: @@ -385,8 +183,6 @@ workspace: base: /src path: loki --- -depends_on: -- check kind: pipeline name: docker-amd64 platform: @@ -399,66 +195,6 @@ steps: - echo $(./tools/image-tag)-amd64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-image - settings: - dockerfile: cmd/loki/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-canary-image - settings: - dockerfile: cmd/loki-canary/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-canary - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-canary-boringcrypto-image - settings: - dockerfile: cmd/loki-canary-boringcrypto/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-canary-boringcrypto - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker - name: build-logcli-image - settings: - dockerfile: cmd/logcli/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/logcli - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -530,8 +266,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: docker-arm64 platform: @@ -544,66 +278,6 @@ steps: - echo $(./tools/image-tag)-arm64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-image - settings: - dockerfile: cmd/loki/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-canary-image - settings: - dockerfile: cmd/loki-canary/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-canary - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-canary-boringcrypto-image - settings: - dockerfile: cmd/loki-canary-boringcrypto/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-canary-boringcrypto - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker - name: build-logcli-image - settings: - dockerfile: cmd/logcli/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/logcli - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -675,8 +349,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: docker-arm platform: @@ -689,66 +361,6 @@ steps: - echo $(./tools/image-tag)-arm > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: build-loki-image - settings: - dockerfile: cmd/loki/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: build-loki-canary-image - settings: - dockerfile: cmd/loki-canary/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-canary - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: build-loki-canary-boringcrypto-image - settings: - dockerfile: cmd/loki-canary-boringcrypto/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-canary-boringcrypto - username: - from_secret: docker_username - when: - event: - - pull_request -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: build-logcli-image - settings: - dockerfile: cmd/logcli/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/logcli - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker:linux-arm @@ -820,8 +432,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: promtail-amd64 platform: @@ -834,21 +444,6 @@ steps: - echo $(./tools/image-tag)-amd64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-promtail-image - settings: - dockerfile: clients/cmd/promtail/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/promtail - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -872,35 +467,18 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: promtail-arm64 platform: arch: arm64 os: linux steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-promtail-image - settings: - dockerfile: clients/cmd/promtail/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/promtail - username: - from_secret: docker_username - when: - event: - - pull_request +- commands: + - apk add --no-cache bash git + - git fetch origin --tags + - echo $(./tools/image-tag)-arm64 > .tags + image: alpine + name: image-tag - depends_on: - image-tag image: plugins/docker @@ -924,8 +502,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: promtail-arm platform: @@ -938,21 +514,6 @@ steps: - echo $(./tools/image-tag)-arm > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: build-promtail-image - settings: - dockerfile: clients/cmd/promtail/Dockerfile.arm32 - dry_run: true - password: - from_secret: docker_password - repo: grafana/promtail - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker:linux-arm @@ -976,8 +537,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: lokioperator-amd64 platform: @@ -990,22 +549,6 @@ steps: - echo $(./tools/image-tag)-amd64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-operator-image - settings: - context: operator - dockerfile: operator/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-operator - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1032,8 +575,6 @@ trigger: - refs/tags/operator/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: lokioperator-arm64 platform: @@ -1046,22 +587,6 @@ steps: - echo $(./tools/image-tag)-arm64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-loki-operator-image - settings: - context: operator - dockerfile: operator/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-operator - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1088,8 +613,6 @@ trigger: - refs/tags/operator/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: lokioperator-arm platform: @@ -1102,22 +625,6 @@ steps: - echo $(./tools/image-tag)-arm > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: build-loki-operator-image - settings: - context: operator - dockerfile: operator/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-operator - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker:linux-arm @@ -1144,8 +651,6 @@ trigger: - refs/tags/operator/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: fluent-bit-amd64 platform: @@ -1158,21 +663,6 @@ steps: - echo $(./tools/image-tag)-amd64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-fluent-bit-image - settings: - dockerfile: clients/cmd/fluent-bit/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/fluent-bit-plugin-loki - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1196,8 +686,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: fluent-bit-arm64 platform: @@ -1210,21 +698,6 @@ steps: - echo $(./tools/image-tag)-arm64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-fluent-bit-image - settings: - dockerfile: clients/cmd/fluent-bit/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/fluent-bit-plugin-loki - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1248,8 +721,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: fluent-bit-arm platform: @@ -1262,21 +733,6 @@ steps: - echo $(./tools/image-tag)-arm > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: build-fluent-bit-image - settings: - dockerfile: clients/cmd/fluent-bit/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/fluent-bit-plugin-loki - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker:linux-arm @@ -1300,8 +756,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: fluentd-amd64 platform: @@ -1315,21 +769,6 @@ steps: - echo ",main" >> .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-fluentd-image - settings: - dockerfile: clients/cmd/fluentd/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/fluent-plugin-loki - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1353,8 +792,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: logstash-amd64 platform: @@ -1368,21 +805,6 @@ steps: - echo ",main" >> .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-logstash-image - settings: - dockerfile: clients/cmd/logstash/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/logstash-output-loki - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1406,8 +828,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: querytee-amd64 platform: @@ -1421,21 +841,6 @@ steps: - echo ",main" >> .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-querytee-image - settings: - dockerfile: cmd/querytee/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/loki-query-tee - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1686,8 +1091,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: logql-analyzer platform: @@ -1700,21 +1103,6 @@ steps: - echo $(./tools/image-tag)-amd64 > .tags image: alpine name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: build-logql-analyzer-image - settings: - dockerfile: cmd/logql-analyzer/Dockerfile - dry_run: true - password: - from_secret: docker_password - repo: grafana/logql-analyzer - username: - from_secret: docker_username - when: - event: - - pull_request - depends_on: - image-tag image: plugins/docker @@ -1738,103 +1126,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: release -services: -- image: jrei/systemd-debian:12 - name: systemd-debian - privileged: true - volumes: - - name: cgroup - path: /sys/fs/cgroup -- image: jrei/systemd-centos:8 - name: systemd-centos - privileged: true - volumes: - - name: cgroup - path: /sys/fs/cgroup -steps: -- commands: - - if [ "$${#TEST_SECRET}" -eq 0 ]; then - - ' echo "Missing a secret to run this pipeline. This branch needs to be re-pushed - as a branch in main grafana/loki repository in order to run." && exit 78' - - fi - environment: - TEST_SECRET: - from_secret: gpg_private_key - image: alpine - name: skip pipeline if missing secret -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - image: alpine - name: fetch-tags -- commands: - - printf "%s" "$NFPM_SIGNING_KEY" > $NFPM_SIGNING_KEY_FILE - environment: - NFPM_SIGNING_KEY: - from_secret: gpg_private_key - NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key - image: grafana/loki-build-image:0.33.0 - name: write-key -- commands: - - make BUILD_IN_CONTAINER=false packages - environment: - NFPM_PASSPHRASE: - from_secret: gpg_passphrase - NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key - image: grafana/loki-build-image:0.33.0 - name: test packaging -- commands: - - ./tools/packaging/verify-deb-install.sh - image: docker - name: test deb package - privileged: true - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./tools/packaging/verify-rpm-install.sh - image: docker - name: test rpm package - privileged: true - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - make BUILD_IN_CONTAINER=false publish - environment: - GITHUB_TOKEN: - from_secret: github_token - NFPM_PASSPHRASE: - from_secret: gpg_passphrase - NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key - image: grafana/loki-build-image:0.33.0 - name: publish - when: - event: - - tag -trigger: - event: - - pull_request - - tag - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -volumes: -- host: - path: /sys/fs/cgroup - name: cgroup -- host: - path: /var/run/docker.sock - name: docker ---- kind: pipeline name: docker-driver steps: @@ -1868,8 +1159,6 @@ volumes: path: /var/run/docker.sock name: docker --- -depends_on: -- check kind: pipeline name: lambda-promtail-amd64 platform: @@ -1892,25 +1181,6 @@ steps: from_secret: ecr_key image: alpine name: skip pipeline if missing secret -- depends_on: - - image-tag - - skip pipeline if missing secret - image: cstyan/ecr - name: build-lambda-promtail-image - privileged: true - settings: - access_key: - from_secret: ecr_key - dockerfile: tools/lambda-promtail/Dockerfile - dry_run: true - region: us-east-1 - registry: public.ecr.aws/grafana - repo: public.ecr.aws/grafana/lambda-promtail - secret_key: - from_secret: ecr_secret_key - when: - event: - - pull_request - depends_on: - image-tag image: cstyan/ecr @@ -1937,8 +1207,6 @@ trigger: - refs/tags/v* - refs/pull/*/head --- -depends_on: -- check kind: pipeline name: lambda-promtail-arm64 platform: @@ -1961,25 +1229,6 @@ steps: from_secret: ecr_key image: alpine name: skip pipeline if missing secret -- depends_on: - - image-tag - - skip pipeline if missing secret - image: cstyan/ecr - name: build-lambda-promtail-image - privileged: true - settings: - access_key: - from_secret: ecr_key - dockerfile: tools/lambda-promtail/Dockerfile - dry_run: true - region: us-east-1 - registry: public.ecr.aws/grafana - repo: public.ecr.aws/grafana/lambda-promtail - secret_key: - from_secret: ecr_secret_key - when: - event: - - pull_request - depends_on: - image-tag image: cstyan/ecr @@ -2113,6 +1362,6 @@ kind: secret name: gpg_private_key --- kind: signature -hmac: 457592d17208477ceb480f81dbdb88f7b95a5ad015c88d9d6fed06c2422a52f9 +hmac: 32b44aecaad0258ed9494225595e1016a56bea960bcd0b15b2db3449bed957e0 ... diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json new file mode 100644 index 000000000000..cd4469eb6e50 --- /dev/null +++ b/.github/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/loki-release.git", + "subdir": "workflows" + } + }, + "version": "release-1.10.x" + } + ], + "legacyImports": true +} diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json new file mode 100644 index 000000000000..ee1f7b9596b4 --- /dev/null +++ b/.github/jsonnetfile.lock.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/loki-release.git", + "subdir": "workflows" + } + }, + "version": "c005223f58b83f288b655dde5bcfeff7490c7aa5", + "sum": "5K+r6Bsb8JMR1ytQjSObjvHFpH7SJBi5D4ysSwvC4/g=" + } + ], + "legacyImports": false +} diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet new file mode 100644 index 000000000000..ae1f868fa651 --- /dev/null +++ b/.github/release-workflows.jsonnet @@ -0,0 +1,58 @@ +local lokiRelease = import 'workflows/main.jsonnet'; +local build = lokiRelease.build; +{ + 'patch-release-pr.yml': std.manifestYamlDoc( + lokiRelease.releasePRWorkflow( + imageJobs={ + loki: build.image('loki', 'cmd/loki'), + fluentd: build.image('fluentd', 'clients/cmd/fluentd', platform=['linux/amd64']), + 'fluent-bit': build.image('fluent-bit', 'clients/cmd/fluent-bit', platform=['linux/amd64']), + logstash: build.image('logstash', 'clients/cmd/logstash', platform=['linux/amd64']), + logcli: build.image('logcli', 'cmd/logcli'), + 'loki-canary': build.image('loki-canary', 'cmd/loki-canary'), + 'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), + 'loki-operator': build.image('loki-operator', 'operator', context='release/operator', platform=['linux/amd64']), + promtail: build.image('promtail', 'clients/cmd/promtail'), + querytee: build.image('querytee', 'cmd/querytee', platform=['linux/amd64']), + }, + branches=['release-[0-9]+.[0-9]+.x'], + checkTemplate='grafana/loki-release/.github/workflows/check.yml@release-1.10.x', + imagePrefix='grafana', + releaseRepo='grafana/loki', + skipArm=false, + skipValidation=false, + versioningStrategy='always-bump-patch', + ), false, false + ), + 'minor-release-pr.yml': std.manifestYamlDoc( + lokiRelease.releasePRWorkflow( + imageJobs={ + loki: build.image('loki', 'cmd/loki'), + fluentd: build.image('fluentd', 'clients/cmd/fluentd', platform=['linux/amd64']), + 'fluent-bit': build.image('fluent-bit', 'clients/cmd/fluent-bit', platform=['linux/amd64']), + logstash: build.image('logstash', 'clients/cmd/logstash', platform=['linux/amd64']), + logcli: build.image('logcli', 'cmd/logcli'), + 'loki-canary': build.image('loki-canary', 'cmd/loki-canary'), + 'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), + 'loki-operator': build.image('loki-operator', 'operator', context='release/operator', platform=['linux/amd64']), + promtail: build.image('promtail', 'clients/cmd/promtail'), + querytee: build.image('querytee', 'cmd/querytee', platform=['linux/amd64']), + }, + branches=['k[0-9]+'], + checkTemplate='grafana/loki-release/.github/workflows/check.yml@release-1.10.x', + imagePrefix='grafana', + releaseRepo='grafana/loki', + skipArm=false, + skipValidation=false, + versioningStrategy='always-bump-minor', + ), false, false + ), + 'release.yml': std.manifestYamlDoc( + lokiRelease.releaseWorkflow( + branches=['release-[0-9]+.[0-9]+.x', 'k[0-9]+'], + getDockerCredsFromVault=true, + imagePrefix='grafana', + releaseRepo='grafana/loki', + ), false, false + ), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet new file mode 100644 index 000000000000..cdd6b82463e4 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet @@ -0,0 +1,154 @@ +local common = import 'common.libsonnet'; +local job = common.job; +local step = common.step; +local releaseStep = common.releaseStep; +local releaseLibStep = common.releaseLibStep; + +{ + image: function( + name, + path, + context='release', + platform=[ + 'linux/amd64', + 'linux/arm64', + 'linux/arm', + ] + ) + job.new() + + job.withStrategy({ + 'fail-fast': true, + matrix: { + platform: platform, + }, + }) + + job.withSteps([ + common.fetchReleaseLib, + common.fetchReleaseRepo, + common.setupNode, + common.googleAuth, + + step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), + step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), + + releaseStep('parse image platform') + + step.withId('platform') + + step.withRun(||| + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + |||), + + step.new('Build and export', 'docker/build-push-action@v5') + + step.withTimeoutMinutes(25) + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + + step.with({ + context: context, + file: 'release/%s/Dockerfile' % path, + platforms: '${{ matrix.platform }}', + tags: '${{ env.IMAGE_PREFIX }}/%s:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}' % [name], + outputs: 'type=docker,dest=release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, + }), + step.new('upload artifacts', 'google-github-actions/upload-cloud-storage@v2') + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + + step.with({ + path: 'release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, + destination: 'loki-build-artifacts/${{ github.sha }}/images', //TODO: make bucket configurable + process_gcloudignore: false, + }), + ]), + + version: + job.new() + + job.withSteps([ + common.fetchReleaseLib, + common.fetchReleaseRepo, + common.setupNode, + common.extractBranchName, + releaseLibStep('get release version') + + step.withId('version') + + step.withRun(||| + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --dry-run \ + --dry-run-output release.json \ + --release-type simple \ + --repo-url="${{ env.RELEASE_REPO }}" \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token="${{ secrets.GH_TOKEN }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + if [[ `jq length release.json` -gt 1 ]]; then + echo 'release-please would create more than 1 PR, so cannot determine correct version' + echo "pr_created=false" >> $GITHUB_OUTPUT + exit 1 + fi + + if [[ `jq length release.json` -eq 0 ]]; then + echo "pr_created=false" >> $GITHUB_OUTPUT + else + version="$(npm run --silent get-version)" + echo "Parsed version: ${version}" + echo "version=${version}" >> $GITHUB_OUTPUT + echo "pr_created=true" >> $GITHUB_OUTPUT + fi + |||), + ]) + + job.withOutputs({ + version: '${{ steps.version.outputs.version }}', + pr_created: '${{ steps.version.outputs.pr_created }}', + }), + + dist: function(buildImage, skipArm=true) + job.new() + + job.withSteps([ + common.fetchReleaseRepo, + common.googleAuth, + step.new('get nfpm signing keys', 'grafana/shared-workflows/actions/get-vault-secrets@main') + + step.withId('get-secrets') + + step.with({ + common_secrets: ||| + NFPM_SIGNING_KEY=packages-gpg:private-key + NFPM_PASSPHRASE=packages-gpg:passphrase + |||, + }), + + releaseStep('build artifacts') + + step.withEnv({ + BUILD_IN_CONTAINER: false, + DRONE_TAG: '${{ needs.version.outputs.version }}', + IMAGE_TAG: '${{ needs.version.outputs.version }}', + NFPM_SIGNING_KEY_FILE: 'nfpm-private-key.key', + SKIP_ARM: skipArm, + }) + //TODO: the workdir here is loki specific + + step.withRun(||| + cat < $NFPM_SIGNING_KEY_FILE + make dist packages + EOF + ||| % buildImage), + + step.new('upload build artifacts', 'google-github-actions/upload-cloud-storage@v2') + + step.with({ + path: 'release/dist', + destination: 'loki-build-artifacts/${{ github.sha }}', //TODO: make bucket configurable + process_gcloudignore: false, + }), + ]), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet new file mode 100644 index 000000000000..e3346f2bd5e4 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet @@ -0,0 +1,124 @@ +{ + step: { + new: function(name, uses=null) { + name: name, + } + if uses != null then { + uses: uses, + } else {}, + with: function(with) { + with+: with, + }, + withRun: function(run) { + run: run, + }, + withId: function(id) { + id: id, + }, + withWorkingDirectory: function(workingDirectory) { + 'working-directory': workingDirectory, + }, + withIf: function(_if) { + 'if': _if, + }, + withEnv: function(env) { + env: env, + }, + withSecrets: function(env) { + secrets: env, + }, + withTimeoutMinutes: function(timeout) { + 'timeout-minutes': timeout, + }, + }, + job: { + new: function(runsOn='ubuntu-latest') { + 'runs-on': runsOn, + }, + with: function(with) { + with+: with, + }, + withUses: function(uses) { + uses: uses, + }, + withSteps: function(steps) { + steps: steps, + }, + withStrategy: function(strategy) { + strategy: strategy, + }, + withNeeds: function(needs) { + needs: needs, + }, + withIf: function(_if) { + 'if': _if, + }, + withOutputs: function(outputs) { + outputs: outputs, + }, + withContainer: function(container) { + container: container, + }, + withEnv: function(env) { + env: env, + }, + withSecrets: function(env) { + secrets: env, + }, + }, + + releaseStep: function(name, uses=null) $.step.new(name, uses) + + $.step.withWorkingDirectory('release'), + + releaseLibStep: function(name, uses=null) $.step.new(name, uses) + + $.step.withWorkingDirectory('lib'), + + checkout: + $.step.new('checkout', 'actions/checkout@v4'), + + fetchReleaseRepo: + $.step.new('pull code to release', 'actions/checkout@v4') + + $.step.with({ + repository: '${{ env.RELEASE_REPO }}', + path: 'release', + }), + fetchReleaseLib: + $.step.new('pull release library code', 'actions/checkout@v4') + + $.step.with({ + repository: 'grafana/loki-release', + path: 'lib', + }), + + setupNode: $.step.new('setup node', 'actions/setup-node@v4') + + $.step.with({ + 'node-version': 20, + }), + + makeTarget: function(target) 'make %s' % target, + + alwaysGreen: { + steps: [ + $.step.new('always green') + + $.step.withRun('echo "always green"'), + ], + }, + + googleAuth: $.step.new('auth gcs', 'google-github-actions/auth@v2') + + $.step.with({ + credentials_json: '${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}', + }), + setupGoogleCloudSdk: $.step.new('Set up Cloud SDK', 'google-github-actions/setup-gcloud@v2') + + $.step.with({ + version: '>= 452.0.0', + }), + + extractBranchName: $.releaseStep('extract branch name') + + $.step.withId('extract_branch') + + $.step.withRun(||| + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + |||), + + fixDubiousOwnership: $.step.new('fix git dubious ownership') + + $.step.withRun(||| + git config --global --add safe.directory "$GITHUB_WORKSPACE" + |||), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet new file mode 100644 index 000000000000..0a033b81221f --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet @@ -0,0 +1,111 @@ +{ + common: import 'common.libsonnet', + job: $.common.job, + step: $.common.step, + build: import 'build.libsonnet', + release: import 'release.libsonnet', + validate: import 'validate.libsonnet', + releasePRWorkflow: function( + branches=['release-[0-9]+.[0-9]+.x', 'k[0-9]+'], + buildImage='grafana/loki-build-image:0.33.0', + checkTemplate='./.github/workflows/check.yml', + dockerUsername='grafana', + imageJobs={}, + imagePrefix='grafana', + releaseRepo='grafana/loki-release', + skipArm=true, + skipValidation=false, + versioningStrategy='always-bump-patch', + ) { + name: 'create release PR', + on: { + push: { + branches: branches, + }, + }, + permissions: { + contents: 'write', + 'pull-requests': 'write', + 'id-token': 'write', + }, + concurrency: { + group: 'create-release-pr-${{ github.sha }}', + }, + env: { + RELEASE_REPO: releaseRepo, + DOCKER_USERNAME: dockerUsername, + IMAGE_PREFIX: imagePrefix, + SKIP_VALIDATION: skipValidation, + VERSIONING_STRATEGY: versioningStrategy, + }, + local validationSteps = ['check'], + jobs: { + check: {} + $.job.withUses(checkTemplate) + + $.job.with({ + skip_validation: skipValidation, + }), + version: $.build.version + $.common.job.withNeeds(validationSteps), + dist: $.build.dist(buildImage, skipArm) + $.common.job.withNeeds(['version']), + } + std.mapWithKey(function(name, job) job + $.common.job.withNeeds(['version']), imageJobs) + { + local buildImageSteps = ['dist'] + std.objectFields(imageJobs), + 'create-release-pr': $.release.createReleasePR + $.common.job.withNeeds(buildImageSteps), + }, + }, + releaseWorkflow: function( + releaseRepo='grafana/loki-release', + dockerUsername='grafana', + imagePrefix='grafana', + branches=['release-[0-9].[0-9].x', 'k[0-9]*'], + getDockerCredsFromVault=false + ) { + name: 'create release', + on: { + push: { + branches: branches, + }, + }, + permissions: { + contents: 'write', + 'pull-requests': 'write', + 'id-token': 'write', + }, + concurrency: { + group: 'create-release-${{ github.sha }}', + }, + env: { + RELEASE_REPO: releaseRepo, + IMAGE_PREFIX: imagePrefix, + }, + jobs: { + shouldRelease: $.release.shouldRelease, + createRelease: $.release.createRelease, + publishImages: $.release.publishImages(getDockerCredsFromVault, dockerUsername), + }, + }, + check: function( + buildImage='grafana/loki-build-image:0.33.0', + ) { + name: 'check', + on: { + workflow_call: { + inputs: { + skip_validation: { + default: false, + description: 'skip validation steps', + required: false, + type: 'boolean', + }, + }, + }, + }, + permissions: { + contents: 'write', + 'pull-requests': 'write', + 'id-token': 'write', + }, + concurrency: { + group: 'check-${{ github.sha }}', + }, + jobs: $.validate(buildImage), + }, +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet new file mode 100644 index 000000000000..6bf2daa8f033 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet @@ -0,0 +1,144 @@ +local common = import 'common.libsonnet'; +local job = common.job; +local step = common.step; +local releaseStep = common.releaseStep; +local releaseLibStep = common.releaseLibStep; + +// DO NOT MODIFY THIS FOOTER TEMPLATE +// This template is matched by the should-release action to detect the correct +// sha to release and pull aritfacts from. If you need to change this, make sure +// to change it in both places. +//TODO: make bucket configurable +local pullRequestFooter = 'Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}'; + +{ + createReleasePR: + job.new() + + job.withSteps([ + common.fetchReleaseRepo, + common.fetchReleaseLib, + common.setupNode, + common.extractBranchName, + + releaseLibStep('release please') + + step.withId('release') + + step.withEnv({ + SHA: '${{ github.sha }}', + }) + //TODO make bucket configurable + //TODO make a type/release in the backport action + //TODO backport action should not bring over autorelease: pending label + + step.withRun(||| + npm install + echo "Pull request footer: %s" + npm exec -- release-please release-pr \ + --consider-all-branches \ + --label "backport main,autorelease: pending,type/docs" \ + --pull-request-footer "%s" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ secrets.GH_TOKEN }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" \ + --separate-pull-requests false \ + --debug + ||| % [pullRequestFooter, pullRequestFooter]), + ]), + + shouldRelease: job.new() + + job.withSteps([ + common.fetchReleaseRepo, + common.fetchReleaseLib, + common.extractBranchName, + + step.new('should a release be created?', './lib/actions/should-release') + + step.withId('should_release') + + step.with({ + baseBranch: '${{ steps.extract_branch.outputs.branch }}', + }), + ]) + + job.withOutputs({ + shouldRelease: '${{ steps.should_release.outputs.shouldRelease }}', + sha: '${{ steps.should_release.outputs.sha }}', + name: '${{ steps.should_release.outputs.name }}', + branch: '${{ steps.extract_branch.outputs.branch }}', + + }), + createRelease: job.new() + + job.withNeeds(['shouldRelease']) + + job.withIf('${{ fromJSON(needs.shouldRelease.outputs.shouldRelease) }}') + + job.withSteps([ + common.fetchReleaseRepo, + common.fetchReleaseLib, + common.setupNode, + common.googleAuth, + common.setupGoogleCloudSdk, + + // exits with code 1 if the url does not match + // meaning there are no artifacts for that sha + // we need to handle this if we're going to run this pipeline on every merge to main + releaseStep('download binaries') + + step.withRun(||| + echo "downloading binaries to $(pwd)/dist" + gsutil cp -r gs://loki-build-artifacts/${{ needs.shouldRelease.outputs.sha }}/dist . + |||), + + releaseLibStep('create release') + + step.withId('release') + + step.withRun(||| + npm install + npm exec -- release-please github-release \ + --draft \ + --release-type simple \ + --repo-url="${{ env.RELEASE_REPO }}" \ + --target-branch "${{ needs.shouldRelease.outputs.branch }}" \ + --token="${{ secrets.GH_TOKEN }}" + |||), + + releaseStep('upload artifacts') + + step.withId('upload') + + step.withEnv({ + GH_TOKEN: '${{ secrets.GH_TOKEN }}', + }) + + step.withRun(||| + gh release upload ${{ needs.shouldRelease.outputs.name }} dist/* + gh release edit ${{ needs.shouldRelease.outputs.name }} --draft=false + |||), + ]) + + job.withOutputs({ + sha: '${{ needs.shouldRelease.outputs.sha }}', + }), + + publishImages: function(getDockerCredsFromVault=false, dockerUsername='grafanabot') + job.new() + + job.withNeeds(['createRelease']) + + job.withSteps( + [ + common.fetchReleaseLib, + common.googleAuth, + common.setupGoogleCloudSdk, + step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), + step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), + ] + (if getDockerCredsFromVault then [ + step.new('Login to DockerHub (from vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'), + ] else [ + step.new('Login to DockerHub (from secrets)', 'docker/login-action@v3') + + step.with({ + username: dockerUsername, + password: '${{ secrets.DOCKER_PASSWORD }}', + }), + ]) + + [ + step.new('download images') + + step.withRun(||| + echo "downloading images to $(pwd)/images" + gsutil cp -r gs://loki-build-artifacts/${{ needs.createRelease.outputs.sha }}/images . + |||), + step.new('publish docker images', './lib/actions/push-images') + + step.with({ + imageDir: 'images', + imagePrefix: '${{ env.IMAGE_PREFIX }}', + }), + ] + ), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet new file mode 100644 index 000000000000..477e077d8554 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet @@ -0,0 +1,114 @@ +local common = import 'common.libsonnet'; +local job = common.job; +local step = common.step; +local releaseStep = common.releaseStep; + +local setupValidationDeps = function(job) job { + steps: [ + common.checkout, + common.fetchReleaseLib, + common.fixDubiousOwnership, + step.new('install tar') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + apt update + apt install -qy tar xz-utils + |||), + step.new('install shellcheck', './lib/actions/install-binary') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + binary: 'shellcheck', + version: '0.9.0', + download_url: 'https://github.com/koalaman/shellcheck/releases/download/v${version}/shellcheck-v${version}.linux.x86_64.tar.xz', + tarball_binary_path: '*/${binary}', + smoke_test: '${binary} --version', + tar_args: 'xvf', + }), + step.new('install jsonnetfmt', './lib/actions/install-binary') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + binary: 'jsonnetfmt', + version: '0.18.0', + download_url: 'https://github.com/google/go-jsonnet/releases/download/v${version}/go-jsonnet_${version}_Linux_x86_64.tar.gz', + tarball_binary_path: '${binary}', + smoke_test: '${binary} --version', + }), + ] + job.steps, +}; + +local validationJob = function(buildImage) job.new() + + job.withContainer({ + image: buildImage, + }) + + job.withEnv({ + BUILD_IN_CONTAINER: false, + SKIP_VALIDATION: '${{ inputs.skip_validation }}', + }); + + +function(buildImage) { + local validationMakeStep = function(name, target) + step.new(name) + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(common.makeTarget(target)), + + test: setupValidationDeps( + validationJob(buildImage) + + job.withSteps([ + validationMakeStep('test', 'test'), + ]) + ), + + lint: setupValidationDeps( + validationJob(buildImage) + + job.withSteps([ + validationMakeStep('lint', 'lint'), + validationMakeStep('lint jsonnet', 'lint-jsonnet'), + validationMakeStep('lint scripts', 'lint-scripts'), + validationMakeStep('format', 'check-format'), + ]) + { + steps+: [ + step.new('golangci-lint', 'golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + version: 'v1.55.1', + 'only-new-issues': true, + }), + ], + } + ), + + check: setupValidationDeps( + validationJob(buildImage) + + job.withSteps([ + validationMakeStep('check generated files', 'check-generated-files'), + validationMakeStep('check mod', 'check-mod'), + validationMakeStep('check docs', 'check-doc'), + validationMakeStep('validate example configs', 'validate-example-configs'), + validationMakeStep('validate dev cluster config', 'validate-dev-cluster-config'), + validationMakeStep('check example config docs', 'check-example-config-doc'), + validationMakeStep('check helm reference doc', 'documentation-helm-reference-check'), + validationMakeStep('check drone drift', 'check-drone-drift'), + ]) + { + steps+: [ + step.new('build docs website') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + cat <> $GITHUB_OUTPUT + working-directory: "release" + - env: + SHA: "${{ github.sha }}" + id: "release" + name: "release please" + run: | + npm install + echo "Pull request footer: Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" + npm exec -- release-please release-pr \ + --consider-all-branches \ + --label "backport main,autorelease: pending,type/docs" \ + --pull-request-footer "Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ secrets.GH_TOKEN }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" \ + --separate-pull-requests false \ + --debug + working-directory: "lib" + dist: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - id: "get-secrets" + name: "get nfpm signing keys" + uses: "grafana/shared-workflows/actions/get-vault-secrets@main" + with: + common_secrets: | + NFPM_SIGNING_KEY=packages-gpg:private-key + NFPM_PASSPHRASE=packages-gpg:passphrase + - env: + BUILD_IN_CONTAINER: false + DRONE_TAG: "${{ needs.version.outputs.version }}" + IMAGE_TAG: "${{ needs.version.outputs.version }}" + NFPM_SIGNING_KEY_FILE: "nfpm-private-key.key" + SKIP_ARM: false + name: "build artifacts" + run: | + cat < $NFPM_SIGNING_KEY_FILE + make dist packages + EOF + working-directory: "release" + - name: "upload build artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}" + path: "release/dist" + process_gcloudignore: false + fluent-bit: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluent-bit/Dockerfile" + outputs: "type=docker,dest=release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-bit:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + fluentd: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluentd/Dockerfile" + outputs: "type=docker,dest=release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluentd:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + logcli: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/logcli/Dockerfile" + outputs: "type=docker,dest=release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logcli:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + logstash: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/logstash/Dockerfile" + outputs: "type=docker,dest=release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logstash:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + loki: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki/Dockerfile" + outputs: "type=docker,dest=release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-canary: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki-canary/Dockerfile" + outputs: "type=docker,dest=release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-canary:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-canary-boringcrypto: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki-canary-boringcrypto/Dockerfile" + outputs: "type=docker,dest=release/images/loki-canary-boringcrypto-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-canary-boringcrypto:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-canary-boringcrypto-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-operator: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release/operator" + file: "release/operator/Dockerfile" + outputs: "type=docker,dest=release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-operator:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + promtail: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/promtail/Dockerfile" + outputs: "type=docker,dest=release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/promtail:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + querytee: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/querytee/Dockerfile" + outputs: "type=docker,dest=release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/querytee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + version: + needs: + - "check" + outputs: + pr_created: "${{ steps.version.outputs.pr_created }}" + version: "${{ steps.version.outputs.version }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - id: "version" + name: "get release version" + run: | + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --dry-run \ + --dry-run-output release.json \ + --release-type simple \ + --repo-url="${{ env.RELEASE_REPO }}" \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token="${{ secrets.GH_TOKEN }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + if [[ `jq length release.json` -gt 1 ]]; then + echo 'release-please would create more than 1 PR, so cannot determine correct version' + echo "pr_created=false" >> $GITHUB_OUTPUT + exit 1 + fi + + if [[ `jq length release.json` -eq 0 ]]; then + echo "pr_created=false" >> $GITHUB_OUTPUT + else + version="$(npm run --silent get-version)" + echo "Parsed version: ${version}" + echo "version=${version}" >> $GITHUB_OUTPUT + echo "pr_created=true" >> $GITHUB_OUTPUT + fi + working-directory: "lib" +name: "create release PR" +"on": + push: + branches: + - "k[0-9]+" +permissions: + contents: "write" + id-token: "write" + pull-requests: "write" diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index 001b00d93b66..411fff87d410 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -1,21 +1,773 @@ ---- -name: 'create release PR for patch releases' -on: - push: - branches: - - 'release-[0-9].[0-9].x' - workflow_dispatch: {} -permissions: - contents: 'write' - issues: 'write' - pull-requests: 'write' +concurrency: + group: "create-release-pr-${{ github.sha }}" +env: + DOCKER_USERNAME: "grafana" + IMAGE_PREFIX: "grafana" + RELEASE_REPO: "grafana/loki" + SKIP_VALIDATION: false + VERSIONING_STRATEGY: "always-bump-patch" jobs: - create-release-pr: - uses: github/loki-release/.github/workflows/release-pr.yml@main + check: + uses: "grafana/loki-release/.github/workflows/check.yml@release-1.10.x" with: - release_repo: grafana/loki skip_validation: false - versioning_strategy: always-bump-patch - secrets: - GCS_SERVICE_ACCOUNT_KEY: '${{ secrets.BACKEND_ENTERPRISE_DRONE }}' - GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + create-release-pr: + needs: + - "dist" + - "fluent-bit" + - "fluentd" + - "logcli" + - "logstash" + - "loki" + - "loki-canary" + - "loki-canary-boringcrypto" + - "loki-operator" + - "promtail" + - "querytee" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - env: + SHA: "${{ github.sha }}" + id: "release" + name: "release please" + run: | + npm install + echo "Pull request footer: Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" + npm exec -- release-please release-pr \ + --consider-all-branches \ + --label "backport main,autorelease: pending,type/docs" \ + --pull-request-footer "Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ secrets.GH_TOKEN }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" \ + --separate-pull-requests false \ + --debug + working-directory: "lib" + dist: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - id: "get-secrets" + name: "get nfpm signing keys" + uses: "grafana/shared-workflows/actions/get-vault-secrets@main" + with: + common_secrets: | + NFPM_SIGNING_KEY=packages-gpg:private-key + NFPM_PASSPHRASE=packages-gpg:passphrase + - env: + BUILD_IN_CONTAINER: false + DRONE_TAG: "${{ needs.version.outputs.version }}" + IMAGE_TAG: "${{ needs.version.outputs.version }}" + NFPM_SIGNING_KEY_FILE: "nfpm-private-key.key" + SKIP_ARM: false + name: "build artifacts" + run: | + cat < $NFPM_SIGNING_KEY_FILE + make dist packages + EOF + working-directory: "release" + - name: "upload build artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}" + path: "release/dist" + process_gcloudignore: false + fluent-bit: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluent-bit/Dockerfile" + outputs: "type=docker,dest=release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-bit:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + fluentd: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluentd/Dockerfile" + outputs: "type=docker,dest=release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluentd:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + logcli: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/logcli/Dockerfile" + outputs: "type=docker,dest=release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logcli:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + logstash: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/logstash/Dockerfile" + outputs: "type=docker,dest=release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logstash:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + loki: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki/Dockerfile" + outputs: "type=docker,dest=release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-canary: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki-canary/Dockerfile" + outputs: "type=docker,dest=release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-canary:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-canary-boringcrypto: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki-canary-boringcrypto/Dockerfile" + outputs: "type=docker,dest=release/images/loki-canary-boringcrypto-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-canary-boringcrypto:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-canary-boringcrypto-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-operator: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release/operator" + file: "release/operator/Dockerfile" + outputs: "type=docker,dest=release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-operator:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + promtail: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/promtail/Dockerfile" + outputs: "type=docker,dest=release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/promtail:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + querytee: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/querytee/Dockerfile" + outputs: "type=docker,dest=release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/querytee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + version: + needs: + - "check" + outputs: + pr_created: "${{ steps.version.outputs.pr_created }}" + version: "${{ steps.version.outputs.version }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - id: "version" + name: "get release version" + run: | + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --dry-run \ + --dry-run-output release.json \ + --release-type simple \ + --repo-url="${{ env.RELEASE_REPO }}" \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token="${{ secrets.GH_TOKEN }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + if [[ `jq length release.json` -gt 1 ]]; then + echo 'release-please would create more than 1 PR, so cannot determine correct version' + echo "pr_created=false" >> $GITHUB_OUTPUT + exit 1 + fi + + if [[ `jq length release.json` -eq 0 ]]; then + echo "pr_created=false" >> $GITHUB_OUTPUT + else + version="$(npm run --silent get-version)" + echo "Parsed version: ${version}" + echo "version=${version}" >> $GITHUB_OUTPUT + echo "pr_created=true" >> $GITHUB_OUTPUT + fi + working-directory: "lib" +name: "create release PR" +"on": + push: + branches: + - "release-[0-9]+.[0-9]+.x" +permissions: + contents: "write" + id-token: "write" + pull-requests: "write" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cacdacf773a8..64970d1bd719 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,19 +1,131 @@ ---- -name: 'create release' -on: +concurrency: + group: "create-release-${{ github.sha }}" +env: + IMAGE_PREFIX: "grafana" + RELEASE_REPO: "grafana/loki" +jobs: + createRelease: + if: "${{ fromJSON(needs.shouldRelease.outputs.shouldRelease) }}" + needs: + - "shouldRelease" + outputs: + sha: "${{ needs.shouldRelease.outputs.sha }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" + - name: "download binaries" + run: | + echo "downloading binaries to $(pwd)/dist" + gsutil cp -r gs://loki-build-artifacts/${{ needs.shouldRelease.outputs.sha }}/dist . + working-directory: "release" + - id: "release" + name: "create release" + run: | + npm install + npm exec -- release-please github-release \ + --draft \ + --release-type simple \ + --repo-url="${{ env.RELEASE_REPO }}" \ + --target-branch "${{ needs.shouldRelease.outputs.branch }}" \ + --token="${{ secrets.GH_TOKEN }}" + working-directory: "lib" + - env: + GH_TOKEN: "${{ secrets.GH_TOKEN }}" + id: "upload" + name: "upload artifacts" + run: | + gh release upload ${{ needs.shouldRelease.outputs.name }} dist/* + gh release edit ${{ needs.shouldRelease.outputs.name }} --draft=false + working-directory: "release" + publishImages: + needs: + - "createRelease" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - name: "Login to DockerHub (from vault)" + uses: "grafana/shared-workflows/actions/dockerhub-login@main" + - name: "download images" + run: | + echo "downloading images to $(pwd)/images" + gsutil cp -r gs://loki-build-artifacts/${{ needs.createRelease.outputs.sha }}/images . + - name: "publish docker images" + uses: "./lib/actions/push-images" + with: + imageDir: "images" + imagePrefix: "${{ env.IMAGE_PREFIX }}" + shouldRelease: + outputs: + branch: "${{ steps.extract_branch.outputs.branch }}" + name: "${{ steps.should_release.outputs.name }}" + sha: "${{ steps.should_release.outputs.sha }}" + shouldRelease: "${{ steps.should_release.outputs.shouldRelease }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + repository: "grafana/loki-release" + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - id: "should_release" + name: "should a release be created?" + uses: "./lib/actions/should-release" + with: + baseBranch: "${{ steps.extract_branch.outputs.branch }}" +name: "create release" +"on": push: branches: - - 'release-[0-9].[0-9].x' - - 'k[0-9]*' - workflow_dispatch: {} + - "release-[0-9]+.[0-9]+.x" + - "k[0-9]+" permissions: - contents: write - pull-requests: write -jobs: - release: - uses: github/loki-release/.github/workflows/release.yml@main - with: - release_repo: grafana/loki - secrets: - GCS_SERVICE_ACCOUNT_KEY: '${{ secrets.BACKEND_ENTERPRISE_DRONE }}' - GH_TOKEN: '${{ secrets.GH_TOKEN }}' + contents: "write" + id-token: "write" + pull-requests: "write" diff --git a/.gitignore b/.gitignore index 66eb0a8cefeb..83ab9c808d34 100644 --- a/.gitignore +++ b/.gitignore @@ -27,8 +27,8 @@ cmd/querytee/querytee dlv rootfs/ dist -coverage.txt -test_results.txt +*coverage.txt +*test_results.txt .DS_Store .aws-sam .idea diff --git a/.golangci.yml b/.golangci.yml index fb3c1ab689d0..e6475895ad94 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,6 +23,7 @@ run: - linux - cgo - promtail_journal_enabled + - integration # which dirs to skip: they won't be analyzed; # can use regexp here: generated.*, regexp is applied on full path; diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 0e134950eab8..928eee2e123e 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,6 +1,3 @@ { - "cmd/loki": "2.9.4", - "cmd/loki-canary": "2.9.4", - "cmd/logcli": "2.9.4", - "clients/cmd/promtail": "2.9.4" + ".": "2.9.4" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 37599ae8d347..512b9b0bdbb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ ##### Enhancements +* [11840](https://github.com/grafana/loki/pull/11840) **jeschkies**: Allow custom usage trackers for ingested and discarded bytes metric. +* [11814](https://github.com/grafana/loki/pull/11814) **kavirajk**: feat: Support split align and caching for instant metric query results +* [11851](https://github.com/grafana/loki/pull/11851) **elcomtik**: Helm: Allow the definition of resources for GrafanaAgent pods. +* [11819](https://github.com/grafana/loki/pull/11819) **jburnham**: Ruler: Add the ability to disable the `X-Scope-OrgId` tenant identification header in remote write requests. * [11633](https://github.com/grafana/loki/pull/11633) **cyriltovena**: Add profiling integrations to tracing instrumentation. * [11571](https://github.com/grafana/loki/pull/11571) **MichelHollands**: Add a metrics.go log line for requests from querier to ingester * [11477](https://github.com/grafana/loki/pull/11477) **MichelHollands**: support GET for /ingester/shutdown @@ -53,6 +57,9 @@ * [11679](https://github.com/grafana/loki/pull/11679) **dannykopping** Cache: extending #11535 to align custom ingester query split with cache keys for correct caching of results. * [11143](https://github.com/grafana/loki/pull/11143) **sandeepsukhani** otel: Add support for per tenant configuration for mapping otlp data to loki format * [11499](https://github.com/grafana/loki/pull/11284) **jmichalek132** Config: Adds `frontend.log-query-request-headers` to enable logging of request headers in query logs. +* [11817](https://github.com/grafana/loki/pull/11817) **ashwanthgoli** Ruler: Add support for filtering results of `/prometheus/api/v1/rules` endpoint by rule_name, rule_group, file and type. +* [11897](https://github.com/grafana/loki/pull/11897) **ashwanthgoli** Metadata: Introduces a separate split interval of `split_recent_metadata_queries_by_interval` for `recent_metadata_query_window` to help with caching recent metadata query results. +* [11970](https://github.com/grafana/loki/pull/11897) **masslessparticle** Ksonnet: Introduces memory limits to the compactor configuration to avoid unbounded memory usage. ##### Fixes * [11074](https://github.com/grafana/loki/pull/11074) **hainenber** Fix panic in lambda-promtail due to mishandling of empty DROP_LABELS env var. @@ -65,6 +72,7 @@ * [11657](https://github.com/grafana/loki/pull/11657) **ashwanthgoli** Log results cache: compose empty response based on the request being served to avoid returning incorrect limit or direction. * [11587](https://github.com/grafana/loki/pull/11587) **trevorwhitney** Fix semantics of label parsing logic of metrics and logs queries. Both only parse the first label if multiple extractions into the same label are requested. * [11776](https://github.com/grafana/loki/pull/11776) **ashwanthgoli** Background Cache: Fixes a bug that is causing the background queue size to be incremented twice for each enqueued item. +* [11921](https://github.com/grafana/loki/pull/11921) **paul1r**: Parsing: String array elements were not being parsed correctly in JSON processing ##### Changes @@ -121,6 +129,20 @@ * [10542](https://github.com/grafana/loki/pull/10542) **chaudum**: Remove legacy deployment mode for ingester (Deployment, without WAL) and instead always run them as StatefulSet. +## [2.8.10](https://github.com/grafana/loki/compare/v2.8.9...v2.8.10) (2024-02-28) + + +### Bug Fixes + +* image tag from env and pin release to v1.11.5 ([#12073](https://github.com/grafana/loki/issues/12073)) ([8e11cd7](https://github.com/grafana/loki/commit/8e11cd7a8222a64d60bff30a41e399ddbda3372e)) + +## [2.8.9](https://github.com/grafana/loki/compare/v2.8.8...v2.8.9) (2024-02-23) + + +### Bug Fixes + +* bump alpine base image and go to fix CVEs ([#12026](https://github.com/grafana/loki/issues/12026)) ([196650e](https://github.com/grafana/loki/commit/196650e4c119249016df85a50a2cced521cbe9be)) + ## 2.9.2 (2023-10-16) ### All Changes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b643a46ddf6f..94d664954f6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,6 +14,8 @@ LIDs must be created as a pull request using [this template](docs/sources/commun ## Pull Request Prerequisites/Checklist +**NOTE:** The Loki team has adopted the use of [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) for commit messages. + 1. Your PR title is in the form `: Your change`. 1. It does not end the title with punctuation. It will be added in the changelog. 1. It starts with an imperative verb. Example: Fix the latency between System A and System B. @@ -36,6 +38,8 @@ LIDs must be created as a pull request using [this template](docs/sources/commun Please document clearly what changed AND what needs to be done in the upgrade guide. +**NOTE:** A member of the Loki repo maintainers must approve and run the continuous integration (CI) workflows for community contributions. + ## Setup A common problem arises in local environments when you want your module to use a locally modified dependency: @@ -157,3 +161,8 @@ To get a local preview of the documentation: Then you can go to Docker Desktop settings and open the resources, add the temporary directory path `/tmp`. > Note that `make docs` uses a lot of memory. If it crashes, increase the memory allocated to Docker and try again. + +Also note that PRs are merged to the main branch. If your changes need to be immediately published to the latest release, you must add the appropriate backport label to your PR, for example, `backport-release-2.9.x`. If the changes in your PR can be automatically backported, the backport label will trigger GrafanaBot to create the backport PR, otherwise you will need to create a PR to manually backport your changes. + +* [Latest release](https://grafana.com/docs/loki/latest/) +* [Upcoming release](https://grafana.com/docs/loki/next/), at the tip of the main branch diff --git a/Makefile b/Makefile index 2acf8b428504..4789bf7e319d 100644 --- a/Makefile +++ b/Makefile @@ -325,8 +325,12 @@ lint: ## run linters ######## test: all ## run the unit tests - $(GOTEST) -covermode=atomic -coverprofile=coverage.txt -p=4 ./... | sed "s:$$: ${DRONE_STEP_NAME} ${DRONE_SOURCE_BRANCH}:" | tee test_results.txt - cd tools/lambda-promtail/ && $(GOTEST) -covermode=atomic -coverprofile=lambda-promtail-coverage.txt -p=4 ./... | sed "s:$$: ${DRONE_STEP_NAME} ${DRONE_SOURCE_BRANCH}:" | tee lambda_promtail_test_results.txt + $(GOTEST) -covermode=atomic -coverprofile=coverage.txt -p=4 ./... | tee test_results.txt + cd tools/lambda-promtail/ && $(GOTEST) -covermode=atomic -coverprofile=lambda-promtail-coverage.txt -p=4 ./... | tee lambda_promtail_test_results.txt + +test-integration: + $(GOTEST) -count=1 -v -tags=integration -timeout 10m ./integration + compare-coverage: ./tools/diff_coverage.sh $(old) $(new) $(packages) @@ -812,7 +816,7 @@ validate-example-configs: loki for f in $$(grep -rL $(EXAMPLES_SKIP_VALIDATION_FLAG) $(EXAMPLES_YAML_PATH)/*.yaml); do echo "Validating provided example config: $$f" && ./cmd/loki/loki -config.file=$$f -verify-config || exit 1; done validate-dev-cluster-config: loki - ./cmd/loki/loki -config.file=./tools/dev/loki-boltdb-storage-s3/config/loki.yaml -verify-config + ./cmd/loki/loki -config.file=./tools/dev/loki-tsdb-storage-s3/config/loki.yaml -verify-config # Dynamically generate ./docs/sources/configure/examples.md using the example configs that we provide. # This target should be run if any of our example configs change. @@ -863,3 +867,7 @@ snyk: loki-image build-image .PHONY: scan-vulnerabilities scan-vulnerabilities: trivy snyk + +.PHONY: release-workflows +release-workflows: + jsonnet -SJ .github/vendor -m .github/workflows .github/release-workflows.jsonnet diff --git a/clients/cmd/fluent-bit/Dockerfile b/clients/cmd/fluent-bit/Dockerfile index 563614a75f52..f0dfbc90c36a 100644 --- a/clients/cmd/fluent-bit/Dockerfile +++ b/clients/cmd/fluent-bit/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.3-bullseye AS builder +FROM golang:1.22.0-bullseye AS builder COPY . /src diff --git a/clients/pkg/promtail/targets/cloudflare/target.go b/clients/pkg/promtail/targets/cloudflare/target.go index b64e33da4bc2..19d1f1875827 100644 --- a/clients/pkg/promtail/targets/cloudflare/target.go +++ b/clients/pkg/promtail/targets/cloudflare/target.go @@ -8,13 +8,13 @@ import ( "sync" "time" - "github.com/buger/jsonparser" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/cloudflare-go" "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/concurrency" "github.com/grafana/dskit/multierror" + "github.com/grafana/jsonparser" "github.com/prometheus/common/model" "go.uber.org/atomic" diff --git a/clients/pkg/promtail/targets/lokipush/pushtarget.go b/clients/pkg/promtail/targets/lokipush/pushtarget.go index c981de0de3dd..88c7859bd36e 100644 --- a/clients/pkg/promtail/targets/lokipush/pushtarget.go +++ b/clients/pkg/promtail/targets/lokipush/pushtarget.go @@ -111,7 +111,7 @@ func (t *PushTarget) run() error { func (t *PushTarget) handleLoki(w http.ResponseWriter, r *http.Request) { logger := util_log.WithContext(r.Context(), util_log.Logger) userID, _ := tenant.TenantID(r.Context()) - req, err := push.ParseRequest(logger, userID, r, nil, nil, push.ParseLokiRequest) + req, err := push.ParseRequest(logger, userID, r, nil, push.EmptyLimits{}, push.ParseLokiRequest, nil) if err != nil { level.Warn(t.logger).Log("msg", "failed to parse incoming push request", "err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/cmd/loki/loki-local-with-memcached.yaml b/cmd/loki/loki-local-with-memcached.yaml index d1b0ae1c2493..a2f4336cdd48 100644 --- a/cmd/loki/loki-local-with-memcached.yaml +++ b/cmd/loki/loki-local-with-memcached.yaml @@ -22,6 +22,17 @@ query_range: cache_results: true cache_volume_results: true cache_series_results: true + cache_instant_metric_results: true + instant_metric_query_split_align: true + instant_metric_results_cache: + cache: + default_validity: 12h + memcached_client: + consistent_hash: true + addresses: "dns+localhost:11211" + max_idle_conns: 16 + timeout: 500ms + update_interval: 1m series_results_cache: cache: default_validity: 12h diff --git a/docs/README.md b/docs/README.md index a3aa4414ad09..569889c2d645 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,7 @@ Some key things to know about the Loki documentation source: - While you can view the documentation in GitHub, GitHub does not render the images or links correctly and cannot render the Hugo specific shortcodes. To read the Loki documentation, see the [Documentation Site](https://grafana.com/docs/loki/latest/). - If you have a trivial fix or improvement, go ahead and create a pull request. - If you plan to do something more involved, for example creating a new topic, discuss your ideas on the relevant GitHub issue. +- Pull requests are merged to main, and published to [Upcoming release](https://grafana.com/docs/loki/next/). If your change needs to be published to the [Latest release](https://grafana.com/docs/loki/latest/) before the next Loki release (that is, it needs to be published immediately), add the appropriate backport label to your PR. ## Contributing @@ -26,16 +27,26 @@ If you have a GitHub account and you're just making a small fix, for example fix 2. Click the pencil icon. 3. Enter your changes. 4. Click **Commit changes**. GitHub creates a pull request for you. -5. If this is your first contribution to the Loki repository, you will need to sign the Contributor License Agreement (CLA) before your PR can be accepted. -6. Add the `type/docs` label to identify your PR as a docs contribution. -7. If your contribution needs to be added to the current release or previous releases, apply the appropriate `backport` label. You can find more information about backporting in the [Writers' toolkit](https://grafana.com/docs/writers-toolkit/review/backporting/). +5. The Loki team uses [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) for commit messages. Make sure your commit messages for doc updates start with `doc:`. +6. If this is your first contribution to the Loki repository, you will need to sign the Contributor License Agreement (CLA) before your PR can be accepted. +**NOTE:** A member of the Loki repo maintainers must approve and run the continuous integration (CI) workflows for community contributions. +7. Add the `type/docs` label to identify your PR as a docs contribution. This helps the documentation team track our work. +8. If your contribution needs to be added to the current release or previous releases, apply the appropriate `backport` label. You can find more information about backporting in the [Writers' toolkit](https://grafana.com/docs/writers-toolkit/review/backporting/). For larger contributions, for example documenting a new feature or adding a new topic, consider running the project locally to see how the changes look like before making a pull request. -The docs team has created a [Writer's Toolkit](https://grafana.com/docs/writers-toolkit/) that documents how we write documentation at Grafana Labs. The Writer's toolkit contains information about how we structure documentation at Grafana, including templates for different types of topics, information about Hugo shortcodes that extend markdown to add additional features, and information about linters and other tools that we use to write documentation. The Writers' Toolkit also includes our [Style Guide](https://grafana.com/docs/writers-toolkit/write/style-guide/). +The docs team has created a [Writers' Toolkit](https://grafana.com/docs/writers-toolkit/) that documents how we write documentation at Grafana Labs. Writers' Toolkit contains information about how we structure documentation at Grafana, including templates for different types of topics, information about Hugo shortcodes that extend markdown to add additional features, and information about linters and other tools that we use to write documentation. Writers' Toolkit also includes our [Style Guide](https://grafana.com/docs/writers-toolkit/write/style-guide/). Note that in Hugo the structure of the documentation is based on the folder structure of the documentation repository. The URL structure is generated based on the folder structure and file names. Try to avoid moving or renaming files, as this will break cross-references to those files. If you must move or rename files, run `make docs` as described below to find and fix broken links before you submit your pull request. +## Shared content + +**NOTE:** As of Loki/GEL 3.0, there will be shared files between the Loki docs and the GEL docs. The Grafana Enterprise Logs documentation will pull in content from the Loki repo when publishing the GEL docs. Files that are shared between the two doc sets will contain a comment indicating that the content is shared. + +For more information about shared content, see the [reuse content](https://grafana.com/docs/writers-toolkit/write/reuse-content/) section of the Writers' Toolkit. + +For more information about building and testing documentation, see the [build and review](https://grafana.com/docs/writers-toolkit/review/) section of the Writers' Toolkit. + ## Testing documentation Loki uses the static site generator [Hugo](https://gohugo.io/) to generate the documentation. The Loki repository uses a continuous integration (CI) action to sync documentation to the [Grafana website](https://grafana.com/docs/loki/latest). The CI is triggered on every merge to main in the `docs` subfolder. diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index 283a2c9dd59a..70891a044841 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -886,6 +886,28 @@ volume_results_cache: # CLI flag: -frontend.volume-results-cache.compression [compression: | default = ""] +# Cache instant metric query results. +# CLI flag: -querier.cache-instant-metric-results +[cache_instant_metric_results: | default = false] + +# If a cache config is not specified and cache_instant_metric_results is true, +# the config for the results cache is used. +instant_metric_results_cache: + # The cache block configures the cache backend. + # The CLI flags prefix for this block configuration is: + # frontend.instant-metric-results-cache + [cache: ] + + # Use compression in cache. The default is an empty value '', which disables + # compression. Supported values are: 'snappy' and ''. + # CLI flag: -frontend.instant-metric-results-cache.compression + [compression: | default = ""] + +# Whether to align the splits of instant metric query with splitByInterval and +# query's exec time. Useful when instant_metric_cache is enabled +# CLI flag: -querier.instant-metric-query-split-align +[instant_metric_query_split_align: | default = false] + # Cache series query results. # CLI flag: -querier.cache-series-results [cache_series_results: | default = false] @@ -1274,6 +1296,10 @@ remote_write: # CLI flag: -ruler.remote-write.config-refresh-period [config_refresh_period: | default = 10s] + # Add X-Scope-OrgID header in remote write requests. + # CLI flag: -ruler.remote-write.add-org-id-header + [add_org_id_header: | default = true] + # Configuration for rule evaluation. evaluation: # The evaluation mode for the ruler. Can be either 'local' or 'remote'. If set @@ -2327,27 +2353,26 @@ bloom_shipper: [max_tasks_enqueued_per_tenant: | default = 10000] blocks_cache: - # Whether embedded cache is enabled. - # CLI flag: -blocks-cache.enabled + # Cache for bloom blocks. Whether embedded cache is enabled. + # CLI flag: -bloom.blocks-cache.enabled [enabled: | default = false] - # Maximum memory size of the cache in MB. - # CLI flag: -blocks-cache.max-size-mb + # Cache for bloom blocks. Maximum memory size of the cache in MB. + # CLI flag: -bloom.blocks-cache.max-size-mb [max_size_mb: | default = 100] - # Maximum number of entries in the cache. - # CLI flag: -blocks-cache.max-size-items + # Cache for bloom blocks. Maximum number of entries in the cache. + # CLI flag: -bloom.blocks-cache.max-size-items [max_size_items: | default = 0] - # The time to live for items in the cache before they get purged. - # CLI flag: -blocks-cache.ttl - [ttl: | default = 0s] + # Cache for bloom blocks. The time to live for items in the cache before + # they get purged. + # CLI flag: -bloom.blocks-cache.ttl + [ttl: | default = 24h] - # During this period the process waits until the directory becomes not used - # and only after this it will be deleted. If the timeout is reached, the - # directory is force deleted. - # CLI flag: -blocks-cache.remove-directory-graceful-period - [remove_directory_graceful_period: | default = 5m] + # The cache block configures the cache backend. + # The CLI flags prefix for this block configuration is: bloom.metas-cache + [metas_cache: ] ``` ### chunk_store_config @@ -2642,14 +2667,27 @@ ring: # CLI flag: -bloom-compactor.enabled [enabled: | default = false] -# Directory where files can be downloaded for compaction. -# CLI flag: -bloom-compactor.working-directory -[working_directory: | default = ""] - # Interval at which to re-run the compaction operation. # CLI flag: -bloom-compactor.compaction-interval [compaction_interval: | default = 10m] +# How many index periods (days) to wait before compacting a table. This can be +# used to lower cost by not re-writing data to object storage too frequently +# since recent data changes more often. +# CLI flag: -bloom-compactor.min-table-compaction-period +[min_table_compaction_period: | default = 1] + +# How many index periods (days) to wait before compacting a table. This can be +# used to lower cost by not trying to compact older data which doesn't change. +# This can be optimized by aligning it with the maximum +# `reject_old_samples_max_age` setting of any tenant. +# CLI flag: -bloom-compactor.max-table-compaction-period +[max_table_compaction_period: | default = 7] + +# Number of workers to run in parallel for compaction. +# CLI flag: -bloom-compactor.worker-parallelism +[worker_parallelism: | default = 1] + # Minimum backoff time between retries. # CLI flag: -bloom-compactor.compaction-retries-min-backoff [compaction_retries_min_backoff: | default = 10s] @@ -2895,6 +2933,37 @@ The `limits_config` block configures global and per-tenant limits in Loki. # CLI flag: -querier.split-metadata-queries-by-interval [split_metadata_queries_by_interval: | default = 1d] +# Experimental. Split interval to use for the portion of metadata request that +# falls within `recent_metadata_query_window`. Rest of the request which is +# outside the window still uses `split_metadata_queries_by_interval`. If set to +# 0, the entire request defaults to using a split interval of +# `split_metadata_queries_by_interval.`. +# CLI flag: -experimental.querier.split-recent-metadata-queries-by-interval +[split_recent_metadata_queries_by_interval: | default = 1h] + +# Experimental. Metadata query window inside which +# `split_recent_metadata_queries_by_interval` gets applied, portion of the +# metadata request that falls in this window is split using +# `split_recent_metadata_queries_by_interval`. The value 0 disables using a +# different split interval for recent metadata queries. +# +# This is added to improve cacheability of recent metadata queries. Query split +# interval also determines the interval used in cache key. The default split +# interval of 24h is useful for caching long queries, each cache key holding 1 +# day's results. But metadata queries are often shorter than 24h, to cache them +# effectively we need a smaller split interval. `recent_metadata_query_window` +# along with `split_recent_metadata_queries_by_interval` help configure a +# shorter split interval for recent metadata queries. +# CLI flag: -experimental.querier.recent-metadata-query-window +[recent_metadata_query_window: | default = 0s] + +# Split instant metric queries by a time interval and execute in parallel. The +# value 0 disables splitting instant metric queries by time. This also +# determines how cache keys are chosen when instant metric query result caching +# is enabled. +# CLI flag: -querier.split-instant-metric-queries-by-interval +[split_instant_metric_queries_by_interval: | default = 1h] + # Interval to use for time-based splitting when a request is within the # `query_ingesters_within` window; defaults to `split-queries-by-interval` by # setting to 0. @@ -3115,7 +3184,7 @@ shard_streams: # Skip factor for the n-grams created when computing blooms from log lines. # CLI flag: -bloom-compactor.ngram-skip -[bloom_ngram_skip: | default = 0] +[bloom_ngram_skip: | default = 1] # Scalable Bloom Filter desired false-positive rate. # CLI flag: -bloom-compactor.false-positive-rate @@ -3129,6 +3198,12 @@ shard_streams: # CLI flag: -bloom-gateway.cache-key-interval [bloom_gateway_cache_key_interval: | default = 15m] +# The maximum bloom block size. A value of 0 sets an unlimited size. Default is +# 200MB. The actual block size might exceed this limit since blooms will be +# added to blocks until the block exceeds the maximum block size. +# CLI flag: -bloom-compactor.max-block-size +[bloom_compactor_max_block_size: | default = 200MB] + # Allow user to send structured metadata in push payload. # CLI flag: -validation.allow-structured-metadata [allow_structured_metadata: | default = false] @@ -3143,14 +3218,22 @@ shard_streams: # OTLP log ingestion configurations otlp_config: + # Configuration for resource attributes to store them as index labels or + # Structured Metadata or drop them altogether resource_attributes: - [ignore_defaults: ] + # Configure whether to ignore the default list of resource attributes to be + # stored as index labels and only use the given resource attributes config + [ignore_defaults: | default = false] - [attributes: ] + [attributes_config: ] - [scope_attributes: ] + # Configuration for scope attributes to store them as Structured Metadata or + # drop them altogether + [scope_attributes: ] - [log_attributes: ] + # Configuration for log attributes to store them as Structured Metadata or + # drop them altogether + [log_attributes: ] ``` ### frontend_worker @@ -4346,8 +4429,10 @@ The TLS configuration. The cache block configures the cache backend. The supported CLI flags `` used to reference this configuration block are: - `bloom-gateway-client.cache` +- `bloom.metas-cache` - `frontend` - `frontend.index-stats-results-cache` +- `frontend.instant-metric-results-cache` - `frontend.label-results-cache` - `frontend.series-results-cache` - `frontend.volume-results-cache` @@ -4577,7 +4662,7 @@ chunks: [tags: ] # How many shards will be created. Only used if schema is v10 or greater. -[row_shards: ] +[row_shards: | default = 16] ``` ### aws_storage_config @@ -5292,6 +5377,24 @@ Named store from this example can be used by setting object_store to store-1 in [cos: ] ``` +### attributes_config + +Define actions for matching OpenTelemetry (OTEL) attributes. + +```yaml +# Configures action to take on matching attributes. It allows one of +# [structured_metadata, drop] for all attribute types. It additionally allows +# index_label action for resource attributes +[action: | default = ""] + +# List of attributes to configure how to store them or drop them altogether +[attributes: ] + +# Regex to choose attributes to configure how to store them or drop them +# altogether +[regex: ] +``` + ## Runtime Configuration file Loki has a concept of "runtime config" file, which is simply a file that is reloaded while Loki is running. It is used by some Loki components to allow operator to change some aspects of Loki configuration without restarting it. File is specified by using `-runtime-config.file=` flag and reload period (which defaults to 10 seconds) can be changed by `-runtime-config.reload-period=` flag. Previously this mechanism was only used by limits overrides, and flags were called `-limits.per-user-override-config=` and `-limits.per-user-override-period=10s` respectively. These are still used, if `-runtime-config.file=` is not specified. diff --git a/docs/sources/get-started/labels/structured-metadata.md b/docs/sources/get-started/labels/structured-metadata.md index db335e771231..e199402e0b00 100644 --- a/docs/sources/get-started/labels/structured-metadata.md +++ b/docs/sources/get-started/labels/structured-metadata.md @@ -5,25 +5,19 @@ description: Describes how to enable structure metadata for logs and how to quer --- # What is structured metadata -{{% admonition type="warning" %}} -Structured metadata is an experimental feature and is subject to change in future releases of Grafana Loki. -{{% /admonition %}} - {{% admonition type="warning" %}} Structured metadata was added to chunk format V4 which is used if the schema version is greater or equal to `13`. (See [Schema Config]({{< relref "../../storage#schema-config" >}}) for more details about schema versions. ) {{% /admonition %}} -One of the powerful features of Loki is parsing logs at query time to extract metadata and build labels out of it. -However, the parsing of logs at query time comes with a cost which can be significantly high for, as an example, -large JSON blobs or a poorly written query using complex regex patterns. +Selecting proper, low cardinality labels is critical to operating and querying Loki effectively. Some metadata, especially infrastructure related metadata, can be difficult to embed in log lines, and is too high cardinality to effectively store as indexed labels (and therefore reducing performance of the index). -In addition, the data extracted from logs at query time is usually high cardinality, which can’t be stored -in the index as it would increase the cardinality too much, and therefore reduce the performance of the index. - -Structured metadata is a way to attach metadata to logs without indexing them. Examples of useful metadata are -trace IDs, user IDs, and any other label that is often used in queries but has high cardinality and is expensive +Structured metadata is a way to attach metadata to logs without indexing them or including them in the log line content itself. Examples of useful metadata are +kubernetes pod names, process ID's, or any other label that is often used in queries but has high cardinality and is expensive to extract at query time. +Structured metadata can also be used to query commonly needed metadata from log lines without needing to apply a parser at query time. Large json blobs or a poorly written query using complex regex patterns, for example, come with a high performance cost. Examples of useful metadata include trace IDs or user IDs. + + ## Attaching structured metadata to log lines You have the option to attach structured metadata to log lines in the push payload along with each log line and the timestamp. @@ -34,25 +28,37 @@ See the [Promtail: Structured metadata stage]({{< relref "../../send-data/promta With Loki version 1.2.0, support for structured metadata has been added to the Logstash output plugin. For more information, see [logstash]({{< relref "../../send-data/logstash/_index.md" >}}). +{{% admonition type="warning" %}} +There are defaults for how much structured metadata can be attached per log line. +``` +# Maximum size accepted for structured metadata per log line. +# CLI flag: -limits.max-structured-metadata-size +[max_structured_metadata_size: | default = 64KB] + +# Maximum number of structured metadata entries per log line. +# CLI flag: -limits.max-structured-metadata-entries-count +[max_structured_metadata_entries_count: | default = 128] +``` +{{% /admonition %}} + ## Querying structured metadata Structured metadata is extracted automatically for each returned log line and added to the labels returned for the query. You can use labels of structured metadata to filter log line using a [label filter expression]({{< relref "../../query/log_queries#label-filter-expression" >}}). -For example, if you have a label `trace_id` attached to some of your log lines as structured metadata, you can filter log lines using: +For example, if you have a label `pod` attached to some of your log lines as structured metadata, you can filter log lines using: ```logql -{job="example"} | trace_id="0242ac120002" +{job="example"} | pod="myservice-abc1234-56789"` ``` Of course, you can filter by multiple labels of structured metadata at the same time: ```logql -{job="example"} | trace_id="0242ac120002" | user_id="superUser123" +{job="example"} | pod="myservice-abc1234-56789" | trace_id="0242ac120002" ``` -Note that since structured metadata is extracted automatically to the results labels, some metric queries might return -an error like `maximum of series (50000) reached for a single query`. You can use the [Keep]({{< relref "../../query/log_queries#keep-labels-expression" >}}) and [Drop]({{< relref "../../query/log_queries#drop-labels-expression" >}}) stages to filter out labels that you don't need. +Note that since structured metadata is extracted automatically to the results labels, some metric queries might return an error like `maximum of series (50000) reached for a single query`. You can use the [Keep]({{< relref "../../query/log_queries#keep-labels-expression" >}}) and [Drop]({{< relref "../../query/log_queries#drop-labels-expression" >}}) stages to filter out labels that you don't need. For example: ```logql diff --git a/docs/sources/get-started/quick-start.md b/docs/sources/get-started/quick-start.md index cdca6858f271..70cbfc2c57d2 100644 --- a/docs/sources/get-started/quick-start.md +++ b/docs/sources/get-started/quick-start.md @@ -11,27 +11,27 @@ If you want to experiment with Loki, you can run Loki locally using the Docker C The Docker Compose configuration instantiates the following components, each in its own container: -- **Flog** a sample application which generates log lines. -- **Promtail** which scrapes the log lines from Flog, and pushes them to Loki through the gateway. +- **flog** a sample application which generates log lines. [flog](https://github.com/mingrammer/flog) is a log generator for common log formats. +- **Promtail** which scrapes the log lines from flog, and pushes them to Loki through the gateway. - **Gateway** (NGINX) which receives requests and redirects them to the appropriate container based on the request's URL. - One Loki **read** component. - One Loki **write** component. - **Minio** an S3-compatible object store which Loki uses to store its index and chunks. - **Grafana** which provides visualization of the log lines captured within Loki. -{{< figure max-width="75%" src="/media/docs/loki/get-started-flog.png" caption="Getting started sample application" alt="Getting started sample application">}} +{{< figure max-width="75%" src="/media/docs/loki/get-started-flog-v2.png" caption="Getting started sample application" alt="Getting started sample application">}} ## Installing Loki and collecting sample logs Prerequisites -- Docker -- Docker Compose +- [Docker](https://docs.docker.com/install) +- [Docker Compose](https://docs.docker.com/compose/install) {{% admonition type="note" %}} -Note that this quick start assumes you are running Linux. +This quick start assumes you are running Linux. {{% /admonition %}} -**Steps:** +**To install Loki locally, follow these steps:** 1. Create a directory called `evaluate-loki` for the demo environment. Make `evaluate-loki` your current working directory: @@ -80,11 +80,11 @@ Once you have collected logs, you will want to view them. You can view your log The test environment includes [Grafana](https://grafana.com/docs/grafana/latest/), which you can use to query and observe the sample logs generated by the flog application. You can access the Grafana cluster by navigating to [http://localhost:3000](http://localhost:3000). The Grafana instance provided with this demo has a Loki [datasource](https://grafana.com/docs/grafana/latest/datasources/loki/) already configured. -{{< figure src="/media/docs/loki/grafana-query-builder.png" caption="Grafana Explore" alt="Grafana Explore">}} + {{< figure src="/media/docs/loki/grafana-query-builder-v2.png" caption="Grafana Explore" alt="Grafana Explore">}} 1. From the Grafana main menu, click the **Explore** icon (1) to launch the Explore tab. To learn more about Explore, refer the [Explore](https://grafana.com/docs/grafana/latest/explore/) documentation. -1. From the menu in the dashboard header (2), select the Loki data source. This displays the Loki query editor. In the query editor you use the Loki query language, [LogQL](https://grafana.com/docs/loki/latest/query/), to query your logs. +1. From the menu in the dashboard header, select the Loki data source (2). This displays the Loki query editor. In the query editor you use the Loki query language, [LogQL](https://grafana.com/docs/loki/latest/query/), to query your logs. To learn more about the query editor, refer to the [query editor documentation](https://grafana.com/docs/grafana/latest/datasources/loki/query-editor/). 1. The Loki query editor has two modes (3): @@ -96,7 +96,7 @@ Once you have collected logs, you will want to view them. You can view your log 1. Click **Code** (3) to work in Code mode in the query editor. - Here are some basic sample queries to get you started using LogQL. Note that these queries assume that you followed the instructions to create a directory called `evaluate-loki`. If you installed in a different directory, you’ll need to modify these queries to match your installation directory. After copying any of these queries into the query editor, click **Run Query** (6) to execute the query. + Here are some basic sample queries to get you started using LogQL. Note that these queries assume that you followed the instructions to create a directory called `evaluate-loki`. If you installed in a different directory, you’ll need to modify these queries to match your installation directory. After copying any of these queries into the query editor, click **Run Query** (4) to execute the query. 1. View all the log lines which have the container label "flog": ```bash @@ -126,11 +126,10 @@ Once you have collected logs, you will want to view them. You can view your log The final query above is a metric query which returns a time series. This will trigger Grafana to draw a graph of the results. You can change the type of graph for a different view of the data. Click **Bars** to view a bar graph of the data. 1. Click the **Builder** tab (3) to return to Builder mode in the query editor. - 1. In Builder view, click **Kick start your query**(4). + 1. In Builder view, click **Kick start your query**(5). 1. Expand the **Log query starters** section. 1. Select the first choice, **Parse log lines with logfmt parser**, by clicking **Use this query**. - 1. On the Explore tab, select **container** from the **Label filters** menu then select a container from the **value** menu. - 1. Click **Run Query**(6). + 1. On the Explore tab, click **Label browser**, in the dialog select a container and click **Show logs**. For a thorough introduction to LogQL, refer to the [LogQL reference](https://grafana.com/docs/loki/latest/query/). diff --git a/docs/sources/operations/query-fairness/_index.md b/docs/sources/operations/query-fairness/_index.md index 39f9ede21fba..44b3c15f8f9a 100644 --- a/docs/sources/operations/query-fairness/_index.md +++ b/docs/sources/operations/query-fairness/_index.md @@ -115,7 +115,7 @@ you would usually want to avoid this scenario and control yourself where the hea When using Grafana as the Loki user interface, you can, for example, create multiple data sources with the same tenant, but with a different additional HTTP header -`X-Loki-Scope-Actor` and restrict which Grafana user can use which data source. +`X-Loki-Actor-Path` and restrict which Grafana user can use which data source. Alternatively, if you have a proxy for authentication in front of Loki, you can pass the (hashed) user from the authentication as downstream header to Loki. diff --git a/docs/sources/reference/api.md b/docs/sources/reference/api.md index 2e48e178534d..cf384859c6a7 100644 --- a/docs/sources/reference/api.md +++ b/docs/sources/reference/api.md @@ -1178,11 +1178,15 @@ Deletes all the rule groups in a namespace (including the namespace itself). Thi ### List rules ``` -GET /prometheus/api/v1/rules +GET /prometheus/api/v1/rules?type={alert|record}&file={}&rule_group={}&rule_name={} ``` Prometheus-compatible rules endpoint to list alerting and recording rules that are currently loaded. +The `type` parameter is optional. If set, only the specified type of rule is returned. + +The `file`, `rule_group` and `rule_name` parameters are optional, and can accept multiple values. If set, the response content is filtered accordingly. + For more information, refer to the [Prometheus rules](https://prometheus.io/docs/prometheus/latest/querying/api/#rules) documentation. ### List alerts diff --git a/docs/sources/release-notes/cadence.md b/docs/sources/release-notes/cadence.md index f13781cf1c5f..ef6fbcaf072f 100644 --- a/docs/sources/release-notes/cadence.md +++ b/docs/sources/release-notes/cadence.md @@ -8,7 +8,7 @@ weight: 1 ## Stable Releases -Loki releases (this includes [Promtail](/clients/promtail), [Loki Canary](/operations/loki-canary/), etc) use the following +Loki releases (this includes [Promtail](https://grafana.com/docs/loki//send-data/promtail/), [Loki Canary](https://grafana.com/docs/loki//operations/loki-canary/), etc.) use the following naming scheme: `MAJOR`.`MINOR`.`PATCH`. - `MAJOR` (roughly once a year): these releases include large new features and possible backwards-compatibility breaks. @@ -18,14 +18,14 @@ naming scheme: `MAJOR`.`MINOR`.`PATCH`. {{% admonition type="note" %}} While our naming scheme resembles [Semantic Versioning](https://semver.org/), at this time we do not strictly follow its guidelines to the letter. Our goal is to provide regular releases that are as stable as possible, and we take backwards-compatibility -seriously. As with any software, always read the [release notes](/release-notes) and the [upgrade guide](/upgrading) whenever +seriously. As with any software, always read the [release notes](https://grafana.com/docs/loki//release-notes/) and the [upgrade guide](https://grafana.com/docs/loki//setup/upgrade/) whenever choosing a new version of Loki to install. {{% /admonition %}} New releases are based of a [weekly release](#weekly-releases) which we have vetted for stability over a number of weeks. We strongly recommend keeping up-to-date with patch releases as they are released. We post updates of new releases in the `#loki` channel -of our [Slack community](/community/getting-in-touch). +of our [Slack community](https://grafana.com/docs/loki//community/getting-in-touch/). You can find all of our releases [on GitHub](https://github.com/grafana/loki/releases) and on [Docker Hub](https://hub.docker.com/r/grafana/loki). diff --git a/docs/sources/release-notes/next.md b/docs/sources/release-notes/next.md index a2a6e8133008..1aadfcba4db1 100644 --- a/docs/sources/release-notes/next.md +++ b/docs/sources/release-notes/next.md @@ -1,16 +1,18 @@ ---- -title: V?.? -description: Version ?.? release notes -weight: 55 ---- - -# V?.? -Grafana Labs is excited to announce the release of Loki ?.?.? Here's a summary of new enhancements and important fixes: - -:warning: This a placeholder for the next release. Clean up all features listed below - -## Features and enhancements - -## Upgrade Considerations - -## Bug fixes +--- +title: V?.? +description: Version ?.? release notes +weight: 55 +--- + +# V?.? +Grafana Labs is excited to announce the release of Loki ?.?.? Here's a summary of new enhancements and important fixes: + +:warning: This a placeholder for the next release. Clean up all features listed below + +## Features and enhancements + +## Upgrade Considerations + +## Bug fixes + +- **Parse JSON String arrays properly so string elements can be retrieved**: [PR #11921](https://github.com/grafana/loki/pull/11921)] \ No newline at end of file diff --git a/docs/sources/release-notes/v2-9.md b/docs/sources/release-notes/v2-9.md index 8355dd02abf0..68d3da85bc4d 100644 --- a/docs/sources/release-notes/v2-9.md +++ b/docs/sources/release-notes/v2-9.md @@ -9,6 +9,8 @@ Grafana Labs is excited to announce the release of Loki 2.9.0 Here's a summary o ## Features and enhancements +- **Structured metadata**: The [Structured Metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/) feature, which was introduced as experimental in release 2.9.0, is generally available as of release 2.9.4. + - **Query Language Improvements**: Several improvements to the query language that speed up line parsing and regex matching. [PR #8646](https://github.com/grafana/loki/pull/8646), [PR #8659](https://github.com/grafana/loki/pull/8659), [PR #8724](https://github.com/grafana/loki/pull/8724), [PR #8734](https://github.com/grafana/loki/pull/8734), [PR #8739](https://github.com/grafana/loki/pull/8739), [PR #8763](https://github.com/grafana/loki/pull/8763), [PR #8890](https://github.com/grafana/loki/pull/8890), [PR #8914](https://github.com/grafana/loki/pull/8914) - **Remote rule evaluation**: Rule evaluation can now be handled by queriers to improve speed. [PR #8744](https://github.com/grafana/loki/pull/8744) [PR #8848](https://github.com/grafana/loki/pull/8848) @@ -33,13 +35,13 @@ Grafana Labs is excited to announce the release of Loki 2.9.0 Here's a summary o ## Bug fixes -### 2.9.1 (2023-09-14) - -* Update Docker base images to mitigate security vulnerability CVE-2022-48174 -* Fix bugs in indexshipper (`tsdb`, `boltdb-shipper`) that could result in not showing all ingested logs in query results. - ### 2.9.2 (2023-10-16) * Upgrade go to v1.21.3, golang.org/x/net to v0.17.0 and grpc-go to v1.56.3 to patch CVE-2023-39325 / CVE-2023-44487 For a full list of all changes and fixes, look at the [CHANGELOG](https://github.com/grafana/loki/blob/release-2.9.x/CHANGELOG.md). + +### 2.9.1 (2023-09-14) + +* Update Docker base images to mitigate security vulnerability CVE-2022-48174 +* Fix bugs in indexshipper (`tsdb`, `boltdb-shipper`) that could result in not showing all ingested logs in query results. diff --git a/docs/sources/send-data/otel/_index.md b/docs/sources/send-data/otel/_index.md index 915d17f75ab0..12f9cdd0e4af 100644 --- a/docs/sources/send-data/otel/_index.md +++ b/docs/sources/send-data/otel/_index.md @@ -9,10 +9,6 @@ weight: 250 # Ingesting logs to Loki using OpenTelemetry Collector -{{% admonition type="warning" %}} -OpenTelemetry logs ingestion is an experimental feature and is subject to change in future releases of Grafana Loki. -{{% /admonition %}} - Loki natively supports ingesting OpenTelemetry logs over HTTP. For ingesting logs to Loki using the OpenTelemetry Collector, you must use the [`otlphttp` exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter). @@ -69,7 +65,7 @@ service: ## Format considerations -Since the OpenTelemetry protocol differs from the Loki storage model, here is how data in the OpenTelemetry format will be mapped to the Loki data model during ingestion: +Since the OpenTelemetry protocol differs from the Loki storage model, here is how data in the OpenTelemetry format will be mapped by default to the Loki data model during ingestion, which can be changed as explained later: - Index labels: Resource attributes map well to index labels in Loki, since both usually identify the source of the logs. Because Loki has a limit of 30 index labels, we have selected the following resource attributes to be stored as index labels, while the remaining attributes are stored as [Structured Metadata]({{< relref "../../get-started/labels/structured-metadata" >}}) with each log entry: - cloud.availability_zone @@ -116,3 +112,102 @@ Things to note before ingesting OpenTelemetry logs to Loki: - Stringification of non-string Attribute values While converting Attribute values in OTLP to Index label values or Structured Metadata, any non-string values are converted to string using [AsString method from the OTEL collector lib](https://github.com/open-telemetry/opentelemetry-collector/blob/ab3d6c5b64701e690aaa340b0a63f443ff22c1f0/pdata/pcommon/value.go#L353). + +### Changing the default mapping of OTLP to Loki Format + +Loki supports [per tenant]({{< relref "../../configure#limits_config" >}}) OTLP config which lets you change the default mapping of OTLP to Loki format for each tenant. +It currently only supports changing the storage of Attributes. Here is how the config looks like: + +```yaml +# OTLP log ingestion configurations +otlp_config: + # Configuration for Resource Attributes to store them as index labels or + # Structured Metadata or drop them altogether + resource_attributes: + # Configure whether to ignore the default list of Resource Attributes to be + # stored as Index Labels and only use the given Resource Attributes config + [ignore_defaults: ] + + [attributes_config: ] + + # Configuration for Scope Attributes to store them as Structured Metadata or + # drop them altogether + [scope_attributes: ] + + # Configuration for Log Attributes to store them as Structured Metadata or + # drop them altogether + [log_attributes: ] + +attributes_config: + # Configures action to take on matching Attributes. It allows one of + # [structured_metadata, drop] for all Attribute types. It additionally allows + # index_label action for Resource Attributes + [action: | default = ""] + + # List of attributes to configure how to store them or drop them altogether + [attributes: ] + + # Regex to choose attributes to configure how to store them or drop them + # altogether + [regex: ] +``` + +Here are some example configs to change the default mapping of OTLP to Loki format: + +#### Example 1: + +```yaml +otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - service.group +``` + +With the example config, here is how various kinds of Attributes would be stored: +* Store all 17 Resource Attributes mentioned earlier and `service.group` Resource Attribute as index labels. +* Store remaining Resource Attributes as Structured Metadata. +* Store all the Scope and Log Attributes as Structured Metadata. + +#### Example 2: + +```yaml +otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + regex: service.group +``` + +With the example config, here is how various kinds of Attributes would be stored: +* **Only** store `service.group` Resource Attribute as index labels. +* Store remaining Resource Attributes as Structured Metadata. +* Store all the Scope and Log Attributes as Structured Metadata. + +#### Example 2: + +```yaml +otlp_config: + resource_attributes: + attributes_config: + - action: index_label + regex: service.group + scope_attributes: + - action: drop + attributes: + - method.name + log_attributes: + - action: structured_metadata + attributes: + - user.id + - action: drop + regex: .* +``` + +With the example config, here is how various kinds of Attributes would be stored: +* Store all 17 Resource Attributes mentioned earlier and `service.group` Resource Attribute as index labels. +* Store remaining Resource Attributes as Structured Metadata. +* Drop Scope Attribute named `method.name` and store all other Scope Attributes as Structured Metadata. +* Store Log Attribute named `user.id` as Structured Metadata and drop all other Log Attributes. \ No newline at end of file diff --git a/docs/sources/setup/install/docker.md b/docs/sources/setup/install/docker.md index 51df7f9288c8..8e9007c0fdf4 100644 --- a/docs/sources/setup/install/docker.md +++ b/docs/sources/setup/install/docker.md @@ -9,7 +9,7 @@ weight: 400 # Install Loki with Docker or Docker Compose You can install Loki and Promtail with Docker or Docker Compose if you are evaluating, testing, or developing Loki. -For production, we recommend installing with Tanka or Helm. +For production, Grafana recommends installing with Tanka or Helm. The configuration acquired with these installation instructions run Loki as a single binary. diff --git a/docs/sources/setup/install/helm/configure-storage/_index.md b/docs/sources/setup/install/helm/configure-storage/_index.md index 6a28387932d0..e51e8503fee0 100644 --- a/docs/sources/setup/install/helm/configure-storage/_index.md +++ b/docs/sources/setup/install/helm/configure-storage/_index.md @@ -20,9 +20,9 @@ This guide assumes Loki will be installed in one of the modes above and that a ` **To use a managed object store:** -1. Set the `type` of `storage` in `values.yaml` to `gcs` or `s3`. +1. In the `values.yaml` file, set the value for `storage.type` to `azure`, `gcs`, or `s3`. -2. Configure the storage client under `loki.storage.gcs` or `loki.storage.s3`. +1. Configure the storage client under `loki.storage.azure`, `loki.storage.gcs`, or `loki.storage.s3`. **To install Minio alongside Loki:** @@ -41,7 +41,7 @@ This guide assumes Loki will be installed in one of the modes above and that a ` 1. Provision an IAM role, policy and S3 bucket as described in [Storage]({{< relref "../../../../storage#aws-deployment-s3-single-store" >}}). - If the Terraform module was used note the annotation emitted by `terraform output -raw annotation`. -2. Add the IAM role annotation to the service account in `values.yaml`: +1. Add the IAM role annotation to the service account in `values.yaml`: ``` serviceAccount: @@ -49,7 +49,7 @@ This guide assumes Loki will be installed in one of the modes above and that a ` "eks.amazonaws.com/role-arn": "arn:aws:iam:::role/" ``` -3. Configure the storage: +1. Configure the storage: ``` loki: diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md index e687a560ef71..e7dbfdbdd3f6 100644 --- a/docs/sources/setup/install/helm/reference.md +++ b/docs/sources/setup/install/helm/reference.md @@ -2806,6 +2806,15 @@ true 
 null
+ + + + monitoring.selfMonitoring.grafanaAgent.resources + object + Resource requests and limits for the grafanaAgent pods + 
+{}
+
diff --git a/docs/variables.mk b/docs/variables.mk index afa0a9e86736..1ec7dbab5767 100644 --- a/docs/variables.mk +++ b/docs/variables.mk @@ -1,8 +1,5 @@ # List of projects to provide to the make-docs script. PROJECTS := loki -# Use alternative image until make-docs 3.0.0 is rolled out. -export DOCS_IMAGE := grafana/docs-base:dbd975af06 - # Set the DOC_VALIDATOR_IMAGE to match the one defined in CI. export DOC_VALIDATOR_IMAGE := $(shell sed -En 's, *image: "(grafana/doc-validator.*)",\1,p' "$(shell git rev-parse --show-toplevel)/.github/workflows/doc-validator.yml") diff --git a/go.mod b/go.mod index f8db7e46affa..5b4c32ddb214 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/aws/aws-sdk-go v1.44.321 github.com/baidubce/bce-sdk-go v0.9.141 github.com/bmatcuk/doublestar v1.3.4 - github.com/buger/jsonparser v1.1.1 github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash/v2 v2.2.0 @@ -119,12 +118,12 @@ require ( github.com/DmitriyVTitov/size v1.5.0 github.com/IBM/go-sdk-core/v5 v5.13.1 github.com/IBM/ibm-cos-sdk-go v1.10.0 - github.com/aws/smithy-go v1.11.1 - github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc + github.com/axiomhq/hyperloglog v0.0.0-20240124082744-24bca3a5b39b github.com/d4l3k/messagediff v1.2.1 github.com/efficientgo/core v1.0.0-rc.2 github.com/fsnotify/fsnotify v1.6.0 github.com/gogo/googleapis v1.4.0 + github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 github.com/heroku/x v0.0.61 github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b @@ -183,6 +182,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect + github.com/aws/smithy-go v1.11.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect diff --git a/go.sum b/go.sum index dd756d74f7c6..744c904e823c 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 h1:xsOtPAvHqhvQvBza5ohaUcfq1Lce github.com/aws/aws-sdk-go-v2/service/sts v1.16.1/go.mod h1:Aq2/Qggh2oemSfyHH+EO4UBbgWG6zFCXLHYI4ILTY7w= github.com/aws/smithy-go v1.11.1 h1:IQ+lPZVkSM3FRtyaDox41R8YS6iwPMYIreejOgPW49g= github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc h1:Keo7wQ7UODUaHcEi7ltENhbAK2VgZjfat6mLy03tQzo= -github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c= +github.com/axiomhq/hyperloglog v0.0.0-20240124082744-24bca3a5b39b h1:F3yMzKumBUQ6Fn0sYI1YQ16vQRucpZOfBQ9HXWl5+XI= +github.com/axiomhq/hyperloglog v0.0.0-20240124082744-24bca3a5b39b/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c= github.com/baidubce/bce-sdk-go v0.9.141 h1:EV5BH5lfymIGPSmYDo9xYdsVlvWAW6nFeiA6t929zBE= github.com/baidubce/bce-sdk-go v0.9.141/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= @@ -390,8 +390,6 @@ github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/caddyserver/caddy v1.0.4/go.mod h1:uruyfVsyMcDb3IOzSKsi1x0wOjy1my/PxOSTcD+24jM= @@ -1003,6 +1001,8 @@ github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 h1:xLuzPoOzdfNb/RF/I github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85/go.mod h1:crI9WX6p0IhrqB+DqIUHulRW853PaNFf7o4UprV//3I= github.com/grafana/gomemcache v0.0.0-20231204155601-7de47a8c3cb0 h1:aLBiDMjTtXx2800iCIp+8kdjIlvGX0MF/zICQMQO2qU= github.com/grafana/gomemcache v0.0.0-20231204155601-7de47a8c3cb0/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= +github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d h1:YwbJJ/PrVWVdnR+j/EAVuazdeP+Za5qbiH1Vlr+wFXs= +github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d/go.mod h1:796sq+UcONnSlzA3RtlBZ+b/hrerkZXiEmO8oMjyRwY= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= diff --git a/integration/client/client.go b/integration/client/client.go index 2e5a86aa6b3d..1ad94fd0edbb 100644 --- a/integration/client/client.go +++ b/integration/client/client.go @@ -14,9 +14,9 @@ import ( "strings" "time" - "github.com/buger/jsonparser" "github.com/gorilla/websocket" "github.com/grafana/dskit/user" + "github.com/grafana/jsonparser" "github.com/prometheus/common/config" "github.com/prometheus/prometheus/model/labels" "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/integration/cluster/cluster.go b/integration/cluster/cluster.go index 831da46f2cb9..7e978b84eb32 100644 --- a/integration/cluster/cluster.go +++ b/integration/cluster/cluster.go @@ -84,7 +84,6 @@ bloom_gateway: bloom_compactor: enabled: false - working_directory: {{.dataPath}}/bloom-compactor compactor: working_directory: {{.dataPath}}/compactor diff --git a/integration/loki_micro_services_delete_test.go b/integration/loki_micro_services_delete_test.go index 07195d919ee1..d77d7ab11508 100644 --- a/integration/loki_micro_services_delete_test.go +++ b/integration/loki_micro_services_delete_test.go @@ -1,3 +1,5 @@ +//go:build integration + package integration import ( diff --git a/integration/loki_micro_services_test.go b/integration/loki_micro_services_test.go index 3b888314cd68..67f888d41acf 100644 --- a/integration/loki_micro_services_test.go +++ b/integration/loki_micro_services_test.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/loki_rule_eval_test.go b/integration/loki_rule_eval_test.go index 41414d4aef67..025b74df5ad8 100644 --- a/integration/loki_rule_eval_test.go +++ b/integration/loki_rule_eval_test.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/loki_simple_scalable_test.go b/integration/loki_simple_scalable_test.go index 2de17e3420b8..ccbf839b6a6a 100644 --- a/integration/loki_simple_scalable_test.go +++ b/integration/loki_simple_scalable_test.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/loki_single_binary_test.go b/integration/loki_single_binary_test.go index 16bb5b36944d..31b9990d3ad0 100644 --- a/integration/loki_single_binary_test.go +++ b/integration/loki_single_binary_test.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/multi_tenant_queries_test.go b/integration/multi_tenant_queries_test.go index 76fbd63f13bd..cec967fd1318 100644 --- a/integration/multi_tenant_queries_test.go +++ b/integration/multi_tenant_queries_test.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/parse_metrics.go b/integration/parse_metrics.go index 9f2bf5fc8fc2..d2896de6e29c 100644 --- a/integration/parse_metrics.go +++ b/integration/parse_metrics.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/parse_metrics_test.go b/integration/parse_metrics_test.go index 94c19b7584ad..7af3289e36db 100644 --- a/integration/parse_metrics_test.go +++ b/integration/parse_metrics_test.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/per_request_limits_test.go b/integration/per_request_limits_test.go index 85642e0439e6..93e2c440861e 100644 --- a/integration/per_request_limits_test.go +++ b/integration/per_request_limits_test.go @@ -1,3 +1,4 @@ +//go:build integration package integration import ( diff --git a/integration/shared_test.go b/integration/shared_test.go index 61b469ecd004..12338ec99ab2 100644 --- a/integration/shared_test.go +++ b/integration/shared_test.go @@ -1,3 +1,5 @@ +//go:build integration + package integration import ( diff --git a/operator/.bingo/go.mod b/operator/.bingo/go.mod index 610249af0b0b..3aa5b7c946f5 100644 --- a/operator/.bingo/go.mod +++ b/operator/.bingo/go.mod @@ -1 +1 @@ -module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. \ No newline at end of file +module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 18e28a016efe..e6aaec29b99c 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,5 +1,13 @@ ## Main +- [12007](https://github.com/grafana/loki/pull/12007) **xperimental**: Extend Azure secret validation +- [12008](https://github.com/grafana/loki/pull/12008) **xperimental**: Support using multiple buckets with AWS STS +- [11964](https://github.com/grafana/loki/pull/11964) **xperimental**: Provide Azure region for managed credentials using environment variable +- [11920](https://github.com/grafana/loki/pull/11920) **xperimental**: Refactor handling of credentials in managed-auth mode +- [11869](https://github.com/grafana/loki/pull/11869) **periklis**: Add support for running with Google Workload Identity +- [11868](https://github.com/grafana/loki/pull/11868) **xperimental**: Integrate support for OpenShift-managed credentials in Azure +- [11854](https://github.com/grafana/loki/pull/11854) **periklis**: Allow custom audience for managed-auth on STS +- [11802](https://github.com/grafana/loki/pull/11802) **xperimental**: Add support for running with Azure Workload Identity - [11824](https://github.com/grafana/loki/pull/11824) **xperimental**: Improve messages for errors in storage secret - [11524](https://github.com/grafana/loki/pull/11524) **JoaoBraveCoding**, **periklis**: Add OpenShift cloud credentials support for AWS STS - [11513](https://github.com/grafana/loki/pull/11513) **btaani**: Add a custom metric that collects Lokistacks requiring a schema upgrade diff --git a/operator/apis/config/v1/projectconfig_types.go b/operator/apis/config/v1/projectconfig_types.go index ba7cc703c5bb..8e510b5d3ab7 100644 --- a/operator/apis/config/v1/projectconfig_types.go +++ b/operator/apis/config/v1/projectconfig_types.go @@ -52,14 +52,11 @@ type OpenShiftFeatureGates struct { // Dashboards enables the loki-mixin dashboards into the OpenShift Console Dashboards bool `json:"dashboards,omitempty"` - // ManagedAuthEnv enabled when the operator installation is on OpenShift STS clusters. + // ManagedAuthEnv is true when OpenShift-functions are enabled and the operator has detected + // that it is running with some kind of "workload identity" (AWS STS, Azure WIF) enabled. ManagedAuthEnv bool } -func (o OpenShiftFeatureGates) ManagedAuthEnabled() bool { - return o.Enabled && o.ManagedAuthEnv -} - // FeatureGates is the supported set of all operator feature gates. type FeatureGates struct { // ServiceMonitors enables creating a Prometheus-Operator managed ServiceMonitor diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index a50fb48b187e..b652ba0c7a4d 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -1174,6 +1174,27 @@ type LokiStackComponentStatus struct { Ruler PodStatusMap `json:"ruler,omitempty"` } +// CredentialMode represents the type of authentication used for accessing the object storage. +// +// +kubebuilder:validation:Enum=static;token;managed +type CredentialMode string + +const ( + // CredentialModeStatic represents the usage of static, long-lived credentials stored in a Secret. + // This is the default authentication mode and available for all supported object storage types. + CredentialModeStatic CredentialMode = "static" + // CredentialModeToken represents the usage of short-lived tokens retrieved from a credential source. + // In this mode the static configuration does not contain credentials needed for the object storage. + // Instead, they are generated during runtime using a service, which allows for shorter-lived credentials and + // much more granular control. This authentication mode is not supported for all object storage types. + CredentialModeToken CredentialMode = "token" + // CredentialModeManaged represents the usage of short-lived tokens retrieved from a credential source. + // This mode is similar to CredentialModeToken,but instead of having a user-configured credential source, + // it is configured by the environment, for example the Cloud Credential Operator in OpenShift. + // This mode is only supported for certain object storage types in certain runtime environments. + CredentialModeManaged CredentialMode = "managed" +) + // LokiStackStorageStatus defines the observed state of // the Loki storage configuration. type LokiStackStorageStatus struct { @@ -1183,6 +1204,12 @@ type LokiStackStorageStatus struct { // +optional // +kubebuilder:validation:Optional Schemas []ObjectStorageSchema `json:"schemas,omitempty"` + + // CredentialMode contains the authentication mode used for accessing the object storage. + // + // +optional + // +kubebuilder:validation:Optional + CredentialMode CredentialMode `json:"credentialMode,omitempty"` } // LokiStackStatus defines the observed state of LokiStack diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index 4b20f814804a..ad2b2e1bc93b 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.5.0 - createdAt: "2024-01-25T11:08:43Z" + createdAt: "2024-02-12T14:48:52Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. features.operators.openshift.io/disconnected: "true" @@ -1472,6 +1472,7 @@ spec: - delete - get - list + - update - watch - apiGroups: - config.openshift.io diff --git a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml index a8033e692214..e1a7e5578965 100644 --- a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -4064,6 +4064,14 @@ spec: description: Storage provides summary of all changes that have occurred to the storage configuration. properties: + credentialMode: + description: CredentialMode contains the authentication mode used + for accessing the object storage. + enum: + - static + - token + - managed + type: string schemas: description: Schemas is a list of schemas which have been applied to the LokiStack. diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index 81575be404e8..b372a29504e3 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.5.0 - createdAt: "2024-01-25T11:08:41Z" + createdAt: "2024-02-12T14:48:49Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -1452,6 +1452,7 @@ spec: - delete - get - list + - update - watch - apiGroups: - config.openshift.io diff --git a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml index 8b86ddfff8bb..f92665f5095d 100644 --- a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml @@ -4064,6 +4064,14 @@ spec: description: Storage provides summary of all changes that have occurred to the storage configuration. properties: + credentialMode: + description: CredentialMode contains the authentication mode used + for accessing the object storage. + enum: + - static + - token + - managed + type: string schemas: description: Schemas is a list of schemas which have been applied to the LokiStack. diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index b79f4ea7a2f4..8026bbcd0fc4 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:0.1.0 - createdAt: "2024-01-25T11:08:45Z" + createdAt: "2024-02-12T14:48:55Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -165,7 +165,7 @@ metadata: features.operators.openshift.io/proxy-aware: "true" features.operators.openshift.io/tls-profiles: "true" features.operators.openshift.io/token-auth-aws: "true" - features.operators.openshift.io/token-auth-azure: "false" + features.operators.openshift.io/token-auth-azure: "true" features.operators.openshift.io/token-auth-gcp: "false" olm.skipRange: '>=5.7.0-0 <5.9.0' operatorframework.io/cluster-monitoring: "true" @@ -1457,6 +1457,7 @@ spec: - delete - get - list + - update - watch - apiGroups: - config.openshift.io diff --git a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml index f121699ec6fb..3163752ad36f 100644 --- a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -4064,6 +4064,14 @@ spec: description: Storage provides summary of all changes that have occurred to the storage configuration. properties: + credentialMode: + description: CredentialMode contains the authentication mode used + for accessing the object storage. + enum: + - static + - token + - managed + type: string schemas: description: Schemas is a list of schemas which have been applied to the LokiStack. diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 4661097811b7..d603ef2a9b64 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -4046,6 +4046,14 @@ spec: description: Storage provides summary of all changes that have occurred to the storage configuration. properties: + credentialMode: + description: CredentialMode contains the authentication mode used + for accessing the object storage. + enum: + - static + - token + - managed + type: string schemas: description: Schemas is a list of schemas which have been applied to the LokiStack. diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml index 0e724292edbb..48a221736e2d 100644 --- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml @@ -21,7 +21,7 @@ metadata: features.operators.openshift.io/proxy-aware: "true" features.operators.openshift.io/tls-profiles: "true" features.operators.openshift.io/token-auth-aws: "true" - features.operators.openshift.io/token-auth-azure: "false" + features.operators.openshift.io/token-auth-azure: "true" features.operators.openshift.io/token-auth-gcp: "false" olm.skipRange: '>=5.7.0-0 <5.9.0' operatorframework.io/cluster-monitoring: "true" diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index 766a6d7d191e..072efd5b9912 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -56,6 +56,7 @@ rules: - delete - get - list + - update - watch - apiGroups: - config.openshift.io diff --git a/operator/controllers/loki/credentialsrequests_controller.go b/operator/controllers/loki/credentialsrequests_controller.go deleted file mode 100644 index 61d0b58423e9..000000000000 --- a/operator/controllers/loki/credentialsrequests_controller.go +++ /dev/null @@ -1,71 +0,0 @@ -package controllers - -import ( - "context" - - "github.com/go-logr/logr" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" - "github.com/grafana/loki/operator/controllers/loki/internal/lokistack" - "github.com/grafana/loki/operator/controllers/loki/internal/management/state" - "github.com/grafana/loki/operator/internal/external/k8s" - "github.com/grafana/loki/operator/internal/handlers" -) - -// CredentialsRequestsReconciler reconciles a single CredentialsRequest resource for each LokiStack request. -type CredentialsRequestsReconciler struct { - client.Client - Scheme *runtime.Scheme - Log logr.Logger -} - -// Reconcile creates a single CredentialsRequest per LokiStack for the OpenShift cloud-credentials-operator (CCO) to -// provide a managed cloud credentials Secret. On successful creation, the LokiStack resource is annotated -// with `loki.grafana.com/credentials-request-secret-ref` that refers to the secret provided by CCO. If the LokiStack -// resource is not found its accompanying CredentialsRequest resource is deleted. -func (r *CredentialsRequestsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - var stack lokiv1.LokiStack - if err := r.Client.Get(ctx, req.NamespacedName, &stack); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, handlers.DeleteCredentialsRequest(ctx, r.Client, req.NamespacedName) - } - return ctrl.Result{}, err - } - - managed, err := state.IsManaged(ctx, req, r.Client) - if err != nil { - return ctrl.Result{}, err - } - if !managed { - r.Log.Info("Skipping reconciliation for unmanaged LokiStack resource", "name", req.String()) - // Stop requeueing for unmanaged LokiStack custom resources - return ctrl.Result{}, nil - } - - secretRef, err := handlers.CreateCredentialsRequest(ctx, r.Client, req.NamespacedName) - if err != nil { - return ctrl.Result{}, err - } - - if err := lokistack.AnnotateForCredentialsRequest(ctx, r.Client, req.NamespacedName, secretRef); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *CredentialsRequestsReconciler) SetupWithManager(mgr ctrl.Manager) error { - b := ctrl.NewControllerManagedBy(mgr) - return r.buildController(k8s.NewCtrlBuilder(b)) -} - -func (r *CredentialsRequestsReconciler) buildController(bld k8s.Builder) error { - return bld. - For(&lokiv1.LokiStack{}). - Complete(r) -} diff --git a/operator/controllers/loki/credentialsrequests_controller_test.go b/operator/controllers/loki/credentialsrequests_controller_test.go deleted file mode 100644 index e6738c1d1796..000000000000 --- a/operator/controllers/loki/credentialsrequests_controller_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package controllers - -import ( - "context" - "testing" - - cloudcredentialsv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" - "github.com/stretchr/testify/require" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" - "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" - "github.com/grafana/loki/operator/internal/manifests/storage" -) - -func TestCredentialsRequestController_RegistersCustomResource_WithDefaultPredicates(t *testing.T) { - b := &k8sfakes.FakeBuilder{} - k := &k8sfakes.FakeClient{} - c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme} - - b.ForReturns(b) - b.OwnsReturns(b) - - err := c.buildController(b) - require.NoError(t, err) - - // Require only one For-Call for the custom resource - require.Equal(t, 1, b.ForCallCount()) - - // Require For-call with LokiStack resource - obj, _ := b.ForArgsForCall(0) - require.Equal(t, &lokiv1.LokiStack{}, obj) -} - -func TestCredentialsRequestController_DeleteCredentialsRequest_WhenLokiStackNotFound(t *testing.T) { - k := &k8sfakes.FakeClient{} - c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme} - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "ns", - }, - } - - // Set managed auth environment - t.Setenv("ROLEARN", "a-role-arn") - - k.GetStub = func(_ context.Context, key types.NamespacedName, _ client.Object, _ ...client.GetOption) error { - if key.Name == r.Name && key.Namespace == r.Namespace { - return apierrors.NewNotFound(schema.GroupResource{}, "lokistack not found") - } - return nil - } - - res, err := c.Reconcile(context.Background(), r) - require.NoError(t, err) - require.Equal(t, ctrl.Result{}, res) - require.Equal(t, 1, k.DeleteCallCount()) -} - -func TestCredentialsRequestController_CreateCredentialsRequest_WhenLokiStackNotAnnotated(t *testing.T) { - k := &k8sfakes.FakeClient{} - c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme} - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "ns", - }, - } - s := lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "ns", - }, - Spec: lokiv1.LokiStackSpec{ - ManagementState: lokiv1.ManagementStateManaged, - }, - } - - // Set managed auth environment - t.Setenv("ROLEARN", "a-role-arn") - - k.GetStub = func(_ context.Context, key types.NamespacedName, out client.Object, _ ...client.GetOption) error { - if key.Name == r.Name && key.Namespace == r.Namespace { - k.SetClientObject(out, &s) - return nil - } - return apierrors.NewNotFound(schema.GroupResource{}, "lokistack not found") - } - - k.CreateStub = func(_ context.Context, o client.Object, _ ...client.CreateOption) error { - _, isCredReq := o.(*cloudcredentialsv1.CredentialsRequest) - if !isCredReq { - return apierrors.NewBadRequest("something went wrong creating a credentials request") - } - return nil - } - - k.UpdateStub = func(_ context.Context, o client.Object, _ ...client.UpdateOption) error { - stack, ok := o.(*lokiv1.LokiStack) - if !ok { - return apierrors.NewBadRequest("something went wrong creating a credentials request") - } - - _, hasSecretRef := stack.Annotations[storage.AnnotationCredentialsRequestsSecretRef] - if !hasSecretRef { - return apierrors.NewBadRequest("something went updating the lokistack annotations") - } - return nil - } - - res, err := c.Reconcile(context.Background(), r) - require.NoError(t, err) - require.Equal(t, ctrl.Result{}, res) - require.Equal(t, 1, k.CreateCallCount()) - require.Equal(t, 1, k.UpdateCallCount()) -} - -func TestCredentialsRequestController_SkipsUnmanaged(t *testing.T) { - k := &k8sfakes.FakeClient{} - c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme} - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "ns", - }, - } - - s := lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "ns", - }, - Spec: lokiv1.LokiStackSpec{ - ManagementState: lokiv1.ManagementStateUnmanaged, - }, - } - - k.GetStub = func(_ context.Context, key types.NamespacedName, out client.Object, _ ...client.GetOption) error { - if key.Name == s.Name && key.Namespace == s.Namespace { - k.SetClientObject(out, &s) - return nil - } - return apierrors.NewNotFound(schema.GroupResource{}, "something not found") - } - - res, err := c.Reconcile(context.Background(), r) - require.NoError(t, err) - require.Equal(t, ctrl.Result{}, res) -} diff --git a/operator/controllers/loki/internal/lokistack/credentialsrequest_discovery.go b/operator/controllers/loki/internal/lokistack/credentialsrequest_discovery.go deleted file mode 100644 index c911c1196eed..000000000000 --- a/operator/controllers/loki/internal/lokistack/credentialsrequest_discovery.go +++ /dev/null @@ -1,30 +0,0 @@ -package lokistack - -import ( - "context" - - "github.com/ViaQ/logerr/v2/kverrors" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/grafana/loki/operator/internal/external/k8s" - "github.com/grafana/loki/operator/internal/manifests/storage" -) - -// AnnotateForCredentialsRequest adds the `loki.grafana.com/credentials-request-secret-ref` annotation -// to the named Lokistack. If no LokiStack is found, then skip reconciliation. Or else return an error. -func AnnotateForCredentialsRequest(ctx context.Context, k k8s.Client, key client.ObjectKey, secretRef string) error { - stack, err := getLokiStack(ctx, k, key) - if stack == nil || err != nil { - return err - } - - if val, ok := stack.Annotations[storage.AnnotationCredentialsRequestsSecretRef]; ok && val == secretRef { - return nil - } - - if err := updateAnnotation(ctx, k, stack, storage.AnnotationCredentialsRequestsSecretRef, secretRef); err != nil { - return kverrors.Wrap(err, "failed to update lokistack `credentialsRequestSecretRef` annotation", "key", key) - } - - return nil -} diff --git a/operator/controllers/loki/internal/lokistack/credentialsrequest_discovery_test.go b/operator/controllers/loki/internal/lokistack/credentialsrequest_discovery_test.go deleted file mode 100644 index ef073ca853ba..000000000000 --- a/operator/controllers/loki/internal/lokistack/credentialsrequest_discovery_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package lokistack - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" - "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" - "github.com/grafana/loki/operator/internal/manifests/storage" -) - -func TestAnnotateForCredentialsRequest_ReturnError_WhenLokiStackMissing(t *testing.T) { - k := &k8sfakes.FakeClient{} - annotationVal := "ns-my-stack-aws-creds" - stackKey := client.ObjectKey{Name: "my-stack", Namespace: "ns"} - - k.GetStub = func(_ context.Context, _ types.NamespacedName, out client.Object, _ ...client.GetOption) error { - return apierrors.NewBadRequest("failed to get lokistack") - } - - err := AnnotateForCredentialsRequest(context.Background(), k, stackKey, annotationVal) - require.Error(t, err) -} - -func TestAnnotateForCredentialsRequest_DoNothing_WhenAnnotationExists(t *testing.T) { - k := &k8sfakes.FakeClient{} - - annotationVal := "ns-my-stack-aws-creds" - s := &lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "ns", - Annotations: map[string]string{ - storage.AnnotationCredentialsRequestsSecretRef: annotationVal, - }, - }, - } - stackKey := client.ObjectKeyFromObject(s) - - k.GetStub = func(_ context.Context, key types.NamespacedName, out client.Object, _ ...client.GetOption) error { - if key.Name == stackKey.Name && key.Namespace == stackKey.Namespace { - k.SetClientObject(out, s) - return nil - } - return nil - } - - err := AnnotateForCredentialsRequest(context.Background(), k, stackKey, annotationVal) - require.NoError(t, err) - require.Equal(t, 0, k.UpdateCallCount()) -} - -func TestAnnotateForCredentialsRequest_UpdateLokistack_WhenAnnotationMissing(t *testing.T) { - k := &k8sfakes.FakeClient{} - - annotationVal := "ns-my-stack-aws-creds" - s := &lokiv1.LokiStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "ns", - Annotations: map[string]string{}, - }, - } - stackKey := client.ObjectKeyFromObject(s) - - k.GetStub = func(_ context.Context, key types.NamespacedName, out client.Object, _ ...client.GetOption) error { - if key.Name == stackKey.Name && key.Namespace == stackKey.Namespace { - k.SetClientObject(out, s) - return nil - } - return nil - } - - k.UpdateStub = func(_ context.Context, o client.Object, _ ...client.UpdateOption) error { - stack, ok := o.(*lokiv1.LokiStack) - if !ok { - return apierrors.NewBadRequest("failed conversion to *lokiv1.LokiStack") - } - val, ok := stack.Annotations[storage.AnnotationCredentialsRequestsSecretRef] - if !ok { - return apierrors.NewBadRequest("missing annotation") - } - if val != annotationVal { - return apierrors.NewBadRequest("annotations does not match input") - } - return nil - } - - err := AnnotateForCredentialsRequest(context.Background(), k, stackKey, annotationVal) - require.NoError(t, err) - require.Equal(t, 1, k.UpdateCallCount()) -} diff --git a/operator/controllers/loki/lokistack_controller.go b/operator/controllers/loki/lokistack_controller.go index 40e7691bd1a2..eb30a1a9bf55 100644 --- a/operator/controllers/loki/lokistack_controller.go +++ b/operator/controllers/loki/lokistack_controller.go @@ -3,7 +3,6 @@ package controllers import ( "context" "errors" - "strings" "time" "github.com/go-logr/logr" @@ -16,7 +15,6 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -31,6 +29,7 @@ import ( configv1 "github.com/grafana/loki/operator/apis/config/v1" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" "github.com/grafana/loki/operator/controllers/loki/internal/management/state" + "github.com/grafana/loki/operator/internal/config" "github.com/grafana/loki/operator/internal/external/k8s" "github.com/grafana/loki/operator/internal/handlers" manifestsocp "github.com/grafana/loki/operator/internal/manifests/openshift" @@ -111,6 +110,7 @@ type LokiStackReconciler struct { Log logr.Logger Scheme *runtime.Scheme FeatureGates configv1.FeatureGates + AuthConfig *config.ManagedAuthConfig } // +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks,verbs=get;list;watch;create;update;patch;delete @@ -128,7 +128,7 @@ type LokiStackReconciler struct { // +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update // +kubebuilder:rbac:groups=config.openshift.io,resources=dnses;apiservers;proxies,verbs=get;list;watch // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;delete -// +kubebuilder:rbac:groups=cloudcredential.openshift.io,resources=credentialsrequests,verbs=get;list;watch;create;delete +// +kubebuilder:rbac:groups=cloudcredential.openshift.io,resources=credentialsrequests,verbs=get;list;watch;create;update;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -150,7 +150,7 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } var degraded *status.DegradedError - err = r.updateResources(ctx, req) + credentialMode, err := r.updateResources(ctx, req) switch { case errors.As(err, °raded): // degraded errors are handled by status.Refresh below @@ -158,7 +158,7 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, err } - err = status.Refresh(ctx, r.Client, req, time.Now(), degraded) + err = status.Refresh(ctx, r.Client, req, time.Now(), credentialMode, degraded) if err != nil { return ctrl.Result{}, err } @@ -172,18 +172,25 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } -func (r *LokiStackReconciler) updateResources(ctx context.Context, req ctrl.Request) error { +func (r *LokiStackReconciler) updateResources(ctx context.Context, req ctrl.Request) (lokiv1.CredentialMode, error) { if r.FeatureGates.BuiltInCertManagement.Enabled { if err := handlers.CreateOrRotateCertificates(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates); err != nil { - return err + return "", err } } - if err := handlers.CreateOrUpdateLokiStack(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates); err != nil { - return err + if r.FeatureGates.OpenShift.ManagedAuthEnv { + if err := handlers.CreateCredentialsRequest(ctx, r.Log, r.Scheme, r.AuthConfig, r.Client, req); err != nil { + return "", err + } + } + + credentialMode, err := handlers.CreateOrUpdateLokiStack(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates) + if err != nil { + return "", err } - return nil + return credentialMode, nil } // SetupWithManager sets up the controller with the Manager. @@ -216,7 +223,7 @@ func (r *LokiStackReconciler) buildController(bld k8s.Builder) error { if r.FeatureGates.OpenShift.Enabled { bld = bld. Owns(&routev1.Route{}, updateOrDeleteOnlyPred). - Watches(&cloudcredentialv1.CredentialsRequest{}, r.enqueueForCredentialsRequest(), updateOrDeleteOnlyPred) + Owns(&cloudcredentialv1.CredentialsRequest{}, updateOrDeleteOnlyPred) if r.FeatureGates.OpenShift.ClusterTLSPolicy { bld = bld.Watches(&openshiftconfigv1.APIServer{}, r.enqueueAllLokiStacksHandler(), updateOrDeleteOnlyPred) @@ -358,34 +365,3 @@ func (r *LokiStackReconciler) enqueueForStorageCA() handler.EventHandler { return requests }) } - -func (r *LokiStackReconciler) enqueueForCredentialsRequest() handler.EventHandler { - return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { - a := obj.GetAnnotations() - owner, ok := a[manifestsocp.AnnotationCredentialsRequestOwner] - if !ok { - return nil - } - - var ( - ownerParts = strings.Split(owner, "/") - namespace = ownerParts[0] - name = ownerParts[1] - key = client.ObjectKey{Namespace: namespace, Name: name} - ) - - var stack lokiv1.LokiStack - if err := r.Client.Get(ctx, key, &stack); err != nil { - if !apierrors.IsNotFound(err) { - r.Log.Error(err, "failed retrieving CredentialsRequest owning Lokistack", "key", key) - } - return nil - } - - return []reconcile.Request{ - { - NamespacedName: key, - }, - } - }) -} diff --git a/operator/controllers/loki/lokistack_controller_test.go b/operator/controllers/loki/lokistack_controller_test.go index 515d829766aa..6be22022c19d 100644 --- a/operator/controllers/loki/lokistack_controller_test.go +++ b/operator/controllers/loki/lokistack_controller_test.go @@ -161,7 +161,18 @@ func TestLokiStackController_RegisterOwnedResourcesForUpdateOrDeleteOnly(t *test { obj: &routev1.Route{}, index: 10, - ownCallsCount: 11, + ownCallsCount: 12, + featureGates: configv1.FeatureGates{ + OpenShift: configv1.OpenShiftFeatureGates{ + Enabled: true, + }, + }, + pred: updateOrDeleteOnlyPred, + }, + { + obj: &cloudcredentialv1.CredentialsRequest{}, + index: 11, + ownCallsCount: 12, featureGates: configv1.FeatureGates{ OpenShift: configv1.OpenShiftFeatureGates{ Enabled: true, @@ -203,20 +214,9 @@ func TestLokiStackController_RegisterWatchedResources(t *testing.T) { } table := []test{ { - src: &cloudcredentialv1.CredentialsRequest{}, + src: &openshiftconfigv1.APIServer{}, index: 3, watchesCallsCount: 4, - featureGates: configv1.FeatureGates{ - OpenShift: configv1.OpenShiftFeatureGates{ - Enabled: true, - }, - }, - pred: updateOrDeleteOnlyPred, - }, - { - src: &openshiftconfigv1.APIServer{}, - index: 4, - watchesCallsCount: 5, featureGates: configv1.FeatureGates{ OpenShift: configv1.OpenShiftFeatureGates{ Enabled: true, @@ -227,8 +227,8 @@ func TestLokiStackController_RegisterWatchedResources(t *testing.T) { }, { src: &openshiftconfigv1.Proxy{}, - index: 4, - watchesCallsCount: 5, + index: 3, + watchesCallsCount: 4, featureGates: configv1.FeatureGates{ OpenShift: configv1.OpenShiftFeatureGates{ Enabled: true, diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index 92f93dd97022..48fbe0c8a7e4 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -1100,6 +1100,40 @@ string +## CredentialMode { #loki-grafana-com-v1-CredentialMode } +(string alias) +

+(Appears on:LokiStackStorageStatus) +

+
+

CredentialMode represents the type of authentication used for accessing the object storage.

+
+ + + + + + + + + + + + + + +
ValueDescription

"managed"

CredentialModeManaged represents the usage of short-lived tokens retrieved from a credential source. +This mode is similar to CredentialModeToken,but instead of having a user-configured credential source, +it is configured by the environment, for example the Cloud Credential Operator in OpenShift. +This mode is only supported for certain object storage types in certain runtime environments.

+

"static"

CredentialModeStatic represents the usage of static, long-lived credentials stored in a Secret. +This is the default authentication mode and available for all supported object storage types.

+

"token"

CredentialModeToken represents the usage of short-lived tokens retrieved from a credential source. +In this mode the static configuration does not contain credentials needed for the object storage. +Instead, they are generated during runtime using a service, which allows for shorter-lived credentials and +much more granular control. This authentication mode is not supported for all object storage types.

+
+ ## HashRingSpec { #loki-grafana-com-v1-HashRingSpec }

(Appears on:LokiStackSpec) @@ -2152,6 +2186,20 @@ the Loki storage configuration.

to the LokiStack.

+ + +credentialMode
+ + +CredentialMode + + + + +(Optional) +

CredentialMode contains the authentication mode used for accessing the object storage.

+ + diff --git a/operator/docs/operator/feature-gates.md b/operator/docs/operator/feature-gates.md index 34fbdf4b69a4..189b72e4ddb1 100644 --- a/operator/docs/operator/feature-gates.md +++ b/operator/docs/operator/feature-gates.md @@ -417,7 +417,8 @@ bool -

ManagedAuthEnv enabled when the operator installation is on OpenShift STS clusters.

+

ManagedAuthEnv is true when OpenShift-functions are enabled and the operator has detected +that it is running with some kind of “workload identity” (AWS STS, Azure WIF) enabled.

diff --git a/operator/internal/config/managed_auth.go b/operator/internal/config/managed_auth.go new file mode 100644 index 000000000000..76f9d72f3c26 --- /dev/null +++ b/operator/internal/config/managed_auth.go @@ -0,0 +1,50 @@ +package config + +import "os" + +type AWSEnvironment struct { + RoleARN string +} + +type AzureEnvironment struct { + ClientID string + SubscriptionID string + TenantID string + Region string +} + +type ManagedAuthConfig struct { + AWS *AWSEnvironment + Azure *AzureEnvironment +} + +func discoverManagedAuthConfig() *ManagedAuthConfig { + // AWS + roleARN := os.Getenv("ROLEARN") + + // Azure + clientID := os.Getenv("CLIENTID") + tenantID := os.Getenv("TENANTID") + subscriptionID := os.Getenv("SUBSCRIPTIONID") + region := os.Getenv("REGION") + + switch { + case roleARN != "": + return &ManagedAuthConfig{ + AWS: &AWSEnvironment{ + RoleARN: roleARN, + }, + } + case clientID != "" && tenantID != "" && subscriptionID != "": + return &ManagedAuthConfig{ + Azure: &AzureEnvironment{ + ClientID: clientID, + SubscriptionID: subscriptionID, + TenantID: tenantID, + Region: region, + }, + } + } + + return nil +} diff --git a/operator/internal/config/options.go b/operator/internal/config/options.go index 7ed9abb526a7..dc54404f2245 100644 --- a/operator/internal/config/options.go +++ b/operator/internal/config/options.go @@ -17,19 +17,24 @@ import ( // LoadConfig initializes the controller configuration, optionally overriding the defaults // from a provided configuration file. -func LoadConfig(scheme *runtime.Scheme, configFile string) (*configv1.ProjectConfig, ctrl.Options, error) { +func LoadConfig(scheme *runtime.Scheme, configFile string) (*configv1.ProjectConfig, *ManagedAuthConfig, ctrl.Options, error) { options := ctrl.Options{Scheme: scheme} if configFile == "" { - return &configv1.ProjectConfig{}, options, nil + return &configv1.ProjectConfig{}, nil, options, nil } ctrlCfg, err := loadConfigFile(scheme, configFile) if err != nil { - return nil, options, fmt.Errorf("failed to parse controller manager config file: %w", err) + return nil, nil, options, fmt.Errorf("failed to parse controller manager config file: %w", err) + } + + managedAuth := discoverManagedAuthConfig() + if ctrlCfg.Gates.OpenShift.Enabled && managedAuth != nil { + ctrlCfg.Gates.OpenShift.ManagedAuthEnv = true } options = mergeOptionsFromFile(options, ctrlCfg) - return ctrlCfg, options, nil + return ctrlCfg, managedAuth, options, nil } func mergeOptionsFromFile(o manager.Options, cfg *configv1.ProjectConfig) manager.Options { diff --git a/operator/internal/handlers/credentialsrequest.go b/operator/internal/handlers/credentialsrequest.go new file mode 100644 index 000000000000..0d562332dc9d --- /dev/null +++ b/operator/internal/handlers/credentialsrequest.go @@ -0,0 +1,73 @@ +package handlers + +import ( + "context" + "fmt" + + "github.com/ViaQ/logerr/v2/kverrors" + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/config" + "github.com/grafana/loki/operator/internal/external/k8s" + "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/manifests/openshift" +) + +// CreateCredentialsRequest creates a new CredentialsRequest resource for a Lokistack +// to request a cloud credentials Secret resource from the OpenShift cloud-credentials-operator. +func CreateCredentialsRequest(ctx context.Context, log logr.Logger, scheme *runtime.Scheme, managedAuth *config.ManagedAuthConfig, k k8s.Client, req ctrl.Request) error { + ll := log.WithValues("lokistack", req.NamespacedName, "event", "createCredentialsRequest") + + var stack lokiv1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { + if apierrors.IsNotFound(err) { + // maybe the user deleted it before we could react? Either way this isn't an issue + ll.Error(err, "could not find the requested LokiStack", "name", req.String()) + return nil + } + return kverrors.Wrap(err, "failed to lookup LokiStack", "name", req.String()) + } + + opts := openshift.Options{ + BuildOpts: openshift.BuildOptions{ + LokiStackName: stack.Name, + LokiStackNamespace: stack.Namespace, + RulerName: manifests.RulerName(stack.Name), + }, + ManagedAuth: managedAuth, + } + + credReq, err := openshift.BuildCredentialsRequest(opts) + if err != nil { + return err + } + + err = ctrl.SetControllerReference(&stack, credReq, scheme) + if err != nil { + return kverrors.Wrap(err, "failed to set controller owner reference to resource") + } + + desired := credReq.DeepCopyObject().(client.Object) + mutateFn := manifests.MutateFuncFor(credReq, desired, map[string]string{}) + + op, err := ctrl.CreateOrUpdate(ctx, k, credReq, mutateFn) + if err != nil { + return kverrors.Wrap(err, "failed to configure CredentialRequest") + } + + msg := fmt.Sprintf("Resource has been %s", op) + switch op { + case ctrlutil.OperationResultNone: + ll.V(1).Info(msg) + default: + ll.Info(msg) + } + + return nil +} diff --git a/operator/internal/handlers/credentialsrequest_create.go b/operator/internal/handlers/credentialsrequest_create.go deleted file mode 100644 index 477528326b9a..000000000000 --- a/operator/internal/handlers/credentialsrequest_create.go +++ /dev/null @@ -1,42 +0,0 @@ -package handlers - -import ( - "context" - - "github.com/ViaQ/logerr/v2/kverrors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/grafana/loki/operator/internal/external/k8s" - "github.com/grafana/loki/operator/internal/manifests/openshift" -) - -// CreateCredentialsRequest creates a new CredentialsRequest resource for a Lokistack -// to request a cloud credentials Secret resource from the OpenShift cloud-credentials-operator. -func CreateCredentialsRequest(ctx context.Context, k k8s.Client, stack client.ObjectKey) (string, error) { - managedAuthEnv := openshift.DiscoverManagedAuthEnv() - if managedAuthEnv == nil { - return "", nil - } - - opts := openshift.Options{ - BuildOpts: openshift.BuildOptions{ - LokiStackName: stack.Name, - LokiStackNamespace: stack.Namespace, - }, - ManagedAuthEnv: managedAuthEnv, - } - - credReq, err := openshift.BuildCredentialsRequest(opts) - if err != nil { - return "", err - } - - if err := k.Create(ctx, credReq); err != nil { - if !apierrors.IsAlreadyExists(err) { - return "", kverrors.Wrap(err, "failed to create credentialsrequest", "key", client.ObjectKeyFromObject(credReq)) - } - } - - return credReq.Spec.SecretRef.Name, nil -} diff --git a/operator/internal/handlers/credentialsrequest_create_test.go b/operator/internal/handlers/credentialsrequest_create_test.go deleted file mode 100644 index f6bf9c0f1b52..000000000000 --- a/operator/internal/handlers/credentialsrequest_create_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package handlers - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" -) - -func TestCreateCredentialsRequest_DoNothing_WhenManagedAuthEnvMissing(t *testing.T) { - k := &k8sfakes.FakeClient{} - key := client.ObjectKey{Name: "my-stack", Namespace: "ns"} - - secretRef, err := CreateCredentialsRequest(context.Background(), k, key) - require.NoError(t, err) - require.Empty(t, secretRef) -} - -func TestCreateCredentialsRequest_CreateNewResource(t *testing.T) { - k := &k8sfakes.FakeClient{} - key := client.ObjectKey{Name: "my-stack", Namespace: "ns"} - - t.Setenv("ROLEARN", "a-role-arn") - - secretRef, err := CreateCredentialsRequest(context.Background(), k, key) - require.NoError(t, err) - require.NotEmpty(t, secretRef) - require.Equal(t, 1, k.CreateCallCount()) -} - -func TestCreateCredentialsRequest_DoNothing_WhenCredentialsRequestExist(t *testing.T) { - k := &k8sfakes.FakeClient{} - key := client.ObjectKey{Name: "my-stack", Namespace: "ns"} - - t.Setenv("ROLEARN", "a-role-arn") - - k.CreateStub = func(_ context.Context, _ client.Object, _ ...client.CreateOption) error { - return errors.NewAlreadyExists(schema.GroupResource{}, "credentialsrequest exists") - } - - secretRef, err := CreateCredentialsRequest(context.Background(), k, key) - require.NoError(t, err) - require.NotEmpty(t, secretRef) - require.Equal(t, 1, k.CreateCallCount()) -} diff --git a/operator/internal/handlers/credentialsrequest_delete.go b/operator/internal/handlers/credentialsrequest_delete.go deleted file mode 100644 index edf05fcb205d..000000000000 --- a/operator/internal/handlers/credentialsrequest_delete.go +++ /dev/null @@ -1,43 +0,0 @@ -package handlers - -import ( - "context" - - "github.com/ViaQ/logerr/v2/kverrors" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/grafana/loki/operator/internal/external/k8s" - "github.com/grafana/loki/operator/internal/manifests/openshift" -) - -// DeleteCredentialsRequest deletes a LokiStack's accompanying CredentialsRequest resource -// to trigger the OpenShift cloud-credentials-operator to wipe out any credentials related -// Secret resource on the LokiStack namespace. -func DeleteCredentialsRequest(ctx context.Context, k k8s.Client, stack client.ObjectKey) error { - managedAuthEnv := openshift.DiscoverManagedAuthEnv() - if managedAuthEnv == nil { - return nil - } - - opts := openshift.Options{ - BuildOpts: openshift.BuildOptions{ - LokiStackName: stack.Name, - LokiStackNamespace: stack.Namespace, - }, - ManagedAuthEnv: managedAuthEnv, - } - - credReq, err := openshift.BuildCredentialsRequest(opts) - if err != nil { - return kverrors.Wrap(err, "failed to build credentialsrequest", "key", stack) - } - - if err := k.Delete(ctx, credReq); err != nil { - if !errors.IsNotFound(err) { - return kverrors.Wrap(err, "failed to delete credentialsrequest", "key", client.ObjectKeyFromObject(credReq)) - } - } - - return nil -} diff --git a/operator/internal/handlers/credentialsrequest_delete_test.go b/operator/internal/handlers/credentialsrequest_delete_test.go deleted file mode 100644 index 57f1c005ee70..000000000000 --- a/operator/internal/handlers/credentialsrequest_delete_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package handlers - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" -) - -func TestDeleteCredentialsRequest_DoNothing_WhenManagedAuthEnvMissing(t *testing.T) { - k := &k8sfakes.FakeClient{} - key := client.ObjectKey{Name: "my-stack", Namespace: "ns"} - - err := DeleteCredentialsRequest(context.Background(), k, key) - require.NoError(t, err) -} - -func TestDeleteCredentialsRequest_DeleteExistingResource(t *testing.T) { - k := &k8sfakes.FakeClient{} - key := client.ObjectKey{Name: "my-stack", Namespace: "ns"} - - t.Setenv("ROLEARN", "a-role-arn") - - err := DeleteCredentialsRequest(context.Background(), k, key) - require.NoError(t, err) - require.Equal(t, 1, k.DeleteCallCount()) -} - -func TestDeleteCredentialsRequest_DoNothing_WhenCredentialsRequestNotExists(t *testing.T) { - k := &k8sfakes.FakeClient{} - key := client.ObjectKey{Name: "my-stack", Namespace: "ns"} - - t.Setenv("ROLEARN", "a-role-arn") - - k.DeleteStub = func(_ context.Context, _ client.Object, _ ...client.DeleteOption) error { - return errors.NewNotFound(schema.GroupResource{}, "credentials request not found") - } - - err := DeleteCredentialsRequest(context.Background(), k, key) - require.NoError(t, err) - require.Equal(t, 1, k.DeleteCallCount()) -} diff --git a/operator/internal/handlers/credentialsrequest_test.go b/operator/internal/handlers/credentialsrequest_test.go new file mode 100644 index 000000000000..dd6dfb50d77d --- /dev/null +++ b/operator/internal/handlers/credentialsrequest_test.go @@ -0,0 +1,146 @@ +package handlers + +import ( + "context" + "testing" + + cloudcredentialv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/config" + "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" +) + +func credentialsRequestFakeClient(cr *cloudcredentialv1.CredentialsRequest, lokistack *lokiv1.LokiStack) *k8sfakes.FakeClient { + k := &k8sfakes.FakeClient{} + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + switch object.(type) { + case *cloudcredentialv1.CredentialsRequest: + if cr == nil { + return errors.NewNotFound(schema.GroupResource{}, name.Name) + } + k.SetClientObject(object, cr) + case *lokiv1.LokiStack: + if lokistack == nil { + return errors.NewNotFound(schema.GroupResource{}, name.Name) + } + k.SetClientObject(object, lokistack) + } + return nil + } + + return k +} + +func TestCreateCredentialsRequest_CreateNewResource(t *testing.T) { + wantServiceAccountNames := []string{ + "my-stack", + "my-stack-ruler", + } + + lokistack := &lokiv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "ns", + }, + } + + k := credentialsRequestFakeClient(nil, lokistack) + req := ctrl.Request{ + NamespacedName: client.ObjectKey{Name: "my-stack", Namespace: "ns"}, + } + + managedAuth := &config.ManagedAuthConfig{ + AWS: &config.AWSEnvironment{ + RoleARN: "a-role-arn", + }, + } + + err := CreateCredentialsRequest(context.Background(), logger, scheme, managedAuth, k, req) + require.NoError(t, err) + require.Equal(t, 1, k.CreateCallCount()) + + _, obj, _ := k.CreateArgsForCall(0) + credReq, ok := obj.(*cloudcredentialv1.CredentialsRequest) + require.True(t, ok) + + require.Equal(t, wantServiceAccountNames, credReq.Spec.ServiceAccountNames) +} + +func TestCreateCredentialsRequest_CreateNewResourceAzure(t *testing.T) { + wantRegion := "test-region" + + lokistack := &lokiv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "ns", + }, + } + + k := credentialsRequestFakeClient(nil, lokistack) + req := ctrl.Request{ + NamespacedName: client.ObjectKey{Name: "my-stack", Namespace: "ns"}, + } + + managedAuth := &config.ManagedAuthConfig{ + Azure: &config.AzureEnvironment{ + ClientID: "test-client-id", + SubscriptionID: "test-tenant-id", + TenantID: "test-subscription-id", + Region: "test-region", + }, + } + + err := CreateCredentialsRequest(context.Background(), logger, scheme, managedAuth, k, req) + require.NoError(t, err) + + require.Equal(t, 1, k.CreateCallCount()) + _, obj, _ := k.CreateArgsForCall(0) + credReq, ok := obj.(*cloudcredentialv1.CredentialsRequest) + require.True(t, ok) + + providerSpec := &cloudcredentialv1.AzureProviderSpec{} + require.NoError(t, cloudcredentialv1.Codec.DecodeProviderSpec(credReq.Spec.ProviderSpec, providerSpec)) + + require.Equal(t, wantRegion, providerSpec.AzureRegion) +} + +func TestCreateCredentialsRequest_DoNothing_WhenCredentialsRequestExist(t *testing.T) { + req := ctrl.Request{ + NamespacedName: client.ObjectKey{Name: "my-stack", Namespace: "ns"}, + } + + managedAuth := &config.ManagedAuthConfig{ + AWS: &config.AWSEnvironment{ + RoleARN: "a-role-arn", + }, + } + + cr := &cloudcredentialv1.CredentialsRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "ns", + }, + } + lokistack := &lokiv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "ns", + }, + } + + k := credentialsRequestFakeClient(cr, lokistack) + + err := CreateCredentialsRequest(context.Background(), logger, scheme, managedAuth, k, req) + require.NoError(t, err) + require.Equal(t, 2, k.GetCallCount()) + require.Equal(t, 0, k.CreateCallCount()) + require.Equal(t, 1, k.UpdateCallCount()) +} diff --git a/operator/internal/handlers/internal/storage/secrets.go b/operator/internal/handlers/internal/storage/secrets.go index e41fa9c2c5b0..80dde97b6136 100644 --- a/operator/internal/handlers/internal/storage/secrets.go +++ b/operator/internal/handlers/internal/storage/secrets.go @@ -1,10 +1,14 @@ package storage import ( + "bytes" "context" "crypto/sha1" + "encoding/base64" + "encoding/json" "errors" "fmt" + "io" "sort" corev1 "k8s.io/api/core/v1" @@ -28,8 +32,26 @@ var ( errSecretHashError = errors.New("error calculating hash for secret") errS3NoAuth = errors.New("missing secret fields for static or sts authentication") + + errAzureNoCredentials = errors.New("azure storage secret does contain neither account_key or client_id") + errAzureMixedCredentials = errors.New("azure storage secret can not contain both account_key and client_id") + errAzureManagedIdentityNoOverride = errors.New("when in managed mode, storage secret can not contain credentials") + errAzureInvalidEnvironment = errors.New("azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment)") + errAzureInvalidAccountKey = errors.New("azure account key is not valid base64") + + errGCPParseCredentialsFile = errors.New("gcp storage secret cannot be parsed from JSON content") + errGCPWrongCredentialSourceFile = errors.New("credential source in secret needs to point to token file") + + azureValidEnvironments = map[string]bool{ + "AzureGlobal": true, + "AzureChinaCloud": true, + "AzureGermanCloud": true, + "AzureUSGovernment": true, + } ) +const gcpAccountTypeExternal = "external_account" + func getSecrets(ctx context.Context, k k8s.Client, stack *lokiv1.LokiStack, fg configv1.FeatureGates) (*corev1.Secret, *corev1.Secret, error) { var ( storageSecret corev1.Secret @@ -49,15 +71,7 @@ func getSecrets(ctx context.Context, k k8s.Client, stack *lokiv1.LokiStack, fg c } if fg.OpenShift.ManagedAuthEnv { - secretName, ok := stack.Annotations[storage.AnnotationCredentialsRequestsSecretRef] - if !ok { - return nil, nil, &status.DegradedError{ - Message: "Missing OpenShift cloud credentials request", - Reason: lokiv1.ReasonMissingCredentialsRequest, - Requeue: true, - } - } - + secretName := storage.ManagedCredentialsSecretName(stack.Name) managedAuthCredsKey := client.ObjectKey{Name: secretName, Namespace: stack.Namespace} if err := k.Get(ctx, managedAuthCredsKey, &managedAuthSecret); err != nil { if apierrors.IsNotFound(err) { @@ -90,7 +104,7 @@ func extractSecrets(secretType lokiv1.ObjectStorageSecretType, objStore, managed SharedStore: secretType, } - if fg.OpenShift.ManagedAuthEnabled() { + if fg.OpenShift.ManagedAuthEnv { var managedAuthHash string managedAuthHash, err = hashSecretData(managedAuth) if err != nil { @@ -107,7 +121,7 @@ func extractSecrets(secretType lokiv1.ObjectStorageSecretType, objStore, managed switch secretType { case lokiv1.ObjectStorageSecretAzure: - storageOpts.Azure, err = extractAzureConfigSecret(objStore) + storageOpts.Azure, err = extractAzureConfigSecret(objStore, fg) case lokiv1.ObjectStorageSecretGCS: storageOpts.GCS, err = extractGCSConfigSecret(objStore) case lokiv1.ObjectStorageSecretS3: @@ -155,35 +169,99 @@ func hashSecretData(s *corev1.Secret) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -func extractAzureConfigSecret(s *corev1.Secret) (*storage.AzureStorageConfig, error) { +func extractAzureConfigSecret(s *corev1.Secret, fg configv1.FeatureGates) (*storage.AzureStorageConfig, error) { // Extract and validate mandatory fields - env := s.Data[storage.KeyAzureEnvironmentName] - if len(env) == 0 { + env := string(s.Data[storage.KeyAzureEnvironmentName]) + if env == "" { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureEnvironmentName) } + + if !azureValidEnvironments[env] { + return nil, fmt.Errorf("%w: %s", errAzureInvalidEnvironment, env) + } + + accountName := s.Data[storage.KeyAzureStorageAccountName] + if len(accountName) == 0 { + return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountName) + } + container := s.Data[storage.KeyAzureStorageContainerName] if len(container) == 0 { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageContainerName) } - name := s.Data[storage.KeyAzureStorageAccountName] - if len(name) == 0 { - return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountName) - } - key := s.Data[storage.KeyAzureStorageAccountKey] - if len(key) == 0 { - return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountKey) + + workloadIdentity, err := validateAzureCredentials(s, fg) + if err != nil { + return nil, err } // Extract and validate optional fields endpointSuffix := s.Data[storage.KeyAzureStorageEndpointSuffix] + audience := s.Data[storage.KeyAzureAudience] + + if !workloadIdentity && len(audience) > 0 { + return nil, fmt.Errorf("%w: %s", errSecretFieldNotAllowed, storage.KeyAzureAudience) + } return &storage.AzureStorageConfig{ - Env: string(env), - Container: string(container), - EndpointSuffix: string(endpointSuffix), + Env: env, + Container: string(container), + EndpointSuffix: string(endpointSuffix), + Audience: string(audience), + WorkloadIdentity: workloadIdentity, }, nil } +func validateAzureCredentials(s *corev1.Secret, fg configv1.FeatureGates) (workloadIdentity bool, err error) { + accountKey := s.Data[storage.KeyAzureStorageAccountKey] + clientID := s.Data[storage.KeyAzureStorageClientID] + tenantID := s.Data[storage.KeyAzureStorageTenantID] + subscriptionID := s.Data[storage.KeyAzureStorageSubscriptionID] + + if fg.OpenShift.ManagedAuthEnv { + if len(accountKey) > 0 || len(clientID) > 0 || len(tenantID) > 0 || len(subscriptionID) > 0 { + return false, errAzureManagedIdentityNoOverride + } + + return true, nil + } + + if len(accountKey) == 0 && len(clientID) == 0 { + return false, errAzureNoCredentials + } + + if len(accountKey) > 0 && len(clientID) > 0 { + return false, errAzureMixedCredentials + } + + if len(accountKey) > 0 { + if err := validateBase64(accountKey); err != nil { + return false, errAzureInvalidAccountKey + } + + // have both account_name and account_key -> no workload identity federation + return false, nil + } + + // assume workload-identity from here on + if len(tenantID) == 0 { + return false, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageTenantID) + } + + if len(subscriptionID) == 0 { + return false, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageSubscriptionID) + } + + return true, nil +} + +func validateBase64(data []byte) error { + buf := bytes.NewBuffer(data) + reader := base64.NewDecoder(base64.StdEncoding, buf) + _, err := io.ReadAll(reader) + return err +} + func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error) { // Extract and validate mandatory fields bucket := s.Data[storage.KeyGCPStorageBucketName] @@ -197,8 +275,36 @@ func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error) return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyGCPServiceAccountKeyFilename) } + credentialsFile := struct { + CredentialsType string `json:"type"` + CredentialsSource struct { + File string `json:"file"` + } `json:"credential_source"` + }{} + + err := json.Unmarshal(keyJSON, &credentialsFile) + if err != nil { + return nil, errGCPParseCredentialsFile + } + + var ( + audience = s.Data[storage.KeyGCPWorkloadIdentityProviderAudience] + isWorkloadIdentity = credentialsFile.CredentialsType == gcpAccountTypeExternal + ) + if isWorkloadIdentity { + if len(audience) == 0 { + return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyGCPWorkloadIdentityProviderAudience) + } + + if credentialsFile.CredentialsSource.File != storage.ServiceAccountTokenFilePath { + return nil, fmt.Errorf("%w: %s", errGCPWrongCredentialSourceFile, storage.ServiceAccountTokenFilePath) + } + } + return &storage.GCSStorageConfig{ - Bucket: string(bucket), + Bucket: string(bucket), + WorkloadIdentity: isWorkloadIdentity, + Audience: string(audience), }, nil } @@ -238,16 +344,13 @@ func extractS3ConfigSecret(s *corev1.Secret, fg configv1.FeatureGates) (*storage ) switch { - case fg.OpenShift.ManagedAuthEnabled(): + case fg.OpenShift.ManagedAuthEnv: cfg.STS = true - cfg.Audience = storage.AWSOpenShiftAudience + cfg.Audience = string(audience) // Do not allow users overriding the role arn provided on Loki Operator installation if len(roleArn) != 0 { return nil, fmt.Errorf("%w: %s", errSecretFieldNotAllowed, storage.KeyAWSRoleArn) } - if len(audience) != 0 { - return nil, fmt.Errorf("%w: %s", errSecretFieldNotAllowed, storage.KeyAWSAudience) - } // In the STS case region is not an optional field if len(region) == 0 { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAWSRegion) diff --git a/operator/internal/handlers/internal/storage/secrets_test.go b/operator/internal/handlers/internal/storage/secrets_test.go index 70aebd18afc5..647de5632b4b 100644 --- a/operator/internal/handlers/internal/storage/secrets_test.go +++ b/operator/internal/handlers/internal/storage/secrets_test.go @@ -71,9 +71,12 @@ func TestUnknownType(t *testing.T) { func TestAzureExtract(t *testing.T) { type test struct { - name string - secret *corev1.Secret - wantError string + name string + secret *corev1.Secret + managedSecret *corev1.Secret + featureGates configv1.FeatureGates + wantError string + wantCredentialMode lokiv1.CredentialMode } table := []test{ { @@ -82,60 +85,194 @@ func TestAzureExtract(t *testing.T) { wantError: "missing secret field: environment", }, { - name: "missing container", + name: "invalid environment", secret: &corev1.Secret{ Data: map[string][]byte{ - "environment": []byte("here"), + "environment": []byte("invalid-environment"), }, }, - wantError: "missing secret field: container", + wantError: "azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment): invalid-environment", }, { name: "missing account_name", secret: &corev1.Secret{ Data: map[string][]byte{ - "environment": []byte("here"), - "container": []byte("this,that"), + "environment": []byte("AzureGlobal"), }, }, wantError: "missing secret field: account_name", }, { - name: "missing account_key", + name: "missing container", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "account_name": []byte("id"), + }, + }, + wantError: "missing secret field: container", + }, + { + name: "no account_key or client_id", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "container": []byte("this,that"), + "account_name": []byte("id"), + }, + }, + wantError: errAzureNoCredentials.Error(), + }, + { + name: "both account_key and client_id set", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "account_key": []byte("test-account-key"), + "client_id": []byte("test-client-id"), + }, + }, + wantError: errAzureMixedCredentials.Error(), + }, + { + name: "missing tenant_id", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "client_id": []byte("test-client-id"), + }, + }, + wantError: "missing secret field: tenant_id", + }, + { + name: "missing subscription_id", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "client_id": []byte("test-client-id"), + "tenant_id": []byte("test-tenant-id"), + }, + }, + wantError: "missing secret field: subscription_id", + }, + { + name: "managed auth - no auth override", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Data: map[string][]byte{ - "environment": []byte("here"), + "environment": []byte("AzureGlobal"), + "account_name": []byte("test-account-name"), + "container": []byte("this,that"), + "region": []byte("test-region"), + "account_key": []byte("test-account-key"), + }, + }, + managedSecret: &corev1.Secret{ + Data: map[string][]byte{}, + }, + featureGates: configv1.FeatureGates{ + OpenShift: configv1.OpenShiftFeatureGates{ + Enabled: true, + ManagedAuthEnv: true, + }, + }, + wantError: errAzureManagedIdentityNoOverride.Error(), + }, + { + name: "audience used with static authentication", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), "container": []byte("this,that"), "account_name": []byte("id"), + "account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="), // test-account-key + "audience": []byte("test-audience"), }, }, - wantError: "missing secret field: account_key", + wantError: "secret field not allowed: audience", }, { - name: "all mandatory set", + name: "mandatory for normal authentication set", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Data: map[string][]byte{ - "environment": []byte("here"), + "environment": []byte("AzureGlobal"), "container": []byte("this,that"), "account_name": []byte("id"), - "account_key": []byte("secret"), + "account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="), // test-account-key }, }, + wantCredentialMode: lokiv1.CredentialModeStatic, + }, + { + name: "mandatory for workload-identity set", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "client_id": []byte("test-client-id"), + "tenant_id": []byte("test-tenant-id"), + "subscription_id": []byte("test-subscription-id"), + "region": []byte("test-region"), + }, + }, + wantCredentialMode: lokiv1.CredentialModeToken, + }, + { + name: "mandatory for managed workload-identity set", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "account_name": []byte("test-account-name"), + "container": []byte("this,that"), + "region": []byte("test-region"), + }, + }, + managedSecret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "managed-secret", + }, + Data: map[string][]byte{ + "azure_client_id": []byte("test-client-id"), + "azure_tenant_id": []byte("test-tenant-id"), + "azure_subscription_id": []byte("test-subscription-id"), + }, + }, + featureGates: configv1.FeatureGates{ + OpenShift: configv1.OpenShiftFeatureGates{ + Enabled: true, + ManagedAuthEnv: true, + }, + }, + wantCredentialMode: lokiv1.CredentialModeManaged, }, { name: "all set including optional", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Data: map[string][]byte{ - "environment": []byte("here"), + "environment": []byte("AzureGlobal"), "container": []byte("this,that"), "account_name": []byte("id"), - "account_key": []byte("secret"), + "account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="), // test-account-key "endpoint_suffix": []byte("suffix"), }, }, + wantCredentialMode: lokiv1.CredentialModeStatic, }, } for _, tst := range table { @@ -143,12 +280,13 @@ func TestAzureExtract(t *testing.T) { t.Run(tst.name, func(t *testing.T) { t.Parallel() - opts, err := extractSecrets(lokiv1.ObjectStorageSecretAzure, tst.secret, nil, configv1.FeatureGates{}) + opts, err := extractSecrets(lokiv1.ObjectStorageSecretAzure, tst.secret, tst.managedSecret, tst.featureGates) if tst.wantError == "" { require.NoError(t, err) require.NotEmpty(t, opts.SecretName) require.NotEmpty(t, opts.SecretSHA1) - require.Equal(t, opts.SharedStore, lokiv1.ObjectStorageSecretAzure) + require.Equal(t, lokiv1.ObjectStorageSecretAzure, opts.SharedStore) + require.Equal(t, tst.wantCredentialMode, opts.CredentialMode()) } else { require.EqualError(t, err, tst.wantError) } @@ -158,9 +296,10 @@ func TestAzureExtract(t *testing.T) { func TestGCSExtract(t *testing.T) { type test struct { - name string - secret *corev1.Secret - wantError string + name string + secret *corev1.Secret + wantError string + wantCredentialMode lokiv1.CredentialMode } table := []test{ { @@ -177,15 +316,51 @@ func TestGCSExtract(t *testing.T) { }, wantError: "missing secret field: key.json", }, + { + name: "missing audience", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "bucketname": []byte("here"), + "key.json": []byte("{\"type\": \"external_account\"}"), + }, + }, + wantError: "missing secret field: audience", + }, + { + name: "credential_source file no override", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "bucketname": []byte("here"), + "audience": []byte("test"), + "key.json": []byte("{\"type\": \"external_account\", \"credential_source\": {\"file\": \"/custom/path/to/secret/storage/serviceaccount/token\"}}"), + }, + }, + wantError: "credential source in secret needs to point to token file: /var/run/secrets/storage/serviceaccount/token", + }, { name: "all set", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Data: map[string][]byte{ "bucketname": []byte("here"), - "key.json": []byte("{\"type\": \"SA\"}"), + "key.json": []byte("{\"type\": \"service_account\"}"), }, }, + wantCredentialMode: lokiv1.CredentialModeStatic, + }, + { + name: "mandatory for workload-identity set", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "bucketname": []byte("here"), + "audience": []byte("test"), + "key.json": []byte("{\"type\": \"external_account\", \"credential_source\": {\"file\": \"/var/run/secrets/storage/serviceaccount/token\"}}"), + }, + }, + wantCredentialMode: lokiv1.CredentialModeToken, }, } for _, tst := range table { @@ -193,9 +368,10 @@ func TestGCSExtract(t *testing.T) { t.Run(tst.name, func(t *testing.T) { t.Parallel() - _, err := extractSecrets(lokiv1.ObjectStorageSecretGCS, tst.secret, nil, configv1.FeatureGates{}) + opts, err := extractSecrets(lokiv1.ObjectStorageSecretGCS, tst.secret, nil, configv1.FeatureGates{}) if tst.wantError == "" { require.NoError(t, err) + require.Equal(t, tst.wantCredentialMode, opts.CredentialMode()) } else { require.EqualError(t, err, tst.wantError) } @@ -205,9 +381,10 @@ func TestGCSExtract(t *testing.T) { func TestS3Extract(t *testing.T) { type test struct { - name string - secret *corev1.Secret - wantError string + name string + secret *corev1.Secret + wantError string + wantCredentialMode lokiv1.CredentialMode } table := []test{ { @@ -285,6 +462,7 @@ func TestS3Extract(t *testing.T) { "sse_kms_key_id": []byte("kms-key-id"), }, }, + wantCredentialMode: lokiv1.CredentialModeStatic, }, { name: "all set with SSE-KMS with encryption context", @@ -300,6 +478,7 @@ func TestS3Extract(t *testing.T) { "sse_kms_encryption_context": []byte("kms-encryption-ctx"), }, }, + wantCredentialMode: lokiv1.CredentialModeStatic, }, { name: "all set with SSE-S3", @@ -313,6 +492,7 @@ func TestS3Extract(t *testing.T) { "sse_type": []byte("SSE-S3"), }, }, + wantCredentialMode: lokiv1.CredentialModeStatic, }, { name: "all set without SSE", @@ -325,6 +505,7 @@ func TestS3Extract(t *testing.T) { "access_key_secret": []byte("secret"), }, }, + wantCredentialMode: lokiv1.CredentialModeStatic, }, { name: "STS missing region", @@ -347,6 +528,7 @@ func TestS3Extract(t *testing.T) { "region": []byte("here"), }, }, + wantCredentialMode: lokiv1.CredentialModeToken, }, { name: "STS all set", @@ -359,6 +541,7 @@ func TestS3Extract(t *testing.T) { "audience": []byte("audience"), }, }, + wantCredentialMode: lokiv1.CredentialModeToken, }, } for _, tst := range table { @@ -371,7 +554,8 @@ func TestS3Extract(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, opts.SecretName) require.NotEmpty(t, opts.SecretSHA1) - require.Equal(t, opts.SharedStore, lokiv1.ObjectStorageSecretS3) + require.Equal(t, lokiv1.ObjectStorageSecretS3, opts.SharedStore) + require.Equal(t, tst.wantCredentialMode, opts.CredentialMode()) } else { require.EqualError(t, err, tst.wantError) } @@ -421,18 +605,6 @@ func TestS3Extract_WithOpenShiftManagedAuth(t *testing.T) { managedAuthSecret: &corev1.Secret{}, wantError: "secret field not allowed: role_arn", }, - { - name: "override audience not allowed", - secret: &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "test"}, - Data: map[string][]byte{ - "bucketnames": []byte("this,that"), - "audience": []byte("test-audience"), - }, - }, - managedAuthSecret: &corev1.Secret{}, - wantError: "secret field not allowed: audience", - }, { name: "STS all set", secret: &corev1.Secret{ @@ -457,11 +629,11 @@ func TestS3Extract_WithOpenShiftManagedAuth(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, opts.SecretName) require.NotEmpty(t, opts.SecretSHA1) - require.Equal(t, opts.SharedStore, lokiv1.ObjectStorageSecretS3) + require.Equal(t, lokiv1.ObjectStorageSecretS3, opts.SharedStore) require.True(t, opts.S3.STS) - require.Equal(t, opts.S3.Audience, "openshift") - require.Equal(t, opts.OpenShift.CloudCredentials.SecretName, tst.managedAuthSecret.Name) + require.Equal(t, tst.managedAuthSecret.Name, opts.OpenShift.CloudCredentials.SecretName) require.NotEmpty(t, opts.OpenShift.CloudCredentials.SHA1) + require.Equal(t, lokiv1.CredentialModeManaged, opts.CredentialMode()) } else { require.EqualError(t, err, tst.wantError) } @@ -609,7 +781,8 @@ func TestSwiftExtract(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, opts.SecretName) require.NotEmpty(t, opts.SecretSHA1) - require.Equal(t, opts.SharedStore, lokiv1.ObjectStorageSecretSwift) + require.Equal(t, lokiv1.ObjectStorageSecretSwift, opts.SharedStore) + require.Equal(t, lokiv1.CredentialModeStatic, opts.CredentialMode()) } else { require.EqualError(t, err, tst.wantError) } @@ -682,7 +855,8 @@ func TestAlibabaCloudExtract(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, opts.SecretName) require.NotEmpty(t, opts.SecretSHA1) - require.Equal(t, opts.SharedStore, lokiv1.ObjectStorageSecretAlibabaCloud) + require.Equal(t, lokiv1.ObjectStorageSecretAlibabaCloud, opts.SharedStore) + require.Equal(t, lokiv1.CredentialModeStatic, opts.CredentialMode()) } else { require.EqualError(t, err, tst.wantError) } diff --git a/operator/internal/handlers/internal/storage/storage_test.go b/operator/internal/handlers/internal/storage/storage_test.go index 9e041bf99a23..45f5b0f2865b 100644 --- a/operator/internal/handlers/internal/storage/storage_test.go +++ b/operator/internal/handlers/internal/storage/storage_test.go @@ -17,7 +17,6 @@ import ( configv1 "github.com/grafana/loki/operator/apis/config/v1" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" - "github.com/grafana/loki/operator/internal/manifests/storage" "github.com/grafana/loki/operator/internal/status" ) @@ -135,77 +134,6 @@ func TestBuildOptions_WhenMissingSecret_SetDegraded(t *testing.T) { require.Equal(t, degradedErr, err) } -func TestBuildOptions_WhenMissingCloudCredentialsRequest_SetDegraded(t *testing.T) { - sw := &k8sfakes.FakeStatusWriter{} - k := &k8sfakes.FakeClient{} - r := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Name: "my-stack", - Namespace: "some-ns", - }, - } - - fg := configv1.FeatureGates{ - OpenShift: configv1.OpenShiftFeatureGates{ - ManagedAuthEnv: true, - }, - } - - degradedErr := &status.DegradedError{ - Message: "Missing OpenShift cloud credentials request", - Reason: lokiv1.ReasonMissingCredentialsRequest, - Requeue: true, - } - - stack := &lokiv1.LokiStack{ - TypeMeta: metav1.TypeMeta{ - Kind: "LokiStack", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "my-stack", - Namespace: "some-ns", - UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", - Annotations: map[string]string{}, - }, - Spec: lokiv1.LokiStackSpec{ - Size: lokiv1.SizeOneXExtraSmall, - Storage: lokiv1.ObjectStorageSpec{ - Schemas: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV11, - EffectiveDate: "2020-10-11", - }, - }, - Secret: lokiv1.ObjectStorageSecretSpec{ - Name: defaultManagedAuthSecret.Name, - Type: lokiv1.ObjectStorageSecretS3, - }, - }, - }, - } - - k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { - _, isLokiStack := object.(*lokiv1.LokiStack) - if r.Name == name.Name && r.Namespace == name.Namespace && isLokiStack { - k.SetClientObject(object, stack) - return nil - } - if name.Name == defaultManagedAuthSecret.Name { - k.SetClientObject(object, &defaultManagedAuthSecret) - return nil - } - return apierrors.NewNotFound(schema.GroupResource{}, "something is not found") - } - - k.StatusStub = func() client.StatusWriter { return sw } - - _, err := BuildOptions(context.TODO(), k, stack, fg) - - // make sure error is returned - require.Error(t, err) - require.Equal(t, degradedErr, err) -} - func TestBuildOptions_WhenMissingCloudCredentialsSecret_SetDegraded(t *testing.T) { sw := &k8sfakes.FakeStatusWriter{} k := &k8sfakes.FakeClient{} @@ -236,9 +164,6 @@ func TestBuildOptions_WhenMissingCloudCredentialsSecret_SetDegraded(t *testing.T Name: "my-stack", Namespace: "some-ns", UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", - Annotations: map[string]string{ - storage.AnnotationCredentialsRequestsSecretRef: "my-stack-aws-creds", - }, }, Spec: lokiv1.LokiStackSpec{ Size: lokiv1.SizeOneXExtraSmall, diff --git a/operator/internal/handlers/lokistack_create_or_update.go b/operator/internal/handlers/lokistack_create_or_update.go index 2f78f75d02c5..47e7a309bf8b 100644 --- a/operator/internal/handlers/lokistack_create_or_update.go +++ b/operator/internal/handlers/lokistack_create_or_update.go @@ -36,7 +36,7 @@ func CreateOrUpdateLokiStack( k k8s.Client, s *runtime.Scheme, fg configv1.FeatureGates, -) error { +) (lokiv1.CredentialMode, error) { ll := log.WithValues("lokistack", req.NamespacedName, "event", "createOrUpdate") var stack lokiv1.LokiStack @@ -44,9 +44,9 @@ func CreateOrUpdateLokiStack( if apierrors.IsNotFound(err) { // maybe the user deleted it before we could react? Either way this isn't an issue ll.Error(err, "could not find the requested loki stack", "name", req.NamespacedName) - return nil + return "", nil } - return kverrors.Wrap(err, "failed to lookup lokistack", "name", req.NamespacedName) + return "", kverrors.Wrap(err, "failed to lookup lokistack", "name", req.NamespacedName) } img := os.Getenv(manifests.EnvRelatedImageLoki) @@ -61,21 +61,21 @@ func CreateOrUpdateLokiStack( objStore, err := storage.BuildOptions(ctx, k, &stack, fg) if err != nil { - return err + return "", err } baseDomain, tenants, err := gateway.BuildOptions(ctx, ll, k, &stack, fg) if err != nil { - return err + return "", err } if err = rules.Cleanup(ctx, ll, k, &stack); err != nil { - return err + return "", err } alertingRules, recordingRules, ruler, ocpOptions, err := rules.BuildOptions(ctx, ll, k, &stack) if err != nil { - return err + return "", err } certRotationRequiredAt := "" @@ -86,7 +86,7 @@ func CreateOrUpdateLokiStack( timeoutConfig, err := manifests.NewTimeoutConfig(stack.Spec.Limits) if err != nil { ll.Error(err, "failed to parse query timeout") - return &status.DegradedError{ + return "", &status.DegradedError{ Message: fmt.Sprintf("Error parsing query timeout: %s", err), Reason: lokiv1.ReasonQueryTimeoutInvalid, Requeue: false, @@ -116,13 +116,13 @@ func CreateOrUpdateLokiStack( if optErr := manifests.ApplyDefaultSettings(&opts); optErr != nil { ll.Error(optErr, "failed to conform options to build settings") - return optErr + return "", optErr } if fg.LokiStackGateway { if optErr := manifests.ApplyGatewayDefaultOptions(&opts); optErr != nil { ll.Error(optErr, "failed to apply defaults options to gateway settings") - return optErr + return "", optErr } } @@ -140,13 +140,13 @@ func CreateOrUpdateLokiStack( if optErr := manifests.ApplyTLSSettings(&opts, tlsProfile); optErr != nil { ll.Error(optErr, "failed to conform options to tls profile settings") - return optErr + return "", optErr } objects, err := manifests.BuildAll(opts) if err != nil { ll.Error(err, "failed to build manifests") - return err + return "", err } ll.Info("manifests built", "count", len(objects)) @@ -158,7 +158,7 @@ func CreateOrUpdateLokiStack( // a user possibly being unable to read logs. if err := status.SetStorageSchemaStatus(ctx, k, req, objStore.Schemas); err != nil { ll.Error(err, "failed to set storage schema status") - return err + return "", err } var errCount int32 @@ -182,7 +182,7 @@ func CreateOrUpdateLokiStack( depAnnotations, err := dependentAnnotations(ctx, k, obj) if err != nil { l.Error(err, "failed to set dependent annotations") - return err + return "", err } desired := obj.DeepCopyObject().(client.Object) @@ -205,7 +205,7 @@ func CreateOrUpdateLokiStack( } if errCount > 0 { - return kverrors.New("failed to configure lokistack resources", "name", req.NamespacedName) + return "", kverrors.New("failed to configure lokistack resources", "name", req.NamespacedName) } // 1x.demo is used only for development, so the metrics will not @@ -214,7 +214,7 @@ func CreateOrUpdateLokiStack( metrics.Collect(&opts.Stack, opts.Name) } - return nil + return objStore.CredentialMode(), nil } func dependentAnnotations(ctx context.Context, k k8s.Client, obj client.Object) (map[string]string, error) { diff --git a/operator/internal/handlers/lokistack_create_or_update_test.go b/operator/internal/handlers/lokistack_create_or_update_test.go index 4ba9a9affc36..bef5ffc9efb7 100644 --- a/operator/internal/handlers/lokistack_create_or_update_test.go +++ b/operator/internal/handlers/lokistack_create_or_update_test.go @@ -108,7 +108,7 @@ func TestCreateOrUpdateLokiStack_WhenGetReturnsNotFound_DoesNotError(t *testing. k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) require.NoError(t, err) // make sure create was NOT called because the Get failed @@ -132,7 +132,7 @@ func TestCreateOrUpdateLokiStack_WhenGetReturnsAnErrorOtherThanNotFound_ReturnsT k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) require.Equal(t, badRequestErr, errors.Unwrap(err)) @@ -219,7 +219,7 @@ func TestCreateOrUpdateLokiStack_SetsNamespaceOnAllObjects(t *testing.T) { k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) require.NoError(t, err) // make sure create was called @@ -327,7 +327,7 @@ func TestCreateOrUpdateLokiStack_SetsOwnerRefOnAllObjects(t *testing.T) { k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) require.NoError(t, err) // make sure create was called @@ -387,7 +387,7 @@ func TestCreateOrUpdateLokiStack_WhenSetControllerRefInvalid_ContinueWithOtherOb k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) // make sure error is returned to re-trigger reconciliation require.Error(t, err) @@ -490,7 +490,7 @@ func TestCreateOrUpdateLokiStack_WhenGetReturnsNoError_UpdateObjects(t *testing. k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) require.NoError(t, err) // make sure create not called @@ -556,7 +556,7 @@ func TestCreateOrUpdateLokiStack_WhenCreateReturnsError_ContinueWithOtherObjects k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) // make sure error is returned to re-trigger reconciliation require.Error(t, err) @@ -663,7 +663,7 @@ func TestCreateOrUpdateLokiStack_WhenUpdateReturnsError_ContinueWithOtherObjects k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) // make sure error is returned to re-trigger reconciliation require.Error(t, err) @@ -734,7 +734,7 @@ func TestCreateOrUpdateLokiStack_WhenInvalidQueryTimeout_SetDegraded(t *testing. k.StatusStub = func() client.StatusWriter { return sw } - err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) + _, err := CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates) // make sure error is returned require.Error(t, err) diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 187dc6514202..602672c813bf 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -5436,7 +5436,8 @@ func TestBuild_ConfigAndRuntimeConfig_STS(t *testing.T) { } expStorageConfig := ` s3: - s3: s3://my-region/my-bucket + bucketnames: my-bucket + region: my-region s3forcepathstyle: false` expCfg := ` diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index 61c0de401dc1..5d729eaffaf9 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -13,7 +13,11 @@ common: environment: {{ .Env }} container_name: {{ .Container }} account_name: ${AZURE_STORAGE_ACCOUNT_NAME} + {{- if .WorkloadIdentity }} + use_federated_token: true + {{- else }} account_key: ${AZURE_STORAGE_ACCOUNT_KEY} + {{- end }} {{- with .EndpointSuffix }} endpoint_suffix: {{ . }} {{- end }} @@ -25,7 +29,8 @@ common: {{- with .ObjectStorage.S3 }} s3: {{- if .STS }} - s3: "s3://{{.Region}}/{{.Buckets}}" + bucketnames: {{.Buckets}} + region: {{.Region}} s3forcepathstyle: false {{- else }} s3: {{ .Endpoint }} diff --git a/operator/internal/manifests/mutate.go b/operator/internal/manifests/mutate.go index 27421750bf2c..63308bb9ceb6 100644 --- a/operator/internal/manifests/mutate.go +++ b/operator/internal/manifests/mutate.go @@ -6,6 +6,7 @@ import ( "github.com/ViaQ/logerr/v2/kverrors" "github.com/imdario/mergo" routev1 "github.com/openshift/api/route/v1" + cloudcredentialv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -123,6 +124,11 @@ func MutateFuncFor(existing, desired client.Object, depAnnotations map[string]st wantRt := desired.(*routev1.Route) mutateRoute(rt, wantRt) + case *cloudcredentialv1.CredentialsRequest: + cr := existing.(*cloudcredentialv1.CredentialsRequest) + wantCr := desired.(*cloudcredentialv1.CredentialsRequest) + mutateCredentialRequest(cr, wantCr) + case *monitoringv1.PrometheusRule: pr := existing.(*monitoringv1.PrometheusRule) wantPr := desired.(*monitoringv1.PrometheusRule) @@ -213,6 +219,10 @@ func mutateRoute(existing, desired *routev1.Route) { existing.Spec = desired.Spec } +func mutateCredentialRequest(existing, desired *cloudcredentialv1.CredentialsRequest) { + existing.Spec = desired.Spec +} + func mutatePrometheusRule(existing, desired *monitoringv1.PrometheusRule) { existing.Annotations = desired.Annotations existing.Labels = desired.Labels diff --git a/operator/internal/manifests/openshift/credentialsrequest.go b/operator/internal/manifests/openshift/credentialsrequest.go index 8fc2c5d3f512..0c0a19adc98d 100644 --- a/operator/internal/manifests/openshift/credentialsrequest.go +++ b/operator/internal/manifests/openshift/credentialsrequest.go @@ -1,10 +1,6 @@ package openshift import ( - "fmt" - "os" - "path" - "github.com/ViaQ/logerr/v2/kverrors" cloudcredentialv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" corev1 "k8s.io/api/core/v1" @@ -12,48 +8,42 @@ import ( "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/grafana/loki/operator/internal/config" "github.com/grafana/loki/operator/internal/manifests/storage" ) -const ( - ccoNamespace = "openshift-cloud-credential-operator" -) +const azureFallbackRegion = "centralus" func BuildCredentialsRequest(opts Options) (*cloudcredentialv1.CredentialsRequest, error) { stack := client.ObjectKey{Name: opts.BuildOpts.LokiStackName, Namespace: opts.BuildOpts.LokiStackNamespace} - providerSpec, secretName, err := encodeProviderSpec(opts.BuildOpts.LokiStackName, opts.ManagedAuthEnv) + providerSpec, err := encodeProviderSpec(opts.ManagedAuth) if err != nil { return nil, kverrors.Wrap(err, "failed encoding credentialsrequest provider spec") } return &cloudcredentialv1.CredentialsRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", stack.Namespace, secretName), - Namespace: ccoNamespace, - Annotations: map[string]string{ - AnnotationCredentialsRequestOwner: stack.String(), - }, + Name: stack.Name, + Namespace: stack.Namespace, }, Spec: cloudcredentialv1.CredentialsRequestSpec{ SecretRef: corev1.ObjectReference{ - Name: secretName, + Name: storage.ManagedCredentialsSecretName(stack.Name), Namespace: stack.Namespace, }, ProviderSpec: providerSpec, ServiceAccountNames: []string{ stack.Name, + rulerServiceAccountName(opts), }, - CloudTokenPath: path.Join(storage.SATokenVolumeOcpDirectory, "token"), + CloudTokenPath: storage.ServiceAccountTokenFilePath, }, }, nil } -func encodeProviderSpec(stackName string, env *ManagedAuthEnv) (*runtime.RawExtension, string, error) { - var ( - spec runtime.Object - secretName string - ) +func encodeProviderSpec(env *config.ManagedAuthConfig) (*runtime.RawExtension, error) { + var spec runtime.Object switch { case env.AWS != nil: @@ -72,25 +62,44 @@ func encodeProviderSpec(stackName string, env *ManagedAuthEnv) (*runtime.RawExte }, STSIAMRoleARN: env.AWS.RoleARN, } - secretName = fmt.Sprintf("%s-aws-creds", stackName) - } - - encodedSpec, err := cloudcredentialv1.Codec.EncodeProviderSpec(spec.DeepCopyObject()) - return encodedSpec, secretName, err -} - -func DiscoverManagedAuthEnv() *ManagedAuthEnv { - // AWS - roleARN := os.Getenv("ROLEARN") + case env.Azure != nil: + azure := env.Azure + if azure.Region == "" { + // The OpenShift Console currently does not provide a UI to configure the Azure Region + // for an operator using managed credentials. Because the CredentialsRequest is currently + // not used to create a Managed Identity, the region is actually never used. + // We default to the US region if nothing is set, so that the CredentialsRequest can be + // created. This should have no effect on the generated credential secret. + // The region can be configured by setting an environment variable on the operator Subscription. + azure.Region = azureFallbackRegion + } - switch { - case roleARN != "": - return &ManagedAuthEnv{ - AWS: &AWSSTSEnv{ - RoleARN: roleARN, + spec = &cloudcredentialv1.AzureProviderSpec{ + Permissions: []string{ + "Microsoft.Storage/storageAccounts/blobServices/read", + "Microsoft.Storage/storageAccounts/blobServices/containers/read", + "Microsoft.Storage/storageAccounts/blobServices/containers/write", + "Microsoft.Storage/storageAccounts/blobServices/generateUserDelegationKey/action", + "Microsoft.Storage/storageAccounts/read", + "Microsoft.Storage/storageAccounts/write", + "Microsoft.Storage/storageAccounts/delete", + "Microsoft.Storage/storageAccounts/listKeys/action", + "Microsoft.Resources/tags/write", + }, + DataPermissions: []string{ + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/delete", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/write", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/add/action", + "Microsoft.Storage/storageAccounts/blobServices/containers/blobs/move/action", }, + AzureClientID: azure.ClientID, + AzureRegion: azure.Region, + AzureSubscriptionID: azure.SubscriptionID, + AzureTenantID: azure.TenantID, } } - return nil + encodedSpec, err := cloudcredentialv1.Codec.EncodeProviderSpec(spec.DeepCopyObject()) + return encodedSpec, err } diff --git a/operator/internal/manifests/openshift/credentialsrequest_test.go b/operator/internal/manifests/openshift/credentialsrequest_test.go index 0672cadfc210..36c6e2331f7e 100644 --- a/operator/internal/manifests/openshift/credentialsrequest_test.go +++ b/operator/internal/manifests/openshift/credentialsrequest_test.go @@ -1,40 +1,22 @@ package openshift import ( - "strings" "testing" "github.com/stretchr/testify/require" + "github.com/grafana/loki/operator/internal/config" "github.com/grafana/loki/operator/internal/manifests/storage" ) -func TestBuildCredentialsRequest_HasOwnerAnnotation(t *testing.T) { - opts := Options{ - BuildOpts: BuildOptions{ - LokiStackName: "a-stack", - LokiStackNamespace: "ns", - }, - ManagedAuthEnv: &ManagedAuthEnv{ - AWS: &AWSSTSEnv{ - RoleARN: "role-arn", - }, - }, - } - - credReq, err := BuildCredentialsRequest(opts) - require.NoError(t, err) - require.Contains(t, credReq.Annotations, AnnotationCredentialsRequestOwner) -} - func TestBuildCredentialsRequest_HasSecretRef_MatchingLokiStackNamespace(t *testing.T) { opts := Options{ BuildOpts: BuildOptions{ LokiStackName: "a-stack", LokiStackNamespace: "ns", }, - ManagedAuthEnv: &ManagedAuthEnv{ - AWS: &AWSSTSEnv{ + ManagedAuth: &config.ManagedAuthConfig{ + AWS: &config.AWSEnvironment{ RoleARN: "role-arn", }, }, @@ -45,14 +27,14 @@ func TestBuildCredentialsRequest_HasSecretRef_MatchingLokiStackNamespace(t *test require.Equal(t, opts.BuildOpts.LokiStackNamespace, credReq.Spec.SecretRef.Namespace) } -func TestBuildCredentialsRequest_HasServiceAccountNames_ContainsLokiStackName(t *testing.T) { +func TestBuildCredentialsRequest_HasServiceAccountNames_ContainsAllLokiStackServiceAccounts(t *testing.T) { opts := Options{ BuildOpts: BuildOptions{ LokiStackName: "a-stack", LokiStackNamespace: "ns", }, - ManagedAuthEnv: &ManagedAuthEnv{ - AWS: &AWSSTSEnv{ + ManagedAuth: &config.ManagedAuthConfig{ + AWS: &config.AWSEnvironment{ RoleARN: "role-arn", }, }, @@ -61,6 +43,7 @@ func TestBuildCredentialsRequest_HasServiceAccountNames_ContainsLokiStackName(t credReq, err := BuildCredentialsRequest(opts) require.NoError(t, err) require.Contains(t, credReq.Spec.ServiceAccountNames, opts.BuildOpts.LokiStackName) + require.Contains(t, credReq.Spec.ServiceAccountNames, rulerServiceAccountName(opts)) } func TestBuildCredentialsRequest_CloudTokenPath_MatchinOpenShiftSADirectory(t *testing.T) { @@ -69,8 +52,8 @@ func TestBuildCredentialsRequest_CloudTokenPath_MatchinOpenShiftSADirectory(t *t LokiStackName: "a-stack", LokiStackNamespace: "ns", }, - ManagedAuthEnv: &ManagedAuthEnv{ - AWS: &AWSSTSEnv{ + ManagedAuth: &config.ManagedAuthConfig{ + AWS: &config.AWSEnvironment{ RoleARN: "role-arn", }, }, @@ -78,7 +61,7 @@ func TestBuildCredentialsRequest_CloudTokenPath_MatchinOpenShiftSADirectory(t *t credReq, err := BuildCredentialsRequest(opts) require.NoError(t, err) - require.True(t, strings.HasPrefix(credReq.Spec.CloudTokenPath, storage.SATokenVolumeOcpDirectory)) + require.Equal(t, storage.ServiceAccountTokenFilePath, credReq.Spec.CloudTokenPath) } func TestBuildCredentialsRequest_FollowsNamingConventions(t *testing.T) { @@ -95,14 +78,14 @@ func TestBuildCredentialsRequest_FollowsNamingConventions(t *testing.T) { LokiStackName: "a-stack", LokiStackNamespace: "ns", }, - ManagedAuthEnv: &ManagedAuthEnv{ - AWS: &AWSSTSEnv{ + ManagedAuth: &config.ManagedAuthConfig{ + AWS: &config.AWSEnvironment{ RoleARN: "role-arn", }, }, }, - wantName: "ns-a-stack-aws-creds", - wantSecretName: "a-stack-aws-creds", + wantName: "a-stack", + wantSecretName: "a-stack-managed-credentials", }, } for _, test := range tests { diff --git a/operator/internal/manifests/openshift/options.go b/operator/internal/manifests/openshift/options.go index e5d33a335526..572db7fe6445 100644 --- a/operator/internal/manifests/openshift/options.go +++ b/operator/internal/manifests/openshift/options.go @@ -6,6 +6,7 @@ import ( "time" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/config" ) // Options is the set of internal template options for rendering @@ -14,7 +15,7 @@ type Options struct { BuildOpts BuildOptions Authentication []AuthenticationSpec Authorization AuthorizationSpec - ManagedAuthEnv *ManagedAuthEnv + ManagedAuth *config.ManagedAuthConfig } // AuthenticationSpec describes the authentication specification @@ -55,14 +56,6 @@ type TenantData struct { CookieSecret string } -type AWSSTSEnv struct { - RoleARN string -} - -type ManagedAuthEnv struct { - AWS *AWSSTSEnv -} - // NewOptions returns an openshift options struct. func NewOptions( stackName, stackNamespace string, diff --git a/operator/internal/manifests/openshift/var.go b/operator/internal/manifests/openshift/var.go index 84928c48d7e2..5e3ac6300e3e 100644 --- a/operator/internal/manifests/openshift/var.go +++ b/operator/internal/manifests/openshift/var.go @@ -48,8 +48,6 @@ var ( MonitoringSVCUserWorkload = "alertmanager-user-workload" MonitoringUserWorkloadNS = "openshift-user-workload-monitoring" - - AnnotationCredentialsRequestOwner = "loki.grafana.com/credentialsrequest-owner" ) func authorizerRbacName(componentName string) string { diff --git a/operator/internal/manifests/storage/configure.go b/operator/internal/manifests/storage/configure.go index 6f7b22c4bd8c..ede098425323 100644 --- a/operator/internal/manifests/storage/configure.go +++ b/operator/internal/manifests/storage/configure.go @@ -13,6 +13,18 @@ import ( lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" ) +var ( + managedAuthConfigVolumeMount = corev1.VolumeMount{ + Name: managedAuthConfigVolumeName, + MountPath: managedAuthConfigDirectory, + } + + saTokenVolumeMount = corev1.VolumeMount{ + Name: saTokenVolumeName, + MountPath: saTokenVolumeMountPath, + } +) + // ConfigureDeployment appends additional pod volumes and container env vars, args, volume mounts // based on the object storage type. Currently supported amendments: // - All: Ensure object storage secret mounted and auth projected as env vars. @@ -56,7 +68,6 @@ func ConfigureStatefulSet(d *appsv1.StatefulSet, opts Options) error { // With this, the deployment will expose credentials specific environment variables. func configureDeployment(d *appsv1.Deployment, opts Options) error { p := ensureObjectStoreCredentials(&d.Spec.Template.Spec, opts) - if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithOverride); err != nil { return kverrors.Wrap(err, "failed to merge gcs object storage spec ") } @@ -83,7 +94,6 @@ func configureDeploymentCA(d *appsv1.Deployment, tls *TLSConfig) error { // With this, the statefulset will expose credentials specific environment variable. func configureStatefulSet(s *appsv1.StatefulSet, opts Options) error { p := ensureObjectStoreCredentials(&s.Spec.Template.Spec, opts) - if err := mergo.Merge(&s.Spec.Template.Spec, p, mergo.WithOverride); err != nil { return kverrors.Wrap(err, "failed to merge gcs object storage spec ") } @@ -127,14 +137,13 @@ func ensureObjectStoreCredentials(p *corev1.PodSpec, opts Options) corev1.PodSpe }) if managedAuthEnabled(opts) { - setSATokenPath(&opts) container.Env = append(container.Env, managedAuthCredentials(opts)...) volumes = append(volumes, saTokenVolume(opts)) - container.VolumeMounts = append(container.VolumeMounts, saTokenVolumeMount(opts)) + container.VolumeMounts = append(container.VolumeMounts, saTokenVolumeMount) - if opts.OpenShift.ManagedAuthEnabled() { - volumes = append(volumes, managedAuthVolume(opts)) - container.VolumeMounts = append(container.VolumeMounts, managedAuthVolumeMount(opts)) + if opts.OpenShift.ManagedAuthEnabled() && opts.S3 != nil && opts.S3.STS { + volumes = append(volumes, managedAuthConfigVolume(opts)) + container.VolumeMounts = append(container.VolumeMounts, managedAuthConfigVolumeMount) } } else { container.Env = append(container.Env, staticAuthCredentials(opts)...) @@ -186,15 +195,37 @@ func managedAuthCredentials(opts Options) []corev1.EnvVar { case lokiv1.ObjectStorageSecretS3: if opts.OpenShift.ManagedAuthEnabled() { return []corev1.EnvVar{ - envVarFromValue(EnvAWSCredentialsFile, path.Join(managedAuthSecretDirectory, KeyAWSCredentialsFilename)), + envVarFromValue(EnvAWSCredentialsFile, path.Join(managedAuthConfigDirectory, KeyAWSCredentialsFilename)), envVarFromValue(EnvAWSSdkLoadConfig, "true"), } } else { return []corev1.EnvVar{ envVarFromSecret(EnvAWSRoleArn, opts.SecretName, KeyAWSRoleArn), - envVarFromValue(EnvAWSWebIdentityTokenFile, path.Join(opts.S3.WebIdentityTokenFile, "token")), + envVarFromValue(EnvAWSWebIdentityTokenFile, ServiceAccountTokenFilePath), } } + case lokiv1.ObjectStorageSecretAzure: + if opts.OpenShift.ManagedAuthEnabled() { + return []corev1.EnvVar{ + envVarFromSecret(EnvAzureStorageAccountName, opts.SecretName, KeyAzureStorageAccountName), + envVarFromSecret(EnvAzureClientID, opts.OpenShift.CloudCredentials.SecretName, azureManagedCredentialKeyClientID), + envVarFromSecret(EnvAzureTenantID, opts.OpenShift.CloudCredentials.SecretName, azureManagedCredentialKeyTenantID), + envVarFromSecret(EnvAzureSubscriptionID, opts.OpenShift.CloudCredentials.SecretName, azureManagedCredentialKeySubscriptionID), + envVarFromValue(EnvAzureFederatedTokenFile, ServiceAccountTokenFilePath), + } + } + + return []corev1.EnvVar{ + envVarFromSecret(EnvAzureStorageAccountName, opts.SecretName, KeyAzureStorageAccountName), + envVarFromSecret(EnvAzureClientID, opts.SecretName, KeyAzureStorageClientID), + envVarFromSecret(EnvAzureTenantID, opts.SecretName, KeyAzureStorageTenantID), + envVarFromSecret(EnvAzureSubscriptionID, opts.SecretName, KeyAzureStorageSubscriptionID), + envVarFromValue(EnvAzureFederatedTokenFile, ServiceAccountTokenFilePath), + } + case lokiv1.ObjectStorageSecretGCS: + return []corev1.EnvVar{ + envVarFromValue(EnvGoogleApplicationCredentials, path.Join(secretDirectory, KeyGCPServiceAccountKeyFilename)), + } default: return []corev1.EnvVar{} } @@ -273,33 +304,15 @@ func managedAuthEnabled(opts Options) bool { switch opts.SharedStore { case lokiv1.ObjectStorageSecretS3: return opts.S3 != nil && opts.S3.STS + case lokiv1.ObjectStorageSecretAzure: + return opts.Azure != nil && opts.Azure.WorkloadIdentity + case lokiv1.ObjectStorageSecretGCS: + return opts.GCS != nil && opts.GCS.WorkloadIdentity default: return false } } -func setSATokenPath(opts *Options) { - switch opts.SharedStore { - case lokiv1.ObjectStorageSecretS3: - opts.S3.WebIdentityTokenFile = saTokenVolumeK8sDirectory - if opts.OpenShift.Enabled { - opts.S3.WebIdentityTokenFile = SATokenVolumeOcpDirectory - } - } -} - -func saTokenVolumeMount(opts Options) corev1.VolumeMount { - var tokenPath string - switch opts.SharedStore { - case lokiv1.ObjectStorageSecretS3: - tokenPath = opts.S3.WebIdentityTokenFile - } - return corev1.VolumeMount{ - Name: saTokenVolumeName, - MountPath: tokenPath, - } -} - func saTokenVolume(opts Options) corev1.Volume { var audience string storeType := opts.SharedStore @@ -309,9 +322,13 @@ func saTokenVolume(opts Options) corev1.Volume { if opts.S3.Audience != "" { audience = opts.S3.Audience } - if opts.OpenShift.Enabled { - audience = AWSOpenShiftAudience + case lokiv1.ObjectStorageSecretAzure: + audience = azureDefaultAudience + if opts.Azure.Audience != "" { + audience = opts.Azure.Audience } + case lokiv1.ObjectStorageSecretGCS: + audience = opts.GCS.Audience } return corev1.Volume{ Name: saTokenVolumeName, @@ -331,16 +348,9 @@ func saTokenVolume(opts Options) corev1.Volume { } } -func managedAuthVolumeMount(opts Options) corev1.VolumeMount { - return corev1.VolumeMount{ - Name: opts.OpenShift.CloudCredentials.SecretName, - MountPath: managedAuthSecretDirectory, - } -} - -func managedAuthVolume(opts Options) corev1.Volume { +func managedAuthConfigVolume(opts Options) corev1.Volume { return corev1.Volume{ - Name: opts.OpenShift.CloudCredentials.SecretName, + Name: managedAuthConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: opts.OpenShift.CloudCredentials.SecretName, diff --git a/operator/internal/manifests/storage/configure_test.go b/operator/internal/manifests/storage/configure_test.go index 3b3029733554..2cd7b079a4b4 100644 --- a/operator/internal/manifests/storage/configure_test.go +++ b/operator/internal/manifests/storage/configure_test.go @@ -169,10 +169,13 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, { - desc: "object storage GCS", + desc: "object storage Azure with WIF", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretGCS, + SharedStore: lokiv1.ObjectStorageSecretAzure, + Azure: &AzureStorageConfig{ + WorkloadIdentity: true, + }, }, dpl: &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ @@ -200,11 +203,60 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { ReadOnly: false, MountPath: "/etc/storage/secrets", }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, }, Env: []corev1.EnvVar{ { - Name: EnvGoogleApplicationCredentials, - Value: "/etc/storage/secrets/key.json", + Name: EnvAzureStorageAccountName, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageAccountName, + }, + }, + }, + { + Name: EnvAzureClientID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageClientID, + }, + }, + }, + { + Name: EnvAzureTenantID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageTenantID, + }, + }, + }, + { + Name: EnvAzureSubscriptionID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageSubscriptionID, + }, + }, + }, + { + Name: EnvAzureFederatedTokenFile, + Value: "/var/run/secrets/storage/serviceaccount/token", }, }, }, @@ -218,6 +270,22 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: azureDefaultAudience, + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, }, }, }, @@ -225,10 +293,14 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, { - desc: "object storage S3", + desc: "object storage Azure with WIF and custom audience", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretS3, + SharedStore: lokiv1.ObjectStorageSecretAzure, + Azure: &AzureStorageConfig{ + WorkloadIdentity: true, + Audience: "custom-audience", + }, }, dpl: &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ @@ -256,30 +328,61 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { ReadOnly: false, MountPath: "/etc/storage/secrets", }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, }, Env: []corev1.EnvVar{ { - Name: EnvAWSAccessKeyID, + Name: EnvAzureStorageAccountName, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeyAWSAccessKeyID, + Key: KeyAzureStorageAccountName, }, }, }, { - Name: EnvAWSAccessKeySecret, + Name: EnvAzureClientID, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeyAWSAccessKeySecret, + Key: KeyAzureStorageClientID, + }, + }, + }, + { + Name: EnvAzureTenantID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageTenantID, + }, + }, + }, + { + Name: EnvAzureSubscriptionID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageSubscriptionID, }, }, }, + { + Name: EnvAzureFederatedTokenFile, + Value: "/var/run/secrets/storage/serviceaccount/token", + }, }, }, }, @@ -292,6 +395,22 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "custom-audience", + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, }, }, }, @@ -299,13 +418,19 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, { - desc: "object storage S3 in STS Mode", + desc: "object storage Azure with WIF and OpenShift Managed Credentials", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretS3, - S3: &S3StorageConfig{ - STS: true, - Audience: "test", + SharedStore: lokiv1.ObjectStorageSecretAzure, + Azure: &AzureStorageConfig{ + WorkloadIdentity: true, + }, + OpenShift: OpenShiftOptions{ + Enabled: true, + CloudCredentials: CloudCredentials{ + SecretName: "cloud-credentials", + SHA1: "deadbeef", + }, }, }, dpl: &appsv1.Deployment{ @@ -337,24 +462,57 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { { Name: saTokenVolumeName, ReadOnly: false, - MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + MountPath: saTokenVolumeMountPath, }, }, Env: []corev1.EnvVar{ { - Name: EnvAWSRoleArn, + Name: EnvAzureStorageAccountName, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeyAWSRoleArn, + Key: KeyAzureStorageAccountName, }, }, }, { - Name: "AWS_WEB_IDENTITY_TOKEN_FILE", - Value: "/var/run/secrets/kubernetes.io/serviceaccount/token", + Name: EnvAzureClientID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cloud-credentials", + }, + Key: azureManagedCredentialKeyClientID, + }, + }, + }, + { + Name: EnvAzureTenantID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cloud-credentials", + }, + Key: azureManagedCredentialKeyTenantID, + }, + }, + }, + { + Name: EnvAzureSubscriptionID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cloud-credentials", + }, + Key: azureManagedCredentialKeySubscriptionID, + }, + }, + }, + { + Name: EnvAzureFederatedTokenFile, + Value: "/var/run/secrets/storage/serviceaccount/token", }, }, }, @@ -375,7 +533,7 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { Sources: []corev1.VolumeProjection{ { ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Audience: "test", + Audience: azureDefaultAudience, ExpirationSeconds: ptr.To[int64](3600), Path: corev1.ServiceAccountTokenKey, }, @@ -391,22 +549,71 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, { - desc: "object storage S3 in STS Mode in OpenShift", + desc: "object storage GCS", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretS3, - S3: &S3StorageConfig{ - STS: true, - Audience: "test", + SharedStore: lokiv1.ObjectStorageSecretGCS, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, }, - OpenShift: OpenShiftOptions{ - Enabled: true, - CloudCredentials: CloudCredentials{ - SecretName: "cloud-credentials", - SHA1: "deadbeef", + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvGoogleApplicationCredentials, + Value: "/etc/storage/secrets/key.json", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + }, + }, }, }, }, + }, + { + desc: "object storage GCS with Workload Identity", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretGCS, + GCS: &GCSStorageConfig{ + Audience: "test", + WorkloadIdentity: true, + }, + }, dpl: &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -436,22 +643,13 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { { Name: saTokenVolumeName, ReadOnly: false, - MountPath: "/var/run/secrets/openshift/serviceaccount", - }, - { - Name: "cloud-credentials", - ReadOnly: false, - MountPath: "/etc/storage/managed-auth", + MountPath: saTokenVolumeMountPath, }, }, Env: []corev1.EnvVar{ { - Name: "AWS_SHARED_CREDENTIALS_FILE", - Value: "/etc/storage/managed-auth/credentials", - }, - { - Name: "AWS_SDK_LOAD_CONFIG", - Value: "true", + Name: EnvGoogleApplicationCredentials, + Value: "/etc/storage/secrets/key.json", }, }, }, @@ -472,7 +670,7 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { Sources: []corev1.VolumeProjection{ { ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Audience: "openshift", + Audience: "test", ExpirationSeconds: ptr.To[int64](3600), Path: corev1.ServiceAccountTokenKey, }, @@ -481,11 +679,604 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + }, + }, + }, + }, + }, + }, + { + desc: "object storage S3", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretS3, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvAWSAccessKeyID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAWSAccessKeyID, + }, + }, + }, + { + Name: EnvAWSAccessKeySecret, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAWSAccessKeySecret, + }, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "object storage S3 in STS Mode", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretS3, + S3: &S3StorageConfig{ + STS: true, + Audience: "test", + }, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvAWSRoleArn, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAWSRoleArn, + }, + }, + }, + { + Name: "AWS_WEB_IDENTITY_TOKEN_FILE", + Value: "/var/run/secrets/storage/serviceaccount/token", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "test", + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "object storage S3 in STS Mode in OpenShift", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretS3, + S3: &S3StorageConfig{ + STS: true, + }, + OpenShift: OpenShiftOptions{ + Enabled: true, + CloudCredentials: CloudCredentials{ + SecretName: "cloud-credentials", + SHA1: "deadbeef", + }, + }, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, + managedAuthConfigVolumeMount, + }, + Env: []corev1.EnvVar{ + { + Name: "AWS_SHARED_CREDENTIALS_FILE", + Value: "/etc/storage/managed-auth/credentials", + }, + { + Name: "AWS_SDK_LOAD_CONFIG", + Value: "true", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: awsDefaultAudience, + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, + { + Name: managedAuthConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "cloud-credentials", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "object storage S3 with SSE KMS encryption context", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretS3, + S3: &S3StorageConfig{ + SSE: S3SSEConfig{ + Type: SSEKMSType, + KMSEncryptionContext: "test", + }, + }, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvAWSAccessKeyID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAWSAccessKeyID, + }, + }, + }, + { + Name: EnvAWSAccessKeySecret, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAWSAccessKeySecret, + }, + }, + }, + { + Name: EnvAWSSseKmsEncryptionContext, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAWSSseKmsEncryptionContext, + }, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "object storage Swift", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretSwift, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvSwiftUsername, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeySwiftUsername, + }, + }, + }, + { + Name: EnvSwiftPassword, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeySwiftPassword, + }, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range tc { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + err := ConfigureDeployment(tc.dpl, tc.opts) + require.NoError(t, err) + require.Equal(t, tc.want, tc.dpl) + }) + } +} + +func TestConfigureStatefulSetForStorageType(t *testing.T) { + type tt struct { + desc string + opts Options + sts *appsv1.StatefulSet + want *appsv1.StatefulSet + } + + tc := []tt{ + { + desc: "object storage AlibabaCloud", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretAlibabaCloud, + }, + sts: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvAlibabaCloudAccessKeyID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAlibabaCloudAccessKeyID, + }, + }, + }, + { + Name: EnvAlibabaCloudAccessKeySecret, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAlibabaCloudSecretAccessKey, + }, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "object storage Azure", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretAzure, + }, + sts: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvAzureStorageAccountName, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageAccountName, + }, + }, + }, + { + Name: EnvAzureStorageAccountKey, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageAccountKey, + }, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ { - Name: "cloud-credentials", + Name: "test", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "cloud-credentials", + SecretName: "test", }, }, }, @@ -496,19 +1287,16 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, { - desc: "object storage S3 with SSE KMS encryption context", + desc: "object storage Azure with WIF", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretS3, - S3: &S3StorageConfig{ - SSE: S3SSEConfig{ - Type: SSEKMSType, - KMSEncryptionContext: "test", - }, + SharedStore: lokiv1.ObjectStorageSecretAzure, + Azure: &AzureStorageConfig{ + WorkloadIdentity: true, }, }, - dpl: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ + sts: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -520,8 +1308,8 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, - want: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ + want: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -533,41 +1321,61 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { ReadOnly: false, MountPath: "/etc/storage/secrets", }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, }, Env: []corev1.EnvVar{ { - Name: EnvAWSAccessKeyID, + Name: EnvAzureStorageAccountName, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeyAWSAccessKeyID, + Key: KeyAzureStorageAccountName, }, }, }, { - Name: EnvAWSAccessKeySecret, + Name: EnvAzureClientID, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeyAWSAccessKeySecret, + Key: KeyAzureStorageClientID, }, }, }, { - Name: EnvAWSSseKmsEncryptionContext, + Name: EnvAzureTenantID, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeyAWSSseKmsEncryptionContext, + Key: KeyAzureStorageTenantID, + }, + }, + }, + { + Name: EnvAzureSubscriptionID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageSubscriptionID, }, }, }, + { + Name: EnvAzureFederatedTokenFile, + Value: "/var/run/secrets/storage/serviceaccount/token", + }, }, }, }, @@ -580,6 +1388,22 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: azureDefaultAudience, + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, }, }, }, @@ -587,13 +1411,17 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, { - desc: "object storage Swift", + desc: "object storage Azure with WIF and custom audience", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretSwift, + SharedStore: lokiv1.ObjectStorageSecretAzure, + Azure: &AzureStorageConfig{ + WorkloadIdentity: true, + Audience: "custom-audience", + }, }, - dpl: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ + sts: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -605,8 +1433,8 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, - want: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ + want: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -618,30 +1446,61 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { ReadOnly: false, MountPath: "/etc/storage/secrets", }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, }, Env: []corev1.EnvVar{ { - Name: EnvSwiftUsername, + Name: EnvAzureStorageAccountName, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeySwiftUsername, + Key: KeyAzureStorageAccountName, }, }, }, { - Name: EnvSwiftPassword, + Name: EnvAzureClientID, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeySwiftPassword, + Key: KeyAzureStorageClientID, + }, + }, + }, + { + Name: EnvAzureTenantID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageTenantID, + }, + }, + }, + { + Name: EnvAzureSubscriptionID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageSubscriptionID, }, }, }, + { + Name: EnvAzureFederatedTokenFile, + Value: "/var/run/secrets/storage/serviceaccount/token", + }, }, }, }, @@ -654,39 +1513,43 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "custom-audience", + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, }, }, }, }, }, }, - } - - for _, tc := range tc { - tc := tc - t.Run(tc.desc, func(t *testing.T) { - t.Parallel() - err := ConfigureDeployment(tc.dpl, tc.opts) - require.NoError(t, err) - require.Equal(t, tc.want, tc.dpl) - }) - } -} - -func TestConfigureStatefulSetForStorageType(t *testing.T) { - type tt struct { - desc string - opts Options - sts *appsv1.StatefulSet - want *appsv1.StatefulSet - } - - tc := []tt{ { - desc: "object storage AlibabaCloud", + desc: "object storage Azure with WIF and OpenShift Managed Credentials", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretAlibabaCloud, + SharedStore: lokiv1.ObjectStorageSecretAzure, + Azure: &AzureStorageConfig{ + WorkloadIdentity: true, + }, + OpenShift: OpenShiftOptions{ + Enabled: true, + CloudCredentials: CloudCredentials{ + SecretName: "cloud-credentials", + SHA1: "deadbeef", + }, + }, }, sts: &appsv1.StatefulSet{ Spec: appsv1.StatefulSetSpec{ @@ -714,30 +1577,61 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { ReadOnly: false, MountPath: "/etc/storage/secrets", }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, }, Env: []corev1.EnvVar{ { - Name: EnvAlibabaCloudAccessKeyID, + Name: EnvAzureStorageAccountName, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "test", }, - Key: KeyAlibabaCloudAccessKeyID, + Key: KeyAzureStorageAccountName, }, }, }, { - Name: EnvAlibabaCloudAccessKeySecret, + Name: EnvAzureClientID, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "test", + Name: "cloud-credentials", }, - Key: KeyAlibabaCloudSecretAccessKey, + Key: azureManagedCredentialKeyClientID, + }, + }, + }, + { + Name: EnvAzureTenantID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cloud-credentials", + }, + Key: azureManagedCredentialKeyTenantID, + }, + }, + }, + { + Name: EnvAzureSubscriptionID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cloud-credentials", + }, + Key: azureManagedCredentialKeySubscriptionID, }, }, }, + { + Name: EnvAzureFederatedTokenFile, + Value: "/var/run/secrets/storage/serviceaccount/token", + }, }, }, }, @@ -750,6 +1644,22 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { }, }, }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: azureDefaultAudience, + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, }, }, }, @@ -757,10 +1667,10 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { }, }, { - desc: "object storage Azure", + desc: "object storage GCS", opts: Options{ SecretName: "test", - SharedStore: lokiv1.ObjectStorageSecretAzure, + SharedStore: lokiv1.ObjectStorageSecretGCS, }, sts: &appsv1.StatefulSet{ Spec: appsv1.StatefulSetSpec{ @@ -791,26 +1701,8 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { }, Env: []corev1.EnvVar{ { - Name: EnvAzureStorageAccountName, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "test", - }, - Key: KeyAzureStorageAccountName, - }, - }, - }, - { - Name: EnvAzureStorageAccountKey, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "test", - }, - Key: KeyAzureStorageAccountKey, - }, - }, + Name: EnvGoogleApplicationCredentials, + Value: "/etc/storage/secrets/key.json", }, }, }, @@ -831,10 +1723,14 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { }, }, { - desc: "object storage GCS", + desc: "object storage GCS with Workload Identity", opts: Options{ SecretName: "test", SharedStore: lokiv1.ObjectStorageSecretGCS, + GCS: &GCSStorageConfig{ + Audience: "test", + WorkloadIdentity: true, + }, }, sts: &appsv1.StatefulSet{ Spec: appsv1.StatefulSetSpec{ @@ -862,6 +1758,11 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { ReadOnly: false, MountPath: "/etc/storage/secrets", }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, }, Env: []corev1.EnvVar{ { @@ -880,6 +1781,22 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { }, }, }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "test", + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, }, }, }, @@ -966,8 +1883,7 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { SecretName: "test", SharedStore: lokiv1.ObjectStorageSecretS3, S3: &S3StorageConfig{ - STS: true, - Audience: "test", + STS: true, }, OpenShift: OpenShiftOptions{ Enabled: true, @@ -1006,13 +1922,9 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { { Name: saTokenVolumeName, ReadOnly: false, - MountPath: "/var/run/secrets/openshift/serviceaccount", - }, - { - Name: "cloud-credentials", - ReadOnly: false, - MountPath: "/etc/storage/managed-auth", + MountPath: saTokenVolumeMountPath, }, + managedAuthConfigVolumeMount, }, Env: []corev1.EnvVar{ { @@ -1042,7 +1954,7 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { Sources: []corev1.VolumeProjection{ { ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Audience: "openshift", + Audience: awsDefaultAudience, ExpirationSeconds: ptr.To[int64](3600), Path: corev1.ServiceAccountTokenKey, }, @@ -1052,7 +1964,7 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) { }, }, { - Name: "cloud-credentials", + Name: managedAuthConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "cloud-credentials", diff --git a/operator/internal/manifests/storage/options.go b/operator/internal/manifests/storage/options.go index 80efb24f62c8..56e2b8e870df 100644 --- a/operator/internal/manifests/storage/options.go +++ b/operator/internal/manifests/storage/options.go @@ -23,27 +23,64 @@ type Options struct { OpenShift OpenShiftOptions } +// CredentialMode returns which mode is used by the current storage configuration. +// This defaults to CredentialModeStatic, but can be CredentialModeToken +// or CredentialModeManaged depending on the object storage provide, the provided +// secret and whether the operator is running in a managed-auth cluster. +func (o Options) CredentialMode() lokiv1.CredentialMode { + if o.Azure != nil { + if o.OpenShift.ManagedAuthEnabled() { + return lokiv1.CredentialModeManaged + } + + if o.Azure.WorkloadIdentity { + return lokiv1.CredentialModeToken + } + } + + if o.GCS != nil { + if o.GCS.WorkloadIdentity { + return lokiv1.CredentialModeToken + } + } + + if o.S3 != nil { + if o.OpenShift.ManagedAuthEnabled() { + return lokiv1.CredentialModeManaged + } + + if o.S3.STS { + return lokiv1.CredentialModeToken + } + } + + return lokiv1.CredentialModeStatic +} + // AzureStorageConfig for Azure storage config type AzureStorageConfig struct { - Env string - Container string - EndpointSuffix string + Env string + Container string + EndpointSuffix string + Audience string + WorkloadIdentity bool } // GCSStorageConfig for GCS storage config type GCSStorageConfig struct { - Bucket string + Bucket string + Audience string + WorkloadIdentity bool } // S3StorageConfig for S3 storage config type S3StorageConfig struct { - Endpoint string - Region string - Buckets string - WebIdentityTokenFile string - Audience string - STS bool - SSE S3SSEConfig + Endpoint string + Region string + Buckets string + Audience string + STS bool + SSE S3SSEConfig } type S3SSEType string diff --git a/operator/internal/manifests/storage/var.go b/operator/internal/manifests/storage/var.go index d77de3262d31..1f236406bdd0 100644 --- a/operator/internal/manifests/storage/var.go +++ b/operator/internal/manifests/storage/var.go @@ -1,5 +1,7 @@ package storage +import "fmt" + const ( // EnvAlibabaCloudAccessKeyID is the environment variable to specify the AlibabaCloud client id to access S3. EnvAlibabaCloudAccessKeyID = "ALIBABA_CLOUD_ACCESS_KEY_ID" @@ -13,7 +15,7 @@ const ( EnvAWSSseKmsEncryptionContext = "AWS_SSE_KMS_ENCRYPTION_CONTEXT" // EnvAWSRoleArn is the environment variable to specify the AWS role ARN secret for the federated identity workflow. EnvAWSRoleArn = "AWS_ROLE_ARN" - // EnvAWSWebIdentityToken is the environment variable to specify the path to the web identity token file used in the federated identity workflow. + // EnvAWSWebIdentityTokenFile is the environment variable to specify the path to the web identity token file used in the federated identity workflow. EnvAWSWebIdentityTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE" // EnvAWSCredentialsFile is the environment variable to specify the path to the shared credentials file EnvAWSCredentialsFile = "AWS_SHARED_CREDENTIALS_FILE" @@ -23,6 +25,14 @@ const ( EnvAzureStorageAccountName = "AZURE_STORAGE_ACCOUNT_NAME" // EnvAzureStorageAccountKey is the environment variable to specify the Azure storage account key to access the container. EnvAzureStorageAccountKey = "AZURE_STORAGE_ACCOUNT_KEY" + // EnvAzureClientID is the environment variable used to pass the Managed Identity client-ID to the container. + EnvAzureClientID = "AZURE_CLIENT_ID" + // EnvAzureTenantID is the environment variable used to pass the Managed Identity tenant-ID to the container. + EnvAzureTenantID = "AZURE_TENANT_ID" + // EnvAzureSubscriptionID is the environment variable used to pass the Managed Identity subscription-ID to the container. + EnvAzureSubscriptionID = "AZURE_SUBSCRIPTION_ID" + // EnvAzureFederatedTokenFile is the environment variable used to store the path to the Managed Identity token. + EnvAzureFederatedTokenFile = "AZURE_FEDERATED_TOKEN_FILE" // EnvGoogleApplicationCredentials is the environment variable to specify path to key.json EnvGoogleApplicationCredentials = "GOOGLE_APPLICATION_CREDENTIALS" // EnvSwiftPassword is the environment variable to specify the OpenStack Swift password. @@ -66,13 +76,23 @@ const ( KeyAzureStorageAccountKey = "account_key" // KeyAzureStorageAccountName is the secret data key for the Azure storage account name. KeyAzureStorageAccountName = "account_name" + // KeyAzureStorageClientID contains the UUID of the Managed Identity accessing the storage. + KeyAzureStorageClientID = "client_id" + // KeyAzureStorageTenantID contains the UUID of the Tenant hosting the Managed Identity. + KeyAzureStorageTenantID = "tenant_id" + // KeyAzureStorageSubscriptionID contains the UUID of the subscription hosting the Managed Identity. + KeyAzureStorageSubscriptionID = "subscription_id" // KeyAzureStorageContainerName is the secret data key for the Azure storage container name. KeyAzureStorageContainerName = "container" // KeyAzureStorageEndpointSuffix is the secret data key for the Azure storage endpoint URL suffix. KeyAzureStorageEndpointSuffix = "endpoint_suffix" // KeyAzureEnvironmentName is the secret data key for the Azure cloud environment name. KeyAzureEnvironmentName = "environment" + // KeyAzureAudience is the secret data key for customizing the audience used for the ServiceAccount token. + KeyAzureAudience = "audience" + // KeyGCPWorkloadIdentityProviderAudience is the secret data key for the GCP Workload Identity Provider audience. + KeyGCPWorkloadIdentityProviderAudience = "audience" // KeyGCPStorageBucketName is the secret data key for the GCS bucket name. KeyGCPStorageBucketName = "bucketname" // KeyGCPServiceAccountKeyFilename is the service account key filename containing the Google authentication credentials. @@ -100,25 +120,36 @@ const ( KeySwiftRegion = "region" // KeySwiftUserDomainID is the secret data key for the OpenStack Swift user domain id. KeySwiftUserDomainID = "user_domain_id" - // KeySwiftUserDomainID is the secret data key for the OpenStack Swift user domain name. + // KeySwiftUserDomainName is the secret data key for the OpenStack Swift user domain name. KeySwiftUserDomainName = "user_domain_name" // KeySwiftUserID is the secret data key for the OpenStack Swift user id. KeySwiftUserID = "user_id" - // KeySwiftPassword is the secret data key for the OpenStack Swift password. + // KeySwiftUsername is the secret data key for the OpenStack Swift password. KeySwiftUsername = "username" - saTokenVolumeK8sDirectory = "/var/run/secrets/kubernetes.io/serviceaccount" - SATokenVolumeOcpDirectory = "/var/run/secrets/openshift/serviceaccount" - saTokenVolumeName = "bound-sa-token" - saTokenExpiration int64 = 3600 + saTokenVolumeName = "bound-sa-token" + saTokenExpiration int64 = 3600 + saTokenVolumeMountPath = "/var/run/secrets/storage/serviceaccount" + + ServiceAccountTokenFilePath = saTokenVolumeMountPath + "/token" - secretDirectory = "/etc/storage/secrets" - managedAuthSecretDirectory = "/etc/storage/managed-auth" - storageTLSVolume = "storage-tls" - caDirectory = "/etc/storage/ca" + secretDirectory = "/etc/storage/secrets" + storageTLSVolume = "storage-tls" + caDirectory = "/etc/storage/ca" - awsDefaultAudience = "sts.amazonaws.com" - AWSOpenShiftAudience = "openshift" + managedAuthConfigVolumeName = "managed-auth-config" + managedAuthConfigDirectory = "/etc/storage/managed-auth" - AnnotationCredentialsRequestsSecretRef = "loki.grafana.com/credentials-request-secret-ref" + awsDefaultAudience = "sts.amazonaws.com" + + azureDefaultAudience = "api://AzureADTokenExchange" + + azureManagedCredentialKeyClientID = "azure_client_id" + azureManagedCredentialKeyTenantID = "azure_tenant_id" + azureManagedCredentialKeySubscriptionID = "azure_subscription_id" ) + +// ManagedCredentialsSecretName returns the name of the secret holding the managed credentials. +func ManagedCredentialsSecretName(stackName string) string { + return fmt.Sprintf("%s-managed-credentials", stackName) +} diff --git a/operator/internal/status/status.go b/operator/internal/status/status.go index 281a167355c3..c544695d3d2e 100644 --- a/operator/internal/status/status.go +++ b/operator/internal/status/status.go @@ -17,7 +17,7 @@ import ( // Refresh executes an aggregate update of the LokiStack Status struct, i.e. // - It recreates the Status.Components pod status map per component. // - It sets the appropriate Status.Condition to true that matches the pod status maps. -func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time, degradedErr *DegradedError) error { +func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time, credentialMode lokiv1.CredentialMode, degradedErr *DegradedError) error { var stack lokiv1.LokiStack if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { if apierrors.IsNotFound(err) { @@ -45,6 +45,7 @@ func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request, now time.Time, statusUpdater := func(stack *lokiv1.LokiStack) { stack.Status.Components = *cs stack.Status.Conditions = mergeConditions(stack.Status.Conditions, activeConditions, metaTime) + stack.Status.Storage.CredentialMode = credentialMode } statusUpdater(&stack) diff --git a/operator/internal/status/status_test.go b/operator/internal/status/status_test.go index c7895cbe8020..32ef892ed1bd 100644 --- a/operator/internal/status/status_test.go +++ b/operator/internal/status/status_test.go @@ -54,7 +54,9 @@ func TestRefreshSuccess(t *testing.T) { Gateway: map[corev1.PodPhase][]string{corev1.PodRunning: {"lokistack-gateway-pod-0"}}, Ruler: map[corev1.PodPhase][]string{corev1.PodRunning: {"ruler-pod-0"}}, }, - Storage: lokiv1.LokiStackStorageStatus{}, + Storage: lokiv1.LokiStackStorageStatus{ + CredentialMode: lokiv1.CredentialModeStatic, + }, Conditions: []metav1.Condition{ { Type: string(lokiv1.ConditionReady), @@ -68,7 +70,7 @@ func TestRefreshSuccess(t *testing.T) { k, sw := setupListClient(t, stack, componentPods) - err := Refresh(context.Background(), k, req, now, nil) + err := Refresh(context.Background(), k, req, now, lokiv1.CredentialModeStatic, nil) require.NoError(t, err) require.Equal(t, 1, k.GetCallCount()) @@ -130,7 +132,7 @@ func TestRefreshSuccess_ZoneAwarePendingPod(t *testing.T) { return nil } - err := Refresh(context.Background(), k, req, now, nil) + err := Refresh(context.Background(), k, req, now, lokiv1.CredentialModeStatic, nil) require.NoError(t, err) require.Equal(t, 1, k.GetCallCount()) diff --git a/operator/main.go b/operator/main.go index a88a857bcee4..e212c268cbad 100644 --- a/operator/main.go +++ b/operator/main.go @@ -21,7 +21,6 @@ import ( lokiv1beta1 "github.com/grafana/loki/operator/apis/loki/v1beta1" lokictrl "github.com/grafana/loki/operator/controllers/loki" "github.com/grafana/loki/operator/internal/config" - manifestsocp "github.com/grafana/loki/operator/internal/manifests/openshift" "github.com/grafana/loki/operator/internal/metrics" "github.com/grafana/loki/operator/internal/operator" "github.com/grafana/loki/operator/internal/validation" @@ -60,12 +59,16 @@ func main() { var err error - ctrlCfg, options, err := config.LoadConfig(scheme, configFile) + ctrlCfg, managedAuth, options, err := config.LoadConfig(scheme, configFile) if err != nil { logger.Error(err, "failed to load operator configuration") os.Exit(1) } + if managedAuth != nil { + logger.Info("Discovered OpenShift Cluster within a managed authentication environment") + } + if ctrlCfg.Gates.LokiStackAlerts && !ctrlCfg.Gates.ServiceMonitors { logger.Error(kverrors.New("LokiStackAlerts flag requires ServiceMonitors"), "") os.Exit(1) @@ -95,16 +98,12 @@ func main() { os.Exit(1) } - if ctrlCfg.Gates.OpenShift.Enabled && manifestsocp.DiscoverManagedAuthEnv() != nil { - logger.Info("discovered OpenShift Cluster within a managed authentication environment") - ctrlCfg.Gates.OpenShift.ManagedAuthEnv = true - } - if err = (&lokictrl.LokiStackReconciler{ Client: mgr.GetClient(), Log: logger.WithName("controllers").WithName("lokistack"), Scheme: mgr.GetScheme(), FeatureGates: ctrlCfg.Gates, + AuthConfig: managedAuth, }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "lokistack") os.Exit(1) @@ -129,17 +128,6 @@ func main() { } } - if ctrlCfg.Gates.OpenShift.ManagedAuthEnabled() { - if err = (&lokictrl.CredentialsRequestsReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logger.WithName("controllers").WithName("lokistack-credentialsrequest"), - }).SetupWithManager(mgr); err != nil { - logger.Error(err, "unable to create controller", "controller", "lokistack-credentialsrequest") - os.Exit(1) - } - } - if ctrlCfg.Gates.LokiStackWebhook { v := &validation.LokiStackValidator{} if err = v.SetupWebhookWithManager(mgr); err != nil { diff --git a/pkg/bloomcompactor/batch.go b/pkg/bloomcompactor/batch.go new file mode 100644 index 000000000000..660f5642b464 --- /dev/null +++ b/pkg/bloomcompactor/batch.go @@ -0,0 +1,356 @@ +package bloomcompactor + +import ( + "context" + "io" + "math" + "time" + + "github.com/grafana/dskit/multierror" + "golang.org/x/exp/slices" + + "github.com/grafana/loki/pkg/chunkenc" + "github.com/grafana/loki/pkg/logproto" + logql_log "github.com/grafana/loki/pkg/logql/log" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/chunk" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" +) + +type Fetcher[A, B any] interface { + Fetch(ctx context.Context, inputs []A) ([]B, error) +} + +type FetchFunc[A, B any] func(ctx context.Context, inputs []A) ([]B, error) + +func (f FetchFunc[A, B]) Fetch(ctx context.Context, inputs []A) ([]B, error) { + return f(ctx, inputs) +} + +// batchedLoader implements `v1.Iterator[C]` in batches +type batchedLoader[A, B, C any] struct { + metrics *Metrics + batchSize int + ctx context.Context + fetchers []Fetcher[A, B] + work [][]A + + mapper func(B) (C, error) + cur C + batch []B + err error +} + +const batchedLoaderDefaultBatchSize = 50 + +func newBatchedLoader[A, B, C any]( + ctx context.Context, + fetchers []Fetcher[A, B], + inputs [][]A, + mapper func(B) (C, error), + batchSize int, +) *batchedLoader[A, B, C] { + return &batchedLoader[A, B, C]{ + batchSize: max(batchSize, 1), + ctx: ctx, + fetchers: fetchers, + work: inputs, + mapper: mapper, + } +} + +func (b *batchedLoader[A, B, C]) Next() bool { + + // iterate work until we have non-zero length batch + for len(b.batch) == 0 { + + // empty batch + no work remaining = we're done + if len(b.work) == 0 { + return false + } + + // setup next batch + next := b.work[0] + batchSize := min(b.batchSize, len(next)) + toFetch := next[:batchSize] + fetcher := b.fetchers[0] + + // update work + b.work[0] = b.work[0][batchSize:] + if len(b.work[0]) == 0 { + // if we've exhausted work from this set of inputs, + // set pointer to next set of inputs + // and their respective fetcher + b.work = b.work[1:] + b.fetchers = b.fetchers[1:] + } + + // there was no work in this batch; continue (should not happen) + if len(toFetch) == 0 { + continue + } + + b.batch, b.err = fetcher.Fetch(b.ctx, toFetch) + // error fetching, short-circuit iteration + if b.err != nil { + return false + } + } + + return b.prepNext() +} + +func (b *batchedLoader[_, B, C]) prepNext() bool { + b.cur, b.err = b.mapper(b.batch[0]) + b.batch = b.batch[1:] + return b.err == nil +} + +func (b *batchedLoader[_, _, C]) At() C { + return b.cur +} + +func (b *batchedLoader[_, _, _]) Err() error { + return b.err +} + +// to ensure memory is bounded while loading chunks +// TODO(owen-d): testware +func newBatchedChunkLoader( + ctx context.Context, + fetchers []Fetcher[chunk.Chunk, chunk.Chunk], + inputs [][]chunk.Chunk, + metrics *Metrics, + batchSize int, +) *batchedLoader[chunk.Chunk, chunk.Chunk, v1.ChunkRefWithIter] { + + mapper := func(c chunk.Chunk) (v1.ChunkRefWithIter, error) { + chk := c.Data.(*chunkenc.Facade).LokiChunk() + metrics.chunkSize.Observe(float64(chk.UncompressedSize())) + itr, err := chk.Iterator( + ctx, + time.Unix(0, 0), + time.Unix(0, math.MaxInt64), + logproto.FORWARD, + logql_log.NewNoopPipeline().ForStream(nil), + ) + + if err != nil { + return v1.ChunkRefWithIter{}, err + } + + return v1.ChunkRefWithIter{ + Ref: v1.ChunkRef{ + Start: c.From, + End: c.Through, + Checksum: c.Checksum, + }, + Itr: itr, + }, nil + } + return newBatchedLoader(ctx, fetchers, inputs, mapper, batchSize) +} + +func newBatchedBlockLoader( + ctx context.Context, + fetcher Fetcher[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier], + blocks []bloomshipper.BlockRef, + batchSize int, +) *batchedLoader[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier, *bloomshipper.CloseableBlockQuerier] { + + fetchers := []Fetcher[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier]{fetcher} + inputs := [][]bloomshipper.BlockRef{blocks} + mapper := func(a *bloomshipper.CloseableBlockQuerier) (*bloomshipper.CloseableBlockQuerier, error) { + return a, nil + } + + return newBatchedLoader(ctx, fetchers, inputs, mapper, batchSize) +} + +// compiler checks +var _ v1.Iterator[*v1.SeriesWithBloom] = &blockLoadingIter{} +var _ v1.CloseableIterator[*v1.SeriesWithBloom] = &blockLoadingIter{} +var _ v1.ResettableIterator[*v1.SeriesWithBloom] = &blockLoadingIter{} + +// TODO(chaudum): testware +func newBlockLoadingIter(ctx context.Context, blocks []bloomshipper.BlockRef, fetcher FetchFunc[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier], batchSize int) *blockLoadingIter { + + return &blockLoadingIter{ + ctx: ctx, + fetcher: fetcher, + inputs: blocks, + batchSize: batchSize, + loaded: make(map[io.Closer]struct{}), + } +} + +type blockLoadingIter struct { + // constructor arguments + ctx context.Context + fetcher Fetcher[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier] + inputs []bloomshipper.BlockRef + overlapping v1.Iterator[[]bloomshipper.BlockRef] + batchSize int + // optional arguments + filter func(*bloomshipper.CloseableBlockQuerier) bool + // internals + initialized bool + err error + iter v1.Iterator[*v1.SeriesWithBloom] + loader *batchedLoader[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier, *bloomshipper.CloseableBlockQuerier] + loaded map[io.Closer]struct{} +} + +// At implements v1.Iterator. +func (i *blockLoadingIter) At() *v1.SeriesWithBloom { + if !i.initialized { + panic("iterator not initialized") + } + return i.iter.At() +} + +// Err implements v1.Iterator. +func (i *blockLoadingIter) Err() error { + if !i.initialized { + panic("iterator not initialized") + } + if i.err != nil { + return i.err + } + return i.iter.Err() +} + +func (i *blockLoadingIter) init() { + if i.initialized { + return + } + + // group overlapping blocks + i.overlapping = overlappingBlocksIter(i.inputs) + + // set initial iter + i.iter = v1.NewEmptyIter[*v1.SeriesWithBloom]() + + // set "match all" filter function if not present + if i.filter == nil { + i.filter = func(cbq *bloomshipper.CloseableBlockQuerier) bool { return true } + } + + // done + i.initialized = true +} + +// load next populates the underlying iter via relevant batches +// and returns the result of iter.Next() +func (i *blockLoadingIter) loadNext() bool { + for i.overlapping.Next() { + blockRefs := i.overlapping.At() + + loader := newBatchedBlockLoader(i.ctx, i.fetcher, blockRefs, i.batchSize) + filtered := v1.NewFilterIter[*bloomshipper.CloseableBlockQuerier](loader, i.filter) + + iters := make([]v1.PeekingIterator[*v1.SeriesWithBloom], 0, len(blockRefs)) + for filtered.Next() { + bq := loader.At() + i.loaded[bq] = struct{}{} + iter, err := bq.SeriesIter() + if err != nil { + i.err = err + i.iter = v1.NewEmptyIter[*v1.SeriesWithBloom]() + return false + } + iters = append(iters, iter) + } + + if err := filtered.Err(); err != nil { + i.err = err + i.iter = v1.NewEmptyIter[*v1.SeriesWithBloom]() + return false + } + + // edge case: we've filtered out all blocks in the batch; check next batch + if len(iters) == 0 { + continue + } + + // Turn the list of blocks into a single iterator that returns the next series + mergedBlocks := v1.NewHeapIterForSeriesWithBloom(iters...) + // two overlapping blocks can conceivably have the same series, so we need to dedupe, + // preferring the one with the most chunks already indexed since we'll have + // to add fewer chunks to the bloom + i.iter = v1.NewDedupingIter[*v1.SeriesWithBloom, *v1.SeriesWithBloom]( + func(a, b *v1.SeriesWithBloom) bool { + return a.Series.Fingerprint == b.Series.Fingerprint + }, + v1.Identity[*v1.SeriesWithBloom], + func(a, b *v1.SeriesWithBloom) *v1.SeriesWithBloom { + if len(a.Series.Chunks) > len(b.Series.Chunks) { + return a + } + return b + }, + v1.NewPeekingIter(mergedBlocks), + ) + return i.iter.Next() + } + + i.iter = v1.NewEmptyIter[*v1.SeriesWithBloom]() + i.err = i.overlapping.Err() + return false +} + +// Next implements v1.Iterator. +func (i *blockLoadingIter) Next() bool { + i.init() + return i.iter.Next() || i.loadNext() +} + +// Close implements v1.CloseableIterator. +func (i *blockLoadingIter) Close() error { + var err multierror.MultiError + for k := range i.loaded { + err.Add(k.Close()) + } + return err.Err() +} + +// Reset implements v1.ResettableIterator. +// TODO(chaudum) Cache already fetched blocks to to avoid the overhead of +// creating the reader. +func (i *blockLoadingIter) Reset() error { + if !i.initialized { + return nil + } + // close loaded queriers + err := i.Close() + i.initialized = false + clear(i.loaded) + return err +} + +func (i *blockLoadingIter) Filter(filter func(*bloomshipper.CloseableBlockQuerier) bool) { + if i.initialized { + panic("iterator already initialized") + } + i.filter = filter +} + +func overlappingBlocksIter(inputs []bloomshipper.BlockRef) v1.Iterator[[]bloomshipper.BlockRef] { + // can we assume sorted blocks? + peekIter := v1.NewPeekingIter(v1.NewSliceIter(inputs)) + + return v1.NewDedupingIter[bloomshipper.BlockRef, []bloomshipper.BlockRef]( + func(a bloomshipper.BlockRef, b []bloomshipper.BlockRef) bool { + minFp := b[0].Bounds.Min + maxFp := slices.MaxFunc(b, func(a, b bloomshipper.BlockRef) int { return int(a.Bounds.Max - b.Bounds.Max) }).Bounds.Max + return a.Bounds.Overlaps(v1.NewBounds(minFp, maxFp)) + }, + func(a bloomshipper.BlockRef) []bloomshipper.BlockRef { + return []bloomshipper.BlockRef{a} + }, + func(a bloomshipper.BlockRef, b []bloomshipper.BlockRef) []bloomshipper.BlockRef { + return append(b, a) + }, + peekIter, + ) +} diff --git a/pkg/bloomcompactor/batch_test.go b/pkg/bloomcompactor/batch_test.go new file mode 100644 index 000000000000..bd2cb3378cfb --- /dev/null +++ b/pkg/bloomcompactor/batch_test.go @@ -0,0 +1,210 @@ +package bloomcompactor + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" +) + +func TestBatchedLoader(t *testing.T) { + t.Parallel() + + errMapper := func(i int) (int, error) { + return 0, errors.New("bzzt") + } + successMapper := func(i int) (int, error) { + return i, nil + } + + expired, cancel := context.WithCancel(context.Background()) + cancel() + + for _, tc := range []struct { + desc string + ctx context.Context + batchSize int + mapper func(int) (int, error) + err bool + inputs [][]int + exp []int + }{ + { + desc: "OneBatch", + ctx: context.Background(), + batchSize: 2, + mapper: successMapper, + err: false, + inputs: [][]int{{0, 1}}, + exp: []int{0, 1}, + }, + { + desc: "ZeroBatchSizeStillWorks", + ctx: context.Background(), + batchSize: 0, + mapper: successMapper, + err: false, + inputs: [][]int{{0, 1}}, + exp: []int{0, 1}, + }, + { + desc: "OneBatchLessThanFull", + ctx: context.Background(), + batchSize: 2, + mapper: successMapper, + err: false, + inputs: [][]int{{0}}, + exp: []int{0}, + }, + { + desc: "TwoBatches", + ctx: context.Background(), + batchSize: 2, + mapper: successMapper, + err: false, + inputs: [][]int{{0, 1, 2, 3}}, + exp: []int{0, 1, 2, 3}, + }, + { + desc: "MultipleBatchesMultipleLoaders", + ctx: context.Background(), + batchSize: 2, + mapper: successMapper, + err: false, + inputs: [][]int{{0, 1}, {2}, {3, 4, 5}}, + exp: []int{0, 1, 2, 3, 4, 5}, + }, + { + desc: "HandlesEmptyInputs", + ctx: context.Background(), + batchSize: 2, + mapper: successMapper, + err: false, + inputs: [][]int{{0, 1, 2, 3}, nil, {4}}, + exp: []int{0, 1, 2, 3, 4}, + }, + { + desc: "Timeout", + ctx: expired, + batchSize: 2, + mapper: successMapper, + err: true, + inputs: [][]int{{0}}, + }, + { + desc: "MappingFailure", + ctx: context.Background(), + batchSize: 2, + mapper: errMapper, + err: true, + inputs: [][]int{{0}}, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + fetchers := make([]Fetcher[int, int], 0, len(tc.inputs)) + for range tc.inputs { + fetchers = append( + fetchers, + FetchFunc[int, int](func(ctx context.Context, xs []int) ([]int, error) { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return xs, nil + }), + ) + } + + loader := newBatchedLoader[int, int, int]( + tc.ctx, + fetchers, + tc.inputs, + tc.mapper, + tc.batchSize, + ) + + got, err := v1.Collect[int](loader) + if tc.err { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.exp, got) + + }) + } +} + +func TestOverlappingBlocksIter(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + desc string + inp []bloomshipper.BlockRef + exp int // expected groups + }{ + { + desc: "Empty", + inp: []bloomshipper.BlockRef{}, + exp: 0, + }, + { + desc: "NonOverlapping", + inp: []bloomshipper.BlockRef{ + genBlockRef(0x0000, 0x00ff), + genBlockRef(0x0100, 0x01ff), + genBlockRef(0x0200, 0x02ff), + }, + exp: 3, + }, + { + desc: "AllOverlapping", + inp: []bloomshipper.BlockRef{ + genBlockRef(0x0000, 0x02ff), // |-----------| + genBlockRef(0x0100, 0x01ff), // |---| + genBlockRef(0x0200, 0x02ff), // |---| + }, + exp: 1, + }, + { + desc: "PartialOverlapping", + inp: []bloomshipper.BlockRef{ + genBlockRef(0x0000, 0x01ff), // group 1 |-------| + genBlockRef(0x0100, 0x02ff), // group 1 |-------| + genBlockRef(0x0200, 0x03ff), // group 1 |-------| + genBlockRef(0x0200, 0x02ff), // group 1 |---| + }, + exp: 1, + }, + { + desc: "PartialOverlapping", + inp: []bloomshipper.BlockRef{ + genBlockRef(0x0000, 0x01ff), // group 1 |-------| + genBlockRef(0x0100, 0x02ff), // group 1 |-------| + genBlockRef(0x0100, 0x01ff), // group 1 |---| + genBlockRef(0x0300, 0x03ff), // group 2 |---| + genBlockRef(0x0310, 0x03ff), // group 2 |-| + }, + exp: 2, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + it := overlappingBlocksIter(tc.inp) + var overlapping [][]bloomshipper.BlockRef + var i int + for it.Next() && it.Err() == nil { + require.NotNil(t, it.At()) + overlapping = append(overlapping, it.At()) + for _, r := range it.At() { + t.Log(i, r) + } + i++ + } + require.Equal(t, tc.exp, len(overlapping)) + }) + } +} diff --git a/pkg/bloomcompactor/bloomcompactor.go b/pkg/bloomcompactor/bloomcompactor.go index 52799e498b51..da3c70d81b89 100644 --- a/pkg/bloomcompactor/bloomcompactor.go +++ b/pkg/bloomcompactor/bloomcompactor.go @@ -1,36 +1,8 @@ -/* -Bloom-compactor - -This is a standalone service that is responsible for compacting TSDB indexes into bloomfilters. -It creates and merges bloomfilters into an aggregated form, called bloom-blocks. -It maintains a list of references between bloom-blocks and TSDB indexes in files called meta.jsons. - -Bloom-compactor regularly runs to check for changes in meta.jsons and runs compaction only upon changes in TSDBs. - -bloomCompactor.Compactor - - | // Read/Write path - bloomshipper.Store** - | - bloomshipper.Shipper - | - bloomshipper.BloomClient - | - ObjectClient - | - .....................service boundary - | - object storage -*/ package bloomcompactor import ( "context" - "fmt" - "io" - "math" - "math/rand" - "os" + "sync" "time" "github.com/go-kit/log" @@ -38,163 +10,101 @@ import ( "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/concurrency" "github.com/grafana/dskit/multierror" + "github.com/grafana/dskit/ring" "github.com/grafana/dskit/services" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - - "path/filepath" - - "github.com/google/uuid" "github.com/grafana/loki/pkg/bloomutils" "github.com/grafana/loki/pkg/storage" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/chunk/cache" - chunk_client "github.com/grafana/loki/pkg/storage/chunk/client" - "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" + "github.com/grafana/loki/pkg/storage/stores" "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" - shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" - index_storage "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" - tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" - "github.com/grafana/loki/pkg/util" + util_ring "github.com/grafana/loki/pkg/util/ring" +) + +var ( + RingOp = ring.NewOp([]ring.InstanceState{ring.JOINING, ring.ACTIVE}, nil) ) +/* +Bloom-compactor + +This is a standalone service that is responsible for compacting TSDB indexes into bloomfilters. +It creates and merges bloomfilters into an aggregated form, called bloom-blocks. +It maintains a list of references between bloom-blocks and TSDB indexes in files called meta.jsons. + +Bloom-compactor regularly runs to check for changes in meta.jsons and runs compaction only upon changes in TSDBs. +*/ type Compactor struct { services.Service cfg Config - logger log.Logger schemaCfg config.SchemaConfig + logger log.Logger limits Limits - // temporary workaround until store has implemented read/write shipper interface - bloomShipperClient bloomshipper.StoreAndClient + tsdbStore TSDBStore + // TODO(owen-d): ShardingStrategy + controller *SimpleBloomController - // Client used to run operations on the bucket storing bloom blocks. - storeClients map[config.DayTime]storeClient + // temporary workaround until bloomStore has implemented read/write shipper interface + bloomStore bloomshipper.Store - sharding ShardingStrategy + sharding util_ring.TenantSharding - metrics *metrics + metrics *Metrics btMetrics *v1.Metrics - reg prometheus.Registerer -} - -type storeClient struct { - object chunk_client.ObjectClient - index index_storage.Client - chunk chunk_client.Client - indexShipper indexshipper.IndexShipper } func New( cfg Config, - storageCfg storage.Config, - schemaConfig config.SchemaConfig, + schemaCfg config.SchemaConfig, + storeCfg storage.Config, + clientMetrics storage.ClientMetrics, + fetcherProvider stores.ChunkFetcherProvider, + sharding util_ring.TenantSharding, limits Limits, + store bloomshipper.Store, logger log.Logger, - sharding ShardingStrategy, - clientMetrics storage.ClientMetrics, r prometheus.Registerer, ) (*Compactor, error) { c := &Compactor{ - cfg: cfg, - logger: logger, - schemaCfg: schemaConfig, - sharding: sharding, - limits: limits, - reg: r, + cfg: cfg, + schemaCfg: schemaCfg, + logger: logger, + sharding: sharding, + limits: limits, + bloomStore: store, } - // TODO(chaudum): Plug in cache - var metasCache cache.Cache - var blocksCache *cache.EmbeddedCache[string, io.ReadCloser] - bloomClient, err := bloomshipper.NewBloomStore(schemaConfig.Configs, storageCfg, clientMetrics, metasCache, blocksCache, logger) + tsdbStore, err := NewTSDBStores(schemaCfg, storeCfg, clientMetrics) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to create TSDB store") } - - c.storeClients = make(map[config.DayTime]storeClient) + c.tsdbStore = tsdbStore // initialize metrics - c.btMetrics = v1.NewMetrics(prometheus.WrapRegistererWithPrefix("loki_bloom_tokenizer", r)) - - indexShipperReg := prometheus.WrapRegistererWithPrefix("loki_bloom_compactor_tsdb_shipper_", r) - - for i, periodicConfig := range schemaConfig.Configs { - if periodicConfig.IndexType != config.TSDBType { - level.Warn(c.logger).Log("msg", "skipping schema period because index type is not supported", "index_type", periodicConfig.IndexType, "period", periodicConfig.From) - continue - } - - // Configure ObjectClient and IndexShipper for series and chunk management - objectClient, err := storage.NewObjectClient(periodicConfig.ObjectType, storageCfg, clientMetrics) - if err != nil { - return nil, fmt.Errorf("error creating object client '%s': %w", periodicConfig.ObjectType, err) - } - - periodEndTime := config.DayTime{Time: math.MaxInt64} - if i < len(schemaConfig.Configs)-1 { - periodEndTime = config.DayTime{Time: schemaConfig.Configs[i+1].From.Time.Add(-time.Millisecond)} - } - - pReg := prometheus.WrapRegistererWith( - prometheus.Labels{ - "component": fmt.Sprintf( - "index-store-%s-%s", - periodicConfig.IndexType, - periodicConfig.From.String(), - ), - }, indexShipperReg) - pLogger := log.With(logger, "index-store", fmt.Sprintf("%s-%s", periodicConfig.IndexType, periodicConfig.From.String())) - - indexShipper, err := indexshipper.NewIndexShipper( - periodicConfig.IndexTables.PathPrefix, - storageCfg.TSDBShipperConfig, - objectClient, - limits, - nil, - func(p string) (shipperindex.Index, error) { - return tsdb.OpenShippableTSDB(p) - }, - periodicConfig.GetIndexTableNumberRange(periodEndTime), - pReg, - pLogger, - ) - - if err != nil { - return nil, errors.Wrap(err, "create index shipper") - } - - // The ObjectClient does not expose the key encoder it uses, - // so check the concrete type and set the FSEncoder if needed. - var keyEncoder chunk_client.KeyEncoder - switch objectClient.(type) { - case *local.FSObjectClient: - keyEncoder = chunk_client.FSEncoder - } - - c.storeClients[periodicConfig.From] = storeClient{ - object: objectClient, - index: index_storage.NewIndexStorageClient(objectClient, periodicConfig.IndexTables.PathPrefix), - chunk: chunk_client.NewClient(objectClient, keyEncoder, schemaConfig), - indexShipper: indexShipper, - } - } + c.btMetrics = v1.NewMetrics(prometheus.WrapRegistererWithPrefix("loki_bloom_tokenizer_", r)) + c.metrics = NewMetrics(r, c.btMetrics) - // temporary workaround until store has implemented read/write shipper interface - c.bloomShipperClient = bloomClient + chunkLoader := NewStoreChunkLoader( + fetcherProvider, + c.metrics, + ) - c.metrics = newMetrics(r) - c.metrics.compactionRunInterval.Set(cfg.CompactionInterval.Seconds()) + c.controller = NewSimpleBloomController( + c.tsdbStore, + c.bloomStore, + chunkLoader, + c.limits, + c.metrics, + c.logger, + ) c.Service = services.NewBasicService(c.starting, c.running, c.stopping) - return c, nil } @@ -203,259 +113,37 @@ func (c *Compactor) starting(_ context.Context) (err error) { return err } -func (c *Compactor) running(ctx context.Context) error { - // Run an initial compaction before starting the interval. - if err := c.runCompaction(ctx); err != nil { - level.Error(c.logger).Log("msg", "failed to run compaction", "err", err) - } - - ticker := time.NewTicker(util.DurationWithJitter(c.cfg.CompactionInterval, 0.05)) - defer ticker.Stop() - - for { - select { - case start := <-ticker.C: - c.metrics.compactionRunsStarted.Inc() - if err := c.runCompaction(ctx); err != nil { - c.metrics.compactionRunsCompleted.WithLabelValues(statusFailure).Inc() - c.metrics.compactionRunTime.WithLabelValues(statusFailure).Observe(time.Since(start).Seconds()) - level.Error(c.logger).Log("msg", "failed to run compaction", "err", err) - continue - } - c.metrics.compactionRunsCompleted.WithLabelValues(statusSuccess).Inc() - c.metrics.compactionRunTime.WithLabelValues(statusSuccess).Observe(time.Since(start).Seconds()) - case <-ctx.Done(): - return nil - } - } -} - func (c *Compactor) stopping(_ error) error { c.metrics.compactorRunning.Set(0) return nil } -func (c *Compactor) runCompaction(ctx context.Context) error { - var tables []string - for _, sc := range c.storeClients { - // refresh index list cache since previous compaction would have changed the index files in the object store - sc.index.RefreshIndexTableNamesCache(ctx) - tbls, err := sc.index.ListTables(ctx) - if err != nil { - return fmt.Errorf("failed to list tables: %w", err) - } - tables = append(tables, tbls...) - } - - // process most recent tables first - tablesIntervals := getIntervalsForTables(tables) - sortTablesByRange(tables, tablesIntervals) - - parallelism := c.cfg.MaxCompactionParallelism - if parallelism == 0 { - parallelism = len(tables) - } - - // TODO(salvacorts): We currently parallelize at the table level. We may want to parallelize at the tenant and job level as well. - // To do that, we should create a worker pool with c.cfg.MaxCompactionParallelism number of workers. - errs := multierror.New() - _ = concurrency.ForEachJob(ctx, len(tables), parallelism, func(ctx context.Context, i int) error { - tableName := tables[i] - logger := log.With(c.logger, "table", tableName) - err := c.compactTable(ctx, logger, tableName, tablesIntervals[tableName]) - if err != nil { - errs.Add(err) - return nil - } - return nil - }) - - return errs.Err() -} - -func (c *Compactor) compactTable(ctx context.Context, logger log.Logger, tableName string, tableInterval model.Interval) error { - // Ensure the context has not been canceled (ie. compactor shutdown has been triggered). - if err := ctx.Err(); err != nil { - return fmt.Errorf("interrupting compaction of table: %w", err) - } - - schemaCfg, ok := schemaPeriodForTable(c.schemaCfg, tableName) - if !ok { - level.Error(logger).Log("msg", "skipping compaction since we can't find schema for table") - return nil - } - - sc, ok := c.storeClients[schemaCfg.From] - if !ok { - return fmt.Errorf("index store client not found for period starting at %s", schemaCfg.From.String()) - } - - _, tenants, err := sc.index.ListFiles(ctx, tableName, true) - if err != nil { - return fmt.Errorf("failed to list files for table %s: %w", tableName, err) +func (c *Compactor) running(ctx context.Context) error { + // run once at beginning + if err := c.runOne(ctx); err != nil { + return err } - c.metrics.compactionRunDiscoveredTenants.Add(float64(len(tenants))) - level.Info(logger).Log("msg", "discovered tenants from bucket", "users", len(tenants)) - return c.compactUsers(ctx, logger, sc, tableName, tableInterval, tenants) -} - -// See: https://github.com/grafana/mimir/blob/34852137c332d4050e53128481f4f6417daee91e/pkg/compactor/compactor.go#L566-L689 -func (c *Compactor) compactUsers(ctx context.Context, logger log.Logger, sc storeClient, tableName string, tableInterval model.Interval, tenants []string) error { - // Keep track of tenants owned by this shard, so that we can delete the local files for all other users. - errs := multierror.New() - ownedTenants := make(map[string]struct{}, len(tenants)) - for _, tenant := range tenants { - tenantLogger := log.With(logger, "tenant", tenant) - - // Ensure the context has not been canceled (ie. compactor shutdown has been triggered). - if err := ctx.Err(); err != nil { - return fmt.Errorf("interrupting compaction of tenants: %w", err) - } - - // Skip tenant if compaction is not enabled - if !c.limits.BloomCompactorEnabled(tenant) { - level.Info(tenantLogger).Log("msg", "compaction disabled for tenant. Skipping.") - continue - } - - // Skip this table if it is too old for the tenant limits. - now := model.Now() - tableMaxAge := c.limits.BloomCompactorMaxTableAge(tenant) - if tableMaxAge > 0 && tableInterval.Start.Before(now.Add(-tableMaxAge)) { - level.Debug(tenantLogger).Log("msg", "skipping tenant because table is too old", "table-max-age", tableMaxAge, "table-start", tableInterval.Start, "now", now) - continue - } - - // Ensure the tenant ID belongs to our shard. - if !c.sharding.OwnsTenant(tenant) { - c.metrics.compactionRunSkippedTenants.Inc() - level.Debug(tenantLogger).Log("msg", "skipping tenant because it is not owned by this shard") - continue - } + ticker := time.NewTicker(c.cfg.CompactionInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() - ownedTenants[tenant] = struct{}{} - - start := time.Now() - if err := c.compactTenantWithRetries(ctx, tenantLogger, sc, tableName, tenant); err != nil { - switch { - case errors.Is(err, context.Canceled): - // We don't want to count shutdowns as failed compactions because we will pick up with the rest of the compaction after the restart. - level.Info(tenantLogger).Log("msg", "compaction for tenant was interrupted by a shutdown") - return nil - default: - c.metrics.compactionRunTenantsCompleted.WithLabelValues(statusFailure).Inc() - c.metrics.compactionRunTenantsTime.WithLabelValues(statusFailure).Observe(time.Since(start).Seconds()) - level.Error(tenantLogger).Log("msg", "failed to compact tenant", "err", err) - errs.Add(err) + case start := <-ticker.C: + c.metrics.compactionsStarted.Inc() + if err := c.runOne(ctx); err != nil { + level.Error(c.logger).Log("msg", "compaction iteration failed", "err", err, "duration", time.Since(start)) + c.metrics.compactionCompleted.WithLabelValues(statusFailure).Inc() + c.metrics.compactionTime.WithLabelValues(statusFailure).Observe(time.Since(start).Seconds()) + return err } - continue + level.Info(c.logger).Log("msg", "compaction iteration completed", "duration", time.Since(start)) + c.metrics.compactionCompleted.WithLabelValues(statusSuccess).Inc() + c.metrics.compactionTime.WithLabelValues(statusSuccess).Observe(time.Since(start).Seconds()) } - - c.metrics.compactionRunTenantsCompleted.WithLabelValues(statusSuccess).Inc() - c.metrics.compactionRunTenantsTime.WithLabelValues(statusSuccess).Observe(time.Since(start).Seconds()) - level.Info(tenantLogger).Log("msg", "successfully compacted tenant") - } - - return errs.Err() - - // TODO: Delete local files for unowned tenants, if there are any. -} - -func (c *Compactor) compactTenant(ctx context.Context, logger log.Logger, sc storeClient, tableName string, tenant string) error { - level.Info(logger).Log("msg", "starting compaction of tenant") - - // Ensure the context has not been canceled (ie. compactor shutdown has been triggered). - if err := ctx.Err(); err != nil { - return err - } - - // Tokenizer is not thread-safe so we need one per goroutine. - nGramLen := c.limits.BloomNGramLength(tenant) - nGramSkip := c.limits.BloomNGramSkip(tenant) - bt := v1.NewBloomTokenizer(nGramLen, nGramSkip, c.btMetrics) - - errs := multierror.New() - rs, err := c.sharding.GetTenantSubRing(tenant).GetAllHealthy(RingOp) - if err != nil { - return err - } - tokenRanges := bloomutils.GetInstanceWithTokenRange(c.cfg.Ring.InstanceID, rs.Instances) - for _, tr := range tokenRanges { - level.Debug(logger).Log("msg", "got token range for instance", "id", tr.Instance.Id, "min", tr.MinToken, "max", tr.MaxToken) } - - // TODO(owen-d): can be optimized to only query for series within the fp range of the compactor shard(s) rather than scanning all series - // and filtering out the ones that don't belong to the compactor shard(s). - _ = sc.indexShipper.ForEach(ctx, tableName, tenant, func(isMultiTenantIndex bool, idx shipperindex.Index) error { - if isMultiTenantIndex { - // Skip multi-tenant indexes - level.Debug(logger).Log("msg", "skipping multi-tenant index", "table", tableName, "index", idx.Name()) - return nil - } - - tsdbFile, ok := idx.(*tsdb.TSDBFile) - if !ok { - errs.Add(fmt.Errorf("failed to cast to TSDBFile")) - return nil - } - - tsdbIndex, ok := tsdbFile.Index.(*tsdb.TSDBIndex) - if !ok { - errs.Add(fmt.Errorf("failed to cast to TSDBIndex")) - return nil - } - - var seriesMetas []seriesMeta - - err := tsdbIndex.ForSeries( - ctx, nil, - 0, math.MaxInt64, // TODO: Replace with MaxLookBackPeriod - func(labels labels.Labels, fingerprint model.Fingerprint, chksMetas []tsdbindex.ChunkMeta) { - if !tokenRanges.Contains(uint32(fingerprint)) { - return - } - - temp := make([]tsdbindex.ChunkMeta, len(chksMetas)) - ls := labels.Copy() - _ = copy(temp, chksMetas) - //All seriesMetas given a table within fp of this compactor shard - seriesMetas = append(seriesMetas, seriesMeta{seriesFP: fingerprint, seriesLbs: ls, chunkRefs: temp}) - }, - labels.MustNewMatcher(labels.MatchEqual, "", ""), - ) - - if err != nil { - errs.Add(err) - return nil - } - - if len(seriesMetas) == 0 { - level.Debug(logger).Log("msg", "skipping index because it does not have any matching series", "table", tableName, "index", idx.Name()) - return nil - } - - job := NewJob(tenant, tableName, idx.Path(), seriesMetas) - jobLogger := log.With(logger, "job", job.String()) - c.metrics.compactionRunJobStarted.Inc() - - start := time.Now() - err = c.runCompact(ctx, jobLogger, job, bt, sc) - if err != nil { - c.metrics.compactionRunJobCompleted.WithLabelValues(statusFailure).Inc() - c.metrics.compactionRunJobTime.WithLabelValues(statusFailure).Observe(time.Since(start).Seconds()) - errs.Add(errors.Wrap(err, fmt.Sprintf("runBloomCompact failed for job %s", job.String()))) - return nil - } - - c.metrics.compactionRunJobCompleted.WithLabelValues(statusSuccess).Inc() - c.metrics.compactionRunJobTime.WithLabelValues(statusSuccess).Observe(time.Since(start).Seconds()) - level.Debug(logger).Log("msg", "compaction of job succeeded", "job", job.String(), "duration", time.Since(start)) - - return nil - }) - - return errs.Err() } func runWithRetries( @@ -484,175 +172,213 @@ func runWithRetries( return lastErr } -func (c *Compactor) compactTenantWithRetries(ctx context.Context, logger log.Logger, sc storeClient, tableName string, tenant string) error { - return runWithRetries( - ctx, - c.cfg.RetryMinBackoff, - c.cfg.RetryMaxBackoff, - c.cfg.CompactionRetries, - func(ctx context.Context) error { - return c.compactTenant(ctx, logger, sc, tableName, tenant) - }, - ) +type tenantTable struct { + tenant string + table config.DayTable + ownershipRange v1.FingerprintBounds } -func (c *Compactor) runCompact(ctx context.Context, logger log.Logger, job Job, bt *v1.BloomTokenizer, storeClient storeClient) error { - // Ensure the context has not been canceled (ie. compactor shutdown has been triggered). - if err := ctx.Err(); err != nil { - return err +func (c *Compactor) tenants(ctx context.Context, table config.DayTable) (v1.Iterator[string], error) { + tenants, err := c.tsdbStore.UsersForPeriod(ctx, table) + if err != nil { + return nil, errors.Wrap(err, "getting tenants") } - metaSearchParams := bloomshipper.MetaSearchParams{ - TenantID: job.tenantID, - Keyspace: bloomshipper.Keyspace{Min: job.minFp, Max: job.maxFp}, - Interval: bloomshipper.Interval{Start: job.from, End: job.through}, + + return v1.NewSliceIter(tenants), nil +} + +// ownsTenant returns the ownership range for the tenant, if the compactor owns the tenant, and an error. +func (c *Compactor) ownsTenant(tenant string) ([]v1.FingerprintBounds, bool, error) { + tenantRing, owned := c.sharding.OwnsTenant(tenant) + if !owned { + return nil, false, nil } - var metas []bloomshipper.Meta - //TODO Configure pool for these to avoid allocations - var activeBloomBlocksRefs []bloomshipper.BlockRef - metaRefs, fetchers, err := c.bloomShipperClient.ResolveMetas(ctx, metaSearchParams) + // TODO(owen-d): use .GetTokenRangesForInstance() + // when it's supported for non zone-aware rings + // instead of doing all this manually + + rs, err := tenantRing.GetAllHealthy(RingOp) if err != nil { - return err + return nil, false, errors.Wrap(err, "getting ring healthy instances") } - for i := range fetchers { - res, err := fetchers[i].FetchMetas(ctx, metaRefs[i]) - if err != nil { - return err - } - metas = append(metas, res...) + ranges, err := bloomutils.TokenRangesForInstance(c.cfg.Ring.InstanceID, rs.Instances) + if err != nil { + return nil, false, errors.Wrap(err, "getting token ranges for instance") } - // TODO This logic currently is NOT concerned with cutting blocks upon topology changes to bloom-compactors. - // It may create blocks with series outside of the fp range of the compactor. Cutting blocks will be addressed in a follow-up PR. - metasMatchingJob, blocksMatchingJob := matchingBlocks(metas, job) - - localDst := createLocalDirName(c.cfg.WorkingDirectory, job) - blockOptions := v1.NewBlockOptions(bt.GetNGramLength(), bt.GetNGramSkip()) + keyspaces := bloomutils.KeyspacesFromTokenRanges(ranges) + return keyspaces, true, nil +} - defer func() { - //clean up the bloom directory - if err := os.RemoveAll(localDst); err != nil { - level.Error(logger).Log("msg", "failed to remove block directory", "dir", localDst, "err", err) - } +// runs a single round of compaction for all relevant tenants and tables +func (c *Compactor) runOne(ctx context.Context) error { + level.Info(c.logger).Log("msg", "running bloom compaction", "workers", c.cfg.WorkerParallelism) + var workersErr error + var wg sync.WaitGroup + ch := make(chan tenantTable) + wg.Add(1) + go func() { + workersErr = c.runWorkers(ctx, ch) + wg.Done() }() - var resultingBlock bloomshipper.Block - defer func() { - if resultingBlock.Data != nil { - _ = resultingBlock.Data.Close() - } - }() + err := c.loadWork(ctx, ch) - level.Info(logger).Log("msg", "started compacting table", "table", job.tableName, "tenant", job.tenantID) - if len(blocksMatchingJob) == 0 && len(metasMatchingJob) > 0 { - // There is no change to any blocks, no compaction needed - level.Info(logger).Log("msg", "No changes to tsdb, no compaction needed") - return nil - } else if len(metasMatchingJob) == 0 { - // No matching existing blocks for this job, compact all series from scratch - level.Info(logger).Log("msg", "No matching existing blocks for this job, compact all series from scratch") + wg.Wait() + err = multierror.New(workersErr, err, ctx.Err()).Err() + if err != nil { + level.Error(c.logger).Log("msg", "compaction iteration failed", "err", err) + } + return err +} - builder, err := NewPersistentBlockBuilder(localDst, blockOptions) - if err != nil { - level.Error(logger).Log("msg", "failed creating block builder", "err", err) - return err - } +func (c *Compactor) tables(ts time.Time) *dayRangeIterator { + // adjust the minimum by one to make it inclusive, which is more intuitive + // for a configuration variable + adjustedMin := min(c.cfg.MinTableCompactionPeriod - 1) + minCompactionPeriod := time.Duration(adjustedMin) * config.ObjectStorageIndexRequiredPeriod + maxCompactionPeriod := time.Duration(c.cfg.MaxTableCompactionPeriod) * config.ObjectStorageIndexRequiredPeriod - // NB(owen-d): this panics/etc, but the code is being refactored and will be removed. I've replaced `bt` with `nil` - // to pass compiler checks while keeping this code around as reference - resultingBlock, err = compactNewChunks(ctx, logger, job, nil, storeClient.chunk, builder, c.limits) - if err != nil { - return level.Error(logger).Log("msg", "failed compacting new chunks", "err", err) - } + from := ts.Add(-maxCompactionPeriod).UnixNano() / int64(config.ObjectStorageIndexRequiredPeriod) * int64(config.ObjectStorageIndexRequiredPeriod) + through := ts.Add(-minCompactionPeriod).UnixNano() / int64(config.ObjectStorageIndexRequiredPeriod) * int64(config.ObjectStorageIndexRequiredPeriod) - } else if len(blocksMatchingJob) > 0 { - // When already compacted metas exists, we need to merge all blocks with amending blooms with new series - level.Info(logger).Log("msg", "already compacted metas exists, use mergeBlockBuilder") + fromDay := config.NewDayTime(model.TimeFromUnixNano(from)) + throughDay := config.NewDayTime(model.TimeFromUnixNano(through)) + level.Debug(c.logger).Log("msg", "loaded tables for compaction", "from", fromDay, "through", throughDay) + return newDayRangeIterator(fromDay, throughDay, c.schemaCfg) +} - var populate = createPopulateFunc(ctx, job, storeClient, bt, c.limits) +func (c *Compactor) loadWork(ctx context.Context, ch chan<- tenantTable) error { + tables := c.tables(time.Now()) - seriesIter := makeSeriesIterFromSeriesMeta(job) + for tables.Next() && tables.Err() == nil && ctx.Err() == nil { + table := tables.At() - blockIters, blockPaths, err := makeBlockIterFromBlocks(ctx, logger, c.bloomShipperClient, blocksMatchingJob, c.cfg.WorkingDirectory) - defer func() { - for _, path := range blockPaths { - if err := os.RemoveAll(path); err != nil { - level.Error(logger).Log("msg", "failed removing uncompressed bloomDir", "dir", path, "err", err) - } - } - }() + level.Debug(c.logger).Log("msg", "loading work for table", "table", table) + tenants, err := c.tenants(ctx, table) if err != nil { - level.Error(logger).Log("err", err) - return err + return errors.Wrap(err, "getting tenants") } - mergeBlockBuilder, err := NewPersistentBlockBuilder(localDst, blockOptions) - if err != nil { - level.Error(logger).Log("msg", "failed creating block builder", "err", err) - return err + for tenants.Next() && tenants.Err() == nil && ctx.Err() == nil { + c.metrics.tenantsDiscovered.Inc() + tenant := tenants.At() + ownershipRanges, owns, err := c.ownsTenant(tenant) + if err != nil { + return errors.Wrap(err, "checking tenant ownership") + } + level.Debug(c.logger).Log("msg", "enqueueing work for tenant", "tenant", tenant, "table", table, "ranges", len(ownershipRanges), "owns", owns) + if !owns { + c.metrics.tenantsSkipped.Inc() + continue + } + c.metrics.tenantsOwned.Inc() + + for _, ownershipRange := range ownershipRanges { + + select { + case ch <- tenantTable{ + tenant: tenant, + table: table, + ownershipRange: ownershipRange, + }: + case <-ctx.Done(): + return ctx.Err() + } + } } - resultingBlock, err = mergeCompactChunks(logger, populate, mergeBlockBuilder, blockIters, seriesIter, job) - if err != nil { - level.Error(logger).Log("msg", "failed merging existing blocks with new chunks", "err", err) - return err + if err := tenants.Err(); err != nil { + level.Error(c.logger).Log("msg", "error iterating tenants", "err", err) + return errors.Wrap(err, "iterating tenants") } } - archivePath := filepath.Join(c.cfg.WorkingDirectory, uuid.New().String()) - - blockToUpload, err := bloomshipper.CompressBloomBlock(resultingBlock.BlockRef, archivePath, localDst, logger) - if err != nil { - level.Error(logger).Log("msg", "failed compressing bloom blocks into tar file", "err", err) - return err + if err := tables.Err(); err != nil { + level.Error(c.logger).Log("msg", "error iterating tables", "err", err) + return errors.Wrap(err, "iterating tables") } - defer func() { - err = os.Remove(archivePath) - if err != nil { - level.Error(logger).Log("msg", "failed removing archive file", "err", err, "file", archivePath) + close(ch) + return ctx.Err() +} + +func (c *Compactor) runWorkers(ctx context.Context, ch <-chan tenantTable) error { + + return concurrency.ForEachJob(ctx, c.cfg.WorkerParallelism, c.cfg.WorkerParallelism, func(ctx context.Context, idx int) error { + + for { + select { + case <-ctx.Done(): + return ctx.Err() + + case tt, ok := <-ch: + if !ok { + return nil + } + + start := time.Now() + c.metrics.tenantsStarted.Inc() + if err := c.compactTenantTable(ctx, tt); err != nil { + c.metrics.tenantsCompleted.WithLabelValues(statusFailure).Inc() + c.metrics.tenantsCompletedTime.WithLabelValues(statusFailure).Observe(time.Since(start).Seconds()) + return errors.Wrapf( + err, + "compacting tenant table (%s) for tenant (%s) with ownership (%s)", + tt.table, + tt.tenant, + tt.ownershipRange, + ) + } + c.metrics.tenantsCompleted.WithLabelValues(statusSuccess).Inc() + c.metrics.tenantsCompletedTime.WithLabelValues(statusSuccess).Observe(time.Since(start).Seconds()) + } } - }() - // Do not change the signature of PutBlocks yet. - // Once block size is limited potentially, compactNewChunks will return multiple blocks, hence a list is appropriate. - storedBlocks, err := c.bloomShipperClient.PutBlocks(ctx, []bloomshipper.Block{blockToUpload}) - if err != nil { - level.Error(logger).Log("msg", "failed uploading blocks to storage", "err", err) - return err - } + }) - // all blocks are new and active blocks - for _, block := range storedBlocks { - activeBloomBlocksRefs = append(activeBloomBlocksRefs, block.BlockRef) - } +} + +func (c *Compactor) compactTenantTable(ctx context.Context, tt tenantTable) error { + level.Info(c.logger).Log("msg", "compacting", "org_id", tt.tenant, "table", tt.table, "ownership", tt.ownershipRange.String()) + return c.controller.compactTenant(ctx, tt.table, tt.tenant, tt.ownershipRange) +} + +type dayRangeIterator struct { + min, max, cur config.DayTime + curPeriod config.PeriodConfig + schemaCfg config.SchemaConfig + err error +} + +func newDayRangeIterator(min, max config.DayTime, schemaCfg config.SchemaConfig) *dayRangeIterator { + return &dayRangeIterator{min: min, max: max, cur: min.Dec(), schemaCfg: schemaCfg} +} - // TODO delete old metas in later compactions - // After all is done, create one meta file and upload to storage - meta := bloomshipper.Meta{ - MetaRef: bloomshipper.MetaRef{ - Ref: bloomshipper.Ref{ - TenantID: job.tenantID, - TableName: job.tableName, - MinFingerprint: uint64(job.minFp), - MaxFingerprint: uint64(job.maxFp), - StartTimestamp: job.from, - EndTimestamp: job.through, - Checksum: rand.Uint32(), // Discuss if checksum is needed for Metas, why should we read all data again. - }, - }, - Tombstones: blocksMatchingJob, - Blocks: activeBloomBlocksRefs, +func (r *dayRangeIterator) Next() bool { + r.cur = r.cur.Inc() + if !r.cur.Before(r.max) { + return false } - err = c.bloomShipperClient.PutMeta(ctx, meta) + period, err := r.schemaCfg.SchemaForTime(r.cur.ModelTime()) if err != nil { - level.Error(logger).Log("msg", "failed uploading meta.json to storage", "err", err) - return err + r.err = errors.Wrapf(err, "getting schema for time (%s)", r.cur) + return false } - level.Info(logger).Log("msg", "finished compacting table", "table", job.tableName, "tenant", job.tenantID) + r.curPeriod = period + + return true +} + +func (r *dayRangeIterator) At() config.DayTable { + return config.NewDayTable(r.cur, r.curPeriod.IndexTables.Prefix) +} + +func (r *dayRangeIterator) Err() error { return nil } diff --git a/pkg/bloomcompactor/bloomcompactor_test.go b/pkg/bloomcompactor/bloomcompactor_test.go index 6221610321b6..0f4306494880 100644 --- a/pkg/bloomcompactor/bloomcompactor_test.go +++ b/pkg/bloomcompactor/bloomcompactor_test.go @@ -4,242 +4,262 @@ import ( "context" "flag" "fmt" - "path/filepath" + "math" "testing" "time" - "github.com/go-kit/log" - "github.com/grafana/dskit/flagext" - "github.com/grafana/dskit/kv" - "github.com/grafana/dskit/kv/consul" "github.com/grafana/dskit/ring" "github.com/grafana/dskit/services" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/compactor" - "github.com/grafana/loki/pkg/storage" - "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/bloomutils" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + util_log "github.com/grafana/loki/pkg/util/log" lokiring "github.com/grafana/loki/pkg/util/ring" + util_ring "github.com/grafana/loki/pkg/util/ring" "github.com/grafana/loki/pkg/validation" ) -const ( - indexTablePrefix = "table_" - workingDirName = "working-dir" -) - -func parseDayTime(s string) config.DayTime { - t, err := time.Parse("2006-01-02", s) - if err != nil { - panic(err) - } - return config.DayTime{ - Time: model.TimeFromUnix(t.Unix()), - } -} - -func TestCompactor_StartStopService(t *testing.T) { - shardingStrategy := NewNoopStrategy() - logger := log.NewNopLogger() - reg := prometheus.NewRegistry() - - cm := storage.NewClientMetrics() - t.Cleanup(cm.Unregister) - - var limits validation.Limits - limits.RegisterFlags(flag.NewFlagSet("limits", flag.PanicOnError)) - overrides, _ := validation.NewOverrides(limits, nil) - - periodConfigUnsupported := config.PeriodConfig{ - From: parseDayTime("2023-09-01"), - IndexType: config.BoltDBShipperType, - ObjectType: config.StorageTypeFileSystem, - Schema: "v13", - RowShards: 16, - IndexTables: config.IndexPeriodicTableConfig{ - PathPrefix: "index/", - PeriodicTableConfig: config.PeriodicTableConfig{ - Prefix: indexTablePrefix, - Period: config.ObjectStorageIndexRequiredPeriod, +func TestCompactor_ownsTenant(t *testing.T) { + for _, tc := range []struct { + name string + limits Limits + compactors int + + expectedCompactorsOwningTenant int + }{ + { + name: "no sharding with one instance", + limits: mockLimits{ + shardSize: 0, }, + compactors: 1, + expectedCompactorsOwningTenant: 1, }, - } - - periodConfigSupported := config.PeriodConfig{ - From: parseDayTime("2023-10-01"), - IndexType: config.TSDBType, - ObjectType: config.StorageTypeFileSystem, - Schema: "v13", - RowShards: 16, - IndexTables: config.IndexPeriodicTableConfig{ - PathPrefix: "index/", - PeriodicTableConfig: config.PeriodicTableConfig{ - Prefix: indexTablePrefix, - Period: config.ObjectStorageIndexRequiredPeriod, + { + name: "no sharding with multiple instances", + limits: mockLimits{ + shardSize: 0, }, + compactors: 10, + expectedCompactorsOwningTenant: 10, }, - } - - schemaCfg := config.SchemaConfig{ - Configs: []config.PeriodConfig{ - periodConfigUnsupported, - periodConfigSupported, + { + name: "sharding with one instance", + limits: mockLimits{ + shardSize: 5, + }, + compactors: 1, + expectedCompactorsOwningTenant: 1, }, + { + name: "sharding with multiple instances", + limits: mockLimits{ + shardSize: 5, + }, + compactors: 10, + expectedCompactorsOwningTenant: 5, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var ringManagers []*lokiring.RingManager + var compactors []*Compactor + for i := 0; i < tc.compactors; i++ { + var ringCfg lokiring.RingConfig + ringCfg.RegisterFlagsWithPrefix("", "", flag.NewFlagSet("ring", flag.PanicOnError)) + ringCfg.KVStore.Store = "inmemory" + ringCfg.InstanceID = fmt.Sprintf("bloom-compactor-%d", i) + ringCfg.InstanceAddr = fmt.Sprintf("localhost-%d", i) + + ringManager, err := lokiring.NewRingManager("bloom-compactor", lokiring.ServerMode, ringCfg, 1, 1, util_log.Logger, prometheus.NewRegistry()) + require.NoError(t, err) + require.NoError(t, ringManager.StartAsync(context.Background())) + + shuffleSharding := util_ring.NewTenantShuffleSharding(ringManager.Ring, ringManager.RingLifecycler, tc.limits.BloomCompactorShardSize) + + compactor := &Compactor{ + cfg: Config{ + Ring: ringCfg, + }, + sharding: shuffleSharding, + limits: tc.limits, + } + + ringManagers = append(ringManagers, ringManager) + compactors = append(compactors, compactor) + } + defer func() { + // Stop all rings and wait for them to stop. + for _, ringManager := range ringManagers { + ringManager.StopAsync() + require.Eventually(t, func() bool { + return ringManager.State() == services.Terminated + }, 1*time.Minute, 100*time.Millisecond) + } + }() + + // Wait for all rings to see each other. + for _, ringManager := range ringManagers { + require.Eventually(t, func() bool { + running := ringManager.State() == services.Running + discovered := ringManager.Ring.InstancesCount() == tc.compactors + return running && discovered + }, 1*time.Minute, 100*time.Millisecond) + } + + var compactorOwnsTenant int + var compactorOwnershipRange []v1.FingerprintBounds + for _, compactor := range compactors { + ownershipRange, ownsTenant, err := compactor.ownsTenant("tenant") + require.NoError(t, err) + if ownsTenant { + compactorOwnsTenant++ + compactorOwnershipRange = append(compactorOwnershipRange, ownershipRange...) + } + } + require.Equal(t, tc.expectedCompactorsOwningTenant, compactorOwnsTenant) + + coveredKeySpace := v1.NewBounds(math.MaxUint64, 0) + for i, boundsA := range compactorOwnershipRange { + for j, boundsB := range compactorOwnershipRange { + if i == j { + continue + } + // Assert that the fingerprint key-space is not overlapping + require.False(t, boundsA.Overlaps(boundsB)) + } + + if boundsA.Min < coveredKeySpace.Min { + coveredKeySpace.Min = boundsA.Min + } + if boundsA.Max > coveredKeySpace.Max { + coveredKeySpace.Max = boundsA.Max + } + + } + // Assert that the fingerprint key-space is complete + require.True(t, coveredKeySpace.Equal(v1.NewBounds(0, math.MaxUint64))) + }) } +} - fsDir := t.TempDir() - tsdbDir := t.TempDir() +type mockLimits struct { + shardSize int +} - storageCfg := storage.Config{ - FSConfig: local.FSConfig{ - Directory: fsDir, - }, - TSDBShipperConfig: indexshipper.Config{ - ActiveIndexDirectory: filepath.Join(tsdbDir, "index"), - ResyncInterval: 1 * time.Minute, - Mode: indexshipper.ModeReadWrite, - CacheLocation: filepath.Join(tsdbDir, "cache"), - }, - } +func (m mockLimits) AllByUserID() map[string]*validation.Limits { + panic("implement me") +} - t.Run("ignore unsupported index types in schema config", func(t *testing.T) { - kvStore, closer := consul.NewInMemoryClient(ring.GetCodec(), logger, reg) - t.Cleanup(func() { - closer.Close() - }) +func (m mockLimits) DefaultLimits() *validation.Limits { + panic("implement me") +} - var cfg Config - flagext.DefaultValues(&cfg) - cfg.Enabled = true - cfg.WorkingDirectory = filepath.Join(t.TempDir(), workingDirName) - cfg.Ring = lokiring.RingConfig{ - KVStore: kv.Config{ - Mock: kvStore, - }, - } +func (m mockLimits) VolumeMaxSeries(_ string) int { + panic("implement me") +} + +func (m mockLimits) BloomCompactorShardSize(_ string) int { + return m.shardSize +} + +func (m mockLimits) BloomCompactorChunksBatchSize(_ string) int { + panic("implement me") +} - c, err := New(cfg, storageCfg, schemaCfg, overrides, logger, shardingStrategy, cm, reg) - require.NoError(t, err) +func (m mockLimits) BloomCompactorMaxTableAge(_ string) time.Duration { + panic("implement me") +} - err = services.StartAndAwaitRunning(context.Background(), c) - require.NoError(t, err) +func (m mockLimits) BloomCompactorEnabled(_ string) bool { + panic("implement me") +} - require.Equal(t, 1, len(c.storeClients)) +func (m mockLimits) BloomNGramLength(_ string) int { + panic("implement me") +} - // supported index type TSDB is present - sc, ok := c.storeClients[periodConfigSupported.From] - require.True(t, ok) - require.NotNil(t, sc) +func (m mockLimits) BloomNGramSkip(_ string) int { + panic("implement me") +} - // unsupported index type BoltDB is not present - _, ok = c.storeClients[periodConfigUnsupported.From] - require.False(t, ok) +func (m mockLimits) BloomFalsePositiveRate(_ string) float64 { + panic("implement me") +} - err = services.StopAndAwaitTerminated(context.Background(), c) - require.NoError(t, err) - }) +func (m mockLimits) BloomCompactorMaxBlockSize(_ string) int { + panic("implement me") } -func TestCompactor_RunCompaction(t *testing.T) { - logger := log.NewNopLogger() - reg := prometheus.NewRegistry() - - cm := storage.NewClientMetrics() - t.Cleanup(cm.Unregister) - - tempDir := t.TempDir() - indexDir := filepath.Join(tempDir, "index") - - schemaCfg := config.SchemaConfig{ - Configs: []config.PeriodConfig{ - { - From: config.DayTime{Time: model.Time(0)}, - IndexType: "tsdb", - ObjectType: "filesystem", - Schema: "v12", - IndexTables: config.IndexPeriodicTableConfig{ - PathPrefix: "index/", - PeriodicTableConfig: config.PeriodicTableConfig{ - Prefix: indexTablePrefix, - Period: config.ObjectStorageIndexRequiredPeriod, - }}, - }, - }, +func TestTokenRangesForInstance(t *testing.T) { + desc := func(id int, tokens ...uint32) ring.InstanceDesc { + return ring.InstanceDesc{Id: fmt.Sprintf("%d", id), Tokens: tokens} } - daySeconds := int64(24 * time.Hour / time.Second) - tableNumEnd := time.Now().Unix() / daySeconds - tableNumStart := tableNumEnd - 5 - for i := tableNumStart; i <= tableNumEnd; i++ { - compactor.SetupTable( - t, - filepath.Join(indexDir, fmt.Sprintf("%s%d", indexTablePrefix, i)), - compactor.IndexesConfig{ - NumUnCompactedFiles: 5, - NumCompactedFiles: 5, + tests := map[string]struct { + input []ring.InstanceDesc + exp map[string]ring.TokenRanges + err bool + }{ + "no nodes": { + input: []ring.InstanceDesc{}, + exp: map[string]ring.TokenRanges{ + "0": {0, math.MaxUint32}, // have to put one in here to trigger test }, - compactor.PerUserIndexesConfig{ - NumUsers: 5, - IndexesConfig: compactor.IndexesConfig{ - NumUnCompactedFiles: 5, - NumCompactedFiles: 5, - }, + err: true, + }, + "one node": { + input: []ring.InstanceDesc{ + desc(0, 0, 100), + }, + exp: map[string]ring.TokenRanges{ + "0": {0, math.MaxUint32}, + }, + }, + "two nodes": { + input: []ring.InstanceDesc{ + desc(0, 25, 75), + desc(1, 10, 50, 100), + }, + exp: map[string]ring.TokenRanges{ + "0": {10, 24, 50, 74}, + "1": {0, 9, 25, 49, 75, math.MaxUint32}, + }, + }, + "consecutive tokens": { + input: []ring.InstanceDesc{ + desc(0, 99), + desc(1, 100), + }, + exp: map[string]ring.TokenRanges{ + "0": {0, 98, 100, math.MaxUint32}, + "1": {99, 99}, + }, + }, + "extremes": { + input: []ring.InstanceDesc{ + desc(0, 0), + desc(1, math.MaxUint32), + }, + exp: map[string]ring.TokenRanges{ + "0": {math.MaxUint32, math.MaxUint32}, + "1": {0, math.MaxUint32 - 1}, }, - ) - } - - kvStore, cleanUp := consul.NewInMemoryClient(ring.GetCodec(), logger, nil) - t.Cleanup(func() { assert.NoError(t, cleanUp.Close()) }) - - var cfg Config - flagext.DefaultValues(&cfg) - cfg.WorkingDirectory = filepath.Join(tempDir, workingDirName) - cfg.Ring.KVStore.Mock = kvStore - cfg.Ring.ListenPort = 0 - cfg.Ring.InstanceAddr = "bloomcompactor" - cfg.Ring.InstanceID = "bloomcompactor" - - storageConfig := storage.Config{ - FSConfig: local.FSConfig{Directory: tempDir}, - TSDBShipperConfig: indexshipper.Config{ - ActiveIndexDirectory: indexDir, - ResyncInterval: 1 * time.Minute, - Mode: indexshipper.ModeReadWrite, - CacheLocation: filepath.Join(tempDir, "cache"), }, } - var limits validation.Limits - limits.RegisterFlags(flag.NewFlagSet("limits", flag.PanicOnError)) - overrides, _ := validation.NewOverrides(limits, nil) - - ringManager, err := lokiring.NewRingManager("bloom-compactor", lokiring.ServerMode, cfg.Ring, 1, 1, logger, reg) - require.NoError(t, err) - - err = ringManager.StartAsync(context.Background()) - require.NoError(t, err) - require.Eventually(t, func() bool { - return ringManager.State() == services.Running - }, 1*time.Minute, 100*time.Millisecond) - defer func() { - ringManager.StopAsync() - require.Eventually(t, func() bool { - return ringManager.State() == services.Terminated - }, 1*time.Minute, 100*time.Millisecond) - }() - - shuffleSharding := NewShuffleShardingStrategy(ringManager.Ring, ringManager.RingLifecycler, overrides) - - c, err := New(cfg, storageConfig, schemaCfg, overrides, logger, shuffleSharding, cm, nil) - require.NoError(t, err) - - err = c.runCompaction(context.Background()) - require.NoError(t, err) - - // TODO: Once compaction is implemented, verify compaction here. + for desc, test := range tests { + t.Run(desc, func(t *testing.T) { + for id := range test.exp { + ranges, err := bloomutils.TokenRangesForInstance(id, test.input) + if test.err { + require.Error(t, err) + continue + } + require.NoError(t, err) + require.Equal(t, test.exp[id], ranges) + } + }) + } } diff --git a/pkg/bloomcompactor/chunkcompactor.go b/pkg/bloomcompactor/chunkcompactor.go deleted file mode 100644 index c4993ccc62a5..000000000000 --- a/pkg/bloomcompactor/chunkcompactor.go +++ /dev/null @@ -1,245 +0,0 @@ -package bloomcompactor - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/google/uuid" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/prometheus/common/model" - - "github.com/grafana/loki/pkg/logproto" - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/bloom/v1/filter" - "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" - tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" -) - -type compactorTokenizer interface { - PopulateSeriesWithBloom(bloom *v1.SeriesWithBloom, chunkBatchesIterator v1.Iterator[[]chunk.Chunk]) error -} - -type chunkClient interface { - // TODO: Consider using lazyChunks to avoid downloading all requested chunks. - GetChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error) -} - -type blockBuilder interface { - BuildFrom(itr v1.Iterator[v1.SeriesWithBloom]) (uint32, error) - Data() (io.ReadSeekCloser, error) -} - -type PersistentBlockBuilder struct { - builder *v1.BlockBuilder - localDst string -} - -func NewPersistentBlockBuilder(localDst string, blockOptions v1.BlockOptions) (*PersistentBlockBuilder, error) { - // write bloom to a local dir - b, err := v1.NewBlockBuilder(blockOptions, v1.NewDirectoryBlockWriter(localDst)) - if err != nil { - return nil, err - } - builder := PersistentBlockBuilder{ - builder: b, - localDst: localDst, - } - return &builder, nil -} - -func (p *PersistentBlockBuilder) BuildFrom(itr v1.Iterator[v1.SeriesWithBloom]) (uint32, error) { - return p.builder.BuildFrom(itr) -} - -func (p *PersistentBlockBuilder) mergeBuild(builder *v1.MergeBuilder) (uint32, error) { - return builder.Build(p.builder) -} - -func (p *PersistentBlockBuilder) Data() (io.ReadSeekCloser, error) { - blockFile, err := os.Open(filepath.Join(p.localDst, v1.BloomFileName)) - if err != nil { - return nil, err - } - return blockFile, nil -} - -func makeChunkRefs(chksMetas []tsdbindex.ChunkMeta, tenant string, fp model.Fingerprint) []chunk.Chunk { - chunkRefs := make([]chunk.Chunk, 0, len(chksMetas)) - for _, chk := range chksMetas { - chunkRefs = append(chunkRefs, chunk.Chunk{ - ChunkRef: logproto.ChunkRef{ - Fingerprint: uint64(fp), - UserID: tenant, - From: chk.From(), - Through: chk.Through(), - Checksum: chk.Checksum, - }, - }) - } - - return chunkRefs -} - -func buildBloomFromSeries(seriesMeta seriesMeta, fpRate float64, tokenizer compactorTokenizer, chunks v1.Iterator[[]chunk.Chunk]) (v1.SeriesWithBloom, error) { - // Create a bloom for this series - bloomForChks := v1.SeriesWithBloom{ - Series: &v1.Series{ - Fingerprint: seriesMeta.seriesFP, - }, - Bloom: &v1.Bloom{ - ScalableBloomFilter: *filter.NewDefaultScalableBloomFilter(fpRate), - }, - } - - // Tokenize data into n-grams - err := tokenizer.PopulateSeriesWithBloom(&bloomForChks, chunks) - if err != nil { - return v1.SeriesWithBloom{}, err - } - return bloomForChks, nil -} - -// TODO Test this when bloom block size check is implemented -func buildBlockFromBlooms( - ctx context.Context, - logger log.Logger, - builder blockBuilder, - blooms v1.Iterator[v1.SeriesWithBloom], - job Job, -) (bloomshipper.Block, error) { - // Ensure the context has not been canceled (ie. compactor shutdown has been triggered). - if err := ctx.Err(); err != nil { - return bloomshipper.Block{}, err - } - - checksum, err := builder.BuildFrom(blooms) - if err != nil { - level.Error(logger).Log("msg", "failed writing to bloom", "err", err) - return bloomshipper.Block{}, err - } - - data, err := builder.Data() - if err != nil { - level.Error(logger).Log("msg", "failed reading bloom data", "err", err) - return bloomshipper.Block{}, err - } - - block := bloomshipper.Block{ - BlockRef: bloomshipper.BlockRef{ - Ref: bloomshipper.Ref{ - TenantID: job.tenantID, - TableName: job.tableName, - MinFingerprint: uint64(job.minFp), - MaxFingerprint: uint64(job.maxFp), - StartTimestamp: job.from, - EndTimestamp: job.through, - Checksum: checksum, - }, - IndexPath: job.indexPath, - }, - Data: data, - } - - return block, nil -} - -func createLocalDirName(workingDir string, job Job) string { - dir := fmt.Sprintf("bloomBlock-%s-%s-%s-%s-%d-%d-%s", job.tableName, job.tenantID, job.minFp, job.maxFp, job.from, job.through, uuid.New().String()) - return filepath.Join(workingDir, dir) -} - -// Compacts given list of chunks, uploads them to storage and returns a list of bloomBlocks -func compactNewChunks(ctx context.Context, - logger log.Logger, - job Job, - bt compactorTokenizer, - storeClient chunkClient, - builder blockBuilder, - limits Limits, -) (bloomshipper.Block, error) { - // Ensure the context has not been canceled (ie. compactor shutdown has been triggered). - if err := ctx.Err(); err != nil { - return bloomshipper.Block{}, err - } - - bloomIter := newLazyBloomBuilder(ctx, job, storeClient, bt, logger, limits) - - // Build and upload bloomBlock to storage - block, err := buildBlockFromBlooms(ctx, logger, builder, bloomIter, job) - if err != nil { - level.Error(logger).Log("msg", "failed building bloomBlocks", "err", err) - return bloomshipper.Block{}, err - } - - return block, nil -} - -type lazyBloomBuilder struct { - ctx context.Context - metas v1.Iterator[seriesMeta] - tenant string - client chunkClient - bt compactorTokenizer - fpRate float64 - logger log.Logger - chunksBatchSize int - - cur v1.SeriesWithBloom // retured by At() - err error // returned by Err() -} - -// newLazyBloomBuilder returns an iterator that yields v1.SeriesWithBloom -// which are used by the blockBuilder to write a bloom block. -// We use an interator to avoid loading all blooms into memory first, before -// building the block. -func newLazyBloomBuilder(ctx context.Context, job Job, client chunkClient, bt compactorTokenizer, logger log.Logger, limits Limits) *lazyBloomBuilder { - return &lazyBloomBuilder{ - ctx: ctx, - metas: v1.NewSliceIter(job.seriesMetas), - client: client, - tenant: job.tenantID, - bt: bt, - fpRate: limits.BloomFalsePositiveRate(job.tenantID), - logger: logger, - chunksBatchSize: limits.BloomCompactorChunksBatchSize(job.tenantID), - } -} - -func (it *lazyBloomBuilder) Next() bool { - if !it.metas.Next() { - it.cur = v1.SeriesWithBloom{} - level.Debug(it.logger).Log("msg", "No seriesMeta") - return false - } - meta := it.metas.At() - - batchesIterator, err := newChunkBatchesIterator(it.ctx, it.client, makeChunkRefs(meta.chunkRefs, it.tenant, meta.seriesFP), it.chunksBatchSize) - if err != nil { - it.err = err - it.cur = v1.SeriesWithBloom{} - level.Debug(it.logger).Log("msg", "err creating chunks batches iterator", "err", err) - return false - } - it.cur, err = buildBloomFromSeries(meta, it.fpRate, it.bt, batchesIterator) - if err != nil { - it.err = err - it.cur = v1.SeriesWithBloom{} - level.Debug(it.logger).Log("msg", "err in buildBloomFromSeries", "err", err) - return false - } - return true -} - -func (it *lazyBloomBuilder) At() v1.SeriesWithBloom { - return it.cur -} - -func (it *lazyBloomBuilder) Err() error { - return it.err -} diff --git a/pkg/bloomcompactor/chunkcompactor_test.go b/pkg/bloomcompactor/chunkcompactor_test.go deleted file mode 100644 index 8bc94fd26537..000000000000 --- a/pkg/bloomcompactor/chunkcompactor_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package bloomcompactor - -import ( - "context" - "io" - "testing" - "time" - - "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - - "github.com/stretchr/testify/require" - - "github.com/grafana/loki/pkg/chunkenc" - "github.com/grafana/loki/pkg/push" - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" -) - -var ( - userID = "userID" - fpRate = 0.01 - - from = model.Earliest - to = model.Latest - - table = "test_table" - indexPath = "index_test_table" - - testBlockSize = 256 * 1024 - testTargetSize = 1500 * 1024 -) - -func createTestChunk(fp model.Fingerprint, lb labels.Labels) chunk.Chunk { - memChunk := chunkenc.NewMemChunk(chunkenc.ChunkFormatV4, chunkenc.EncSnappy, chunkenc.ChunkHeadFormatFor(chunkenc.ChunkFormatV4), testBlockSize, testTargetSize) - if err := memChunk.Append(&push.Entry{ - Timestamp: time.Unix(0, 1), - Line: "this is a log line", - }); err != nil { - panic(err) - } - c := chunk.NewChunk(userID, - fp, lb, chunkenc.NewFacade(memChunk, testBlockSize, testTargetSize), from, to) - - return c -} - -// Given a seriesMeta and corresponding chunks verify SeriesWithBloom can be built -func TestChunkCompactor_BuildBloomFromSeries(t *testing.T) { - label := labels.FromStrings("foo", "bar") - fp := model.Fingerprint(label.Hash()) - seriesMeta := seriesMeta{ - seriesFP: fp, - seriesLbs: label, - } - - chunks := []chunk.Chunk{createTestChunk(fp, label)} - - mbt := mockBloomTokenizer{} - bloom, err := buildBloomFromSeries(seriesMeta, fpRate, &mbt, v1.NewSliceIter([][]chunk.Chunk{chunks})) - require.NoError(t, err) - require.Equal(t, seriesMeta.seriesFP, bloom.Series.Fingerprint) - require.Equal(t, chunks, mbt.chunks) -} - -func TestChunkCompactor_CompactNewChunks(t *testing.T) { - // Setup - logger := log.NewNopLogger() - label := labels.FromStrings("foo", "bar") - fp1 := model.Fingerprint(100) - fp2 := model.Fingerprint(999) - fp3 := model.Fingerprint(200) - - chunkRef1 := index.ChunkMeta{ - Checksum: 1, - MinTime: 1, - MaxTime: 99, - } - - chunkRef2 := index.ChunkMeta{ - Checksum: 2, - MinTime: 10, - MaxTime: 999, - } - - seriesMetas := []seriesMeta{ - { - seriesFP: fp1, - seriesLbs: label, - chunkRefs: []index.ChunkMeta{chunkRef1}, - }, - { - seriesFP: fp2, - seriesLbs: label, - chunkRefs: []index.ChunkMeta{chunkRef1, chunkRef2}, - }, - { - seriesFP: fp3, - seriesLbs: label, - chunkRefs: []index.ChunkMeta{chunkRef1, chunkRef1, chunkRef2}, - }, - } - - job := NewJob(userID, table, indexPath, seriesMetas) - - mbt := mockBloomTokenizer{} - mcc := mockChunkClient{} - pbb := mockPersistentBlockBuilder{} - - // Run Compaction - compactedBlock, err := compactNewChunks(context.Background(), logger, job, &mbt, &mcc, &pbb, mockLimits{fpRate: fpRate}) - - // Validate Compaction Succeeds - require.NoError(t, err) - require.NotNil(t, compactedBlock) - - // Validate Compacted Block has expected data - require.Equal(t, job.tenantID, compactedBlock.TenantID) - require.Equal(t, job.tableName, compactedBlock.TableName) - require.Equal(t, uint64(fp1), compactedBlock.MinFingerprint) - require.Equal(t, uint64(fp2), compactedBlock.MaxFingerprint) - require.Equal(t, model.Time(chunkRef1.MinTime), compactedBlock.StartTimestamp) - require.Equal(t, model.Time(chunkRef2.MaxTime), compactedBlock.EndTimestamp) - require.Equal(t, indexPath, compactedBlock.IndexPath) -} - -func TestLazyBloomBuilder(t *testing.T) { - logger := log.NewNopLogger() - - label := labels.FromStrings("foo", "bar") - fp1 := model.Fingerprint(100) - fp2 := model.Fingerprint(999) - fp3 := model.Fingerprint(200) - - chunkRef1 := index.ChunkMeta{ - Checksum: 1, - MinTime: 1, - MaxTime: 99, - } - - chunkRef2 := index.ChunkMeta{ - Checksum: 2, - MinTime: 10, - MaxTime: 999, - } - - seriesMetas := []seriesMeta{ - { - seriesFP: fp1, - seriesLbs: label, - chunkRefs: []index.ChunkMeta{chunkRef1}, - }, - { - seriesFP: fp2, - seriesLbs: label, - chunkRefs: []index.ChunkMeta{chunkRef1, chunkRef2}, - }, - { - seriesFP: fp3, - seriesLbs: label, - chunkRefs: []index.ChunkMeta{chunkRef1, chunkRef1, chunkRef2}, - }, - } - - job := NewJob(userID, table, indexPath, seriesMetas) - - mbt := &mockBloomTokenizer{} - mcc := &mockChunkClient{} - - it := newLazyBloomBuilder(context.Background(), job, mcc, mbt, logger, mockLimits{chunksDownloadingBatchSize: 10, fpRate: fpRate}) - - // first seriesMeta has 1 chunks - require.True(t, it.Next()) - require.Equal(t, 1, mcc.requestCount) - require.Equal(t, 1, mcc.chunkCount) - require.Equal(t, fp1, it.At().Series.Fingerprint) - - // first seriesMeta has 2 chunks - require.True(t, it.Next()) - require.Equal(t, 2, mcc.requestCount) - require.Equal(t, 3, mcc.chunkCount) - require.Equal(t, fp2, it.At().Series.Fingerprint) - - // first seriesMeta has 3 chunks - require.True(t, it.Next()) - require.Equal(t, 3, mcc.requestCount) - require.Equal(t, 6, mcc.chunkCount) - require.Equal(t, fp3, it.At().Series.Fingerprint) - - // iterator is done - require.False(t, it.Next()) - require.Error(t, io.EOF, it.Err()) - require.Equal(t, v1.SeriesWithBloom{}, it.At()) -} - -type mockBloomTokenizer struct { - chunks []chunk.Chunk -} - -func (mbt *mockBloomTokenizer) PopulateSeriesWithBloom(_ *v1.SeriesWithBloom, c v1.Iterator[[]chunk.Chunk]) error { - for c.Next() { - mbt.chunks = append(mbt.chunks, c.At()...) - } - return nil -} - -type mockChunkClient struct { - requestCount int - chunkCount int -} - -func (mcc *mockChunkClient) GetChunks(_ context.Context, chks []chunk.Chunk) ([]chunk.Chunk, error) { - mcc.requestCount++ - mcc.chunkCount += len(chks) - return nil, nil -} - -type mockPersistentBlockBuilder struct { -} - -func (pbb *mockPersistentBlockBuilder) BuildFrom(_ v1.Iterator[v1.SeriesWithBloom]) (uint32, error) { - return 0, nil -} - -func (pbb *mockPersistentBlockBuilder) Data() (io.ReadSeekCloser, error) { - return nil, nil -} diff --git a/pkg/bloomcompactor/chunksbatchesiterator.go b/pkg/bloomcompactor/chunksbatchesiterator.go deleted file mode 100644 index a4494b02b7e4..000000000000 --- a/pkg/bloomcompactor/chunksbatchesiterator.go +++ /dev/null @@ -1,48 +0,0 @@ -package bloomcompactor - -import ( - "context" - "errors" - - "github.com/grafana/loki/pkg/storage/chunk" -) - -type chunksBatchesIterator struct { - context context.Context - client chunkClient - chunksToDownload []chunk.Chunk - batchSize int - - currentBatch []chunk.Chunk - err error -} - -func newChunkBatchesIterator(context context.Context, client chunkClient, chunksToDownload []chunk.Chunk, batchSize int) (*chunksBatchesIterator, error) { - if batchSize <= 0 { - return nil, errors.New("batchSize must be greater than 0") - } - return &chunksBatchesIterator{context: context, client: client, chunksToDownload: chunksToDownload, batchSize: batchSize}, nil -} - -func (c *chunksBatchesIterator) Next() bool { - if len(c.chunksToDownload) == 0 { - return false - } - batchSize := c.batchSize - chunksToDownloadCount := len(c.chunksToDownload) - if chunksToDownloadCount < batchSize { - batchSize = chunksToDownloadCount - } - chunksToDownload := c.chunksToDownload[:batchSize] - c.chunksToDownload = c.chunksToDownload[batchSize:] - c.currentBatch, c.err = c.client.GetChunks(c.context, chunksToDownload) - return c.err == nil -} - -func (c *chunksBatchesIterator) Err() error { - return c.err -} - -func (c *chunksBatchesIterator) At() []chunk.Chunk { - return c.currentBatch -} diff --git a/pkg/bloomcompactor/chunksbatchesiterator_test.go b/pkg/bloomcompactor/chunksbatchesiterator_test.go deleted file mode 100644 index 170f2662b508..000000000000 --- a/pkg/bloomcompactor/chunksbatchesiterator_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package bloomcompactor - -import ( - "context" - "errors" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/grafana/loki/pkg/storage/chunk" - tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" -) - -func Test_chunksBatchesIterator(t *testing.T) { - tests := map[string]struct { - batchSize int - chunksToDownload []chunk.Chunk - constructorError error - - hadNextCount int - }{ - "expected error if batch size is set to 0": { - batchSize: 0, - constructorError: errors.New("batchSize must be greater than 0"), - }, - "expected no error if there are no chunks": { - hadNextCount: 0, - batchSize: 10, - }, - "expected 1 call to the client": { - chunksToDownload: createFakeChunks(10), - hadNextCount: 1, - batchSize: 20, - }, - "expected 1 call to the client(2)": { - chunksToDownload: createFakeChunks(10), - hadNextCount: 1, - batchSize: 10, - }, - "expected 2 calls to the client": { - chunksToDownload: createFakeChunks(10), - hadNextCount: 2, - batchSize: 6, - }, - "expected 10 calls to the client": { - chunksToDownload: createFakeChunks(10), - hadNextCount: 10, - batchSize: 1, - }, - } - for name, data := range tests { - t.Run(name, func(t *testing.T) { - client := &fakeClient{} - iterator, err := newChunkBatchesIterator(context.Background(), client, data.chunksToDownload, data.batchSize) - if data.constructorError != nil { - require.Equal(t, err, data.constructorError) - return - } - hadNextCount := 0 - var downloadedChunks []chunk.Chunk - for iterator.Next() { - hadNextCount++ - downloaded := iterator.At() - downloadedChunks = append(downloadedChunks, downloaded...) - require.LessOrEqual(t, len(downloaded), data.batchSize) - } - require.NoError(t, iterator.Err()) - require.Equal(t, data.chunksToDownload, downloadedChunks) - require.Equal(t, data.hadNextCount, client.callsCount) - require.Equal(t, data.hadNextCount, hadNextCount) - }) - } -} - -func createFakeChunks(count int) []chunk.Chunk { - metas := make([]tsdbindex.ChunkMeta, 0, count) - for i := 0; i < count; i++ { - metas = append(metas, tsdbindex.ChunkMeta{ - Checksum: uint32(i), - MinTime: int64(i), - MaxTime: int64(i + 100), - KB: uint32(i * 100), - Entries: uint32(i * 10), - }) - } - return makeChunkRefs(metas, "fake", 0xFFFF) -} - -type fakeClient struct { - callsCount int -} - -func (f *fakeClient) GetChunks(_ context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error) { - f.callsCount++ - return chunks, nil -} diff --git a/pkg/bloomcompactor/config.go b/pkg/bloomcompactor/config.go index 884034fdd043..15f9aa86c040 100644 --- a/pkg/bloomcompactor/config.go +++ b/pkg/bloomcompactor/config.go @@ -2,6 +2,7 @@ package bloomcompactor import ( "flag" + "fmt" "time" "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/downloads" @@ -15,13 +16,14 @@ type Config struct { // section and the ingester configuration by default). Ring ring.RingConfig `yaml:"ring,omitempty" doc:"description=Defines the ring to be used by the bloom-compactor servers. In case this isn't configured, this block supports inheriting configuration from the common ring section."` // Enabled configures whether bloom-compactors should be used to compact index values into bloomfilters - Enabled bool `yaml:"enabled"` - WorkingDirectory string `yaml:"working_directory"` - CompactionInterval time.Duration `yaml:"compaction_interval"` - - RetryMinBackoff time.Duration `yaml:"compaction_retries_min_backoff"` - RetryMaxBackoff time.Duration `yaml:"compaction_retries_max_backoff"` - CompactionRetries int `yaml:"compaction_retries"` + Enabled bool `yaml:"enabled"` + CompactionInterval time.Duration `yaml:"compaction_interval"` + MinTableCompactionPeriod int `yaml:"min_table_compaction_period"` + MaxTableCompactionPeriod int `yaml:"max_table_compaction_period"` + WorkerParallelism int `yaml:"worker_parallelism"` + RetryMinBackoff time.Duration `yaml:"compaction_retries_min_backoff"` + RetryMaxBackoff time.Duration `yaml:"compaction_retries_max_backoff"` + CompactionRetries int `yaml:"compaction_retries"` MaxCompactionParallelism int `yaml:"max_compaction_parallelism"` } @@ -30,14 +32,29 @@ type Config struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.Ring.RegisterFlagsWithPrefix("bloom-compactor.", "collectors/", f) f.BoolVar(&cfg.Enabled, "bloom-compactor.enabled", false, "Flag to enable or disable the usage of the bloom-compactor component.") - f.StringVar(&cfg.WorkingDirectory, "bloom-compactor.working-directory", "", "Directory where files can be downloaded for compaction.") f.DurationVar(&cfg.CompactionInterval, "bloom-compactor.compaction-interval", 10*time.Minute, "Interval at which to re-run the compaction operation.") + f.IntVar(&cfg.WorkerParallelism, "bloom-compactor.worker-parallelism", 1, "Number of workers to run in parallel for compaction.") + f.IntVar(&cfg.MinTableCompactionPeriod, "bloom-compactor.min-table-compaction-period", 1, "How many index periods (days) to wait before compacting a table. This can be used to lower cost by not re-writing data to object storage too frequently since recent data changes more often.") + // TODO(owen-d): ideally we'd set this per tenant based on their `reject_old_samples_max_age` setting, + // but due to how we need to discover tenants, we can't do that yet. Tenant+Period discovery is done by + // iterating the table periods in object storage and looking for tenants within that period. + // In order to have this done dynamically, we'd need to account for tenant specific overrides, which are also + // dynamically reloaded. + // I'm doing it the simple way for now. + f.IntVar(&cfg.MaxTableCompactionPeriod, "bloom-compactor.max-table-compaction-period", 7, "How many index periods (days) to wait before compacting a table. This can be used to lower cost by not trying to compact older data which doesn't change. This can be optimized by aligning it with the maximum `reject_old_samples_max_age` setting of any tenant.") f.DurationVar(&cfg.RetryMinBackoff, "bloom-compactor.compaction-retries-min-backoff", 10*time.Second, "Minimum backoff time between retries.") f.DurationVar(&cfg.RetryMaxBackoff, "bloom-compactor.compaction-retries-max-backoff", time.Minute, "Maximum backoff time between retries.") f.IntVar(&cfg.CompactionRetries, "bloom-compactor.compaction-retries", 3, "Number of retries to perform when compaction fails.") f.IntVar(&cfg.MaxCompactionParallelism, "bloom-compactor.max-compaction-parallelism", 1, "Maximum number of tables to compact in parallel. While increasing this value, please make sure compactor has enough disk space allocated to be able to store and compact as many tables.") } +func (cfg *Config) Validate() error { + if cfg.MinTableCompactionPeriod > cfg.MaxTableCompactionPeriod { + return fmt.Errorf("min_compaction_age must be less than or equal to max_compaction_age") + } + return nil +} + type Limits interface { downloads.Limits BloomCompactorShardSize(tenantID string) int @@ -47,4 +64,5 @@ type Limits interface { BloomNGramLength(tenantID string) int BloomNGramSkip(tenantID string) int BloomFalsePositiveRate(tenantID string) float64 + BloomCompactorMaxBlockSize(tenantID string) int } diff --git a/pkg/bloomcompactor/controller.go b/pkg/bloomcompactor/controller.go new file mode 100644 index 000000000000..2d0f84a7a405 --- /dev/null +++ b/pkg/bloomcompactor/controller.go @@ -0,0 +1,767 @@ +package bloomcompactor + +import ( + "bytes" + "context" + "fmt" + "sort" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/config" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" +) + +type SimpleBloomController struct { + tsdbStore TSDBStore + bloomStore bloomshipper.Store + chunkLoader ChunkLoader + metrics *Metrics + limits Limits + + logger log.Logger +} + +func NewSimpleBloomController( + tsdbStore TSDBStore, + blockStore bloomshipper.Store, + chunkLoader ChunkLoader, + limits Limits, + metrics *Metrics, + logger log.Logger, +) *SimpleBloomController { + return &SimpleBloomController{ + tsdbStore: tsdbStore, + bloomStore: blockStore, + chunkLoader: chunkLoader, + metrics: metrics, + limits: limits, + logger: logger, + } +} + +// TODO(owen-d): pool, evaluate if memory-only is the best choice +func (s *SimpleBloomController) rwFn() (v1.BlockWriter, v1.BlockReader) { + indexBuf := bytes.NewBuffer(nil) + bloomsBuf := bytes.NewBuffer(nil) + return v1.NewMemoryBlockWriter(indexBuf, bloomsBuf), v1.NewByteReader(indexBuf, bloomsBuf) +} + +/* +Compaction works as follows, split across many functions for clarity: + 1. Fetch all meta.jsons for the given tenant and table which overlap the ownership range of this compactor. + 2. Load current TSDBs for this tenant/table. + 3. For each live TSDB (there should be only 1, but this works with multiple), find any gaps + (fingerprint ranges) which are not up date, determined by checking other meta.jsons and comparing + the tsdbs they were generated from + their ownership ranges. + 4. Build new bloom blocks for each gap, using the series and chunks from the TSDBs and any existing + blocks which overlap the gaps to accelerate bloom generation. + 5. Write the new blocks and metas to the store. + 6. Determine if any meta.jsons overlap the ownership range but are outdated, and remove them and + their associated blocks if so. +*/ +func (s *SimpleBloomController) compactTenant( + ctx context.Context, + table config.DayTable, + tenant string, + ownershipRange v1.FingerprintBounds, +) error { + logger := log.With(s.logger, "org_id", tenant, "table", table.Addr(), "ownership", ownershipRange.String()) + + client, err := s.bloomStore.Client(table.ModelTime()) + if err != nil { + level.Error(logger).Log("msg", "failed to get client", "err", err) + return errors.Wrap(err, "failed to get client") + } + + // Fetch source metas to be used in both compaction and cleanup of out-of-date metas+blooms + metas, err := s.bloomStore.FetchMetas( + ctx, + bloomshipper.MetaSearchParams{ + TenantID: tenant, + Interval: bloomshipper.NewInterval(table.Bounds()), + Keyspace: ownershipRange, + }, + ) + if err != nil { + level.Error(logger).Log("msg", "failed to get metas", "err", err) + return errors.Wrap(err, "failed to get metas") + } + + level.Debug(logger).Log("msg", "found relevant metas", "metas", len(metas)) + + // fetch all metas overlapping our ownership range so we can safely + // check which metas can be deleted even if they only partially overlap out ownership range + superset, err := s.fetchSuperSet(ctx, tenant, table, ownershipRange, metas, logger) + if err != nil { + return errors.Wrap(err, "failed to fetch superset") + } + + // build compaction plans + work, err := s.findOutdatedGaps(ctx, tenant, table, ownershipRange, metas, logger) + if err != nil { + return errors.Wrap(err, "failed to find outdated gaps") + } + + // build new blocks + built, err := s.buildGaps(ctx, tenant, table, client, work, logger) + if err != nil { + return errors.Wrap(err, "failed to build gaps") + } + + // combine built and superset metas + // in preparation for removing outdated ones + combined := append(superset, built...) + + outdated := outdatedMetas(combined) + level.Debug(logger).Log("msg", "found outdated metas", "outdated", len(outdated)) + + var ( + deletedMetas int + deletedBlocks int + ) + defer func() { + s.metrics.metasDeleted.Add(float64(deletedMetas)) + s.metrics.blocksDeleted.Add(float64(deletedBlocks)) + }() + + for _, meta := range outdated { + for _, block := range meta.Blocks { + err := client.DeleteBlocks(ctx, []bloomshipper.BlockRef{block}) + if err != nil { + if client.IsObjectNotFoundErr(err) { + level.Debug(logger).Log("msg", "block not found while attempting delete, continuing", "block", block.String()) + } else { + level.Error(logger).Log("msg", "failed to delete block", "err", err, "block", block.String()) + return errors.Wrap(err, "failed to delete block") + } + } + deletedBlocks++ + level.Debug(logger).Log("msg", "removed outdated block", "block", block.String()) + } + + err = client.DeleteMetas(ctx, []bloomshipper.MetaRef{meta.MetaRef}) + if err != nil { + if client.IsObjectNotFoundErr(err) { + level.Debug(logger).Log("msg", "meta not found while attempting delete, continuing", "meta", meta.MetaRef.String()) + } else { + level.Error(logger).Log("msg", "failed to delete meta", "err", err, "meta", meta.MetaRef.String()) + return errors.Wrap(err, "failed to delete meta") + } + } + deletedMetas++ + level.Debug(logger).Log("msg", "removed outdated meta", "meta", meta.MetaRef.String()) + } + + level.Debug(logger).Log("msg", "finished compaction") + return nil +} + +// fetchSuperSet fetches all metas which overlap the ownership range of the first set of metas we've resolved +func (s *SimpleBloomController) fetchSuperSet( + ctx context.Context, + tenant string, + table config.DayTable, + ownershipRange v1.FingerprintBounds, + metas []bloomshipper.Meta, + logger log.Logger, +) ([]bloomshipper.Meta, error) { + // in order to delete outdates metas which only partially fall within the ownership range, + // we need to fetcha all metas in the entire bound range of the first set of metas we've resolved + /* + For instance, we have the following ownership range and we resolve `meta1` in our first Fetch call + because it overlaps the ownership range, we'll need to fetch newer metas that may overlap it in order + to check if it safely can be deleted. This falls partially outside our specific ownership range, but + we can safely run multiple deletes by treating their removal as idempotent. + |-------------ownership range-----------------| + |-------meta1-------| + + we fetch this before possibly deleting meta1 |------| + */ + superset := ownershipRange + for _, meta := range metas { + union := superset.Union(meta.Bounds) + if len(union) > 1 { + level.Error(logger).Log("msg", "meta bounds union is not a single range", "union", union) + return nil, errors.New("meta bounds union is not a single range") + } + superset = union[0] + } + + within := superset.Within(ownershipRange) + level.Debug(logger).Log( + "msg", "looking for superset metas", + "superset", superset.String(), + "superset_within", within, + ) + + if within { + // we don't need to fetch any more metas + // NB(owen-d): here we copy metas into the output. This is slightly inefficient, but + // helps prevent mutability bugs by returning the same slice as the input. + results := make([]bloomshipper.Meta, len(metas)) + copy(results, metas) + return results, nil + } + + supersetMetas, err := s.bloomStore.FetchMetas( + ctx, + bloomshipper.MetaSearchParams{ + TenantID: tenant, + Interval: bloomshipper.NewInterval(table.Bounds()), + Keyspace: superset, + }, + ) + + if err != nil { + level.Error(logger).Log("msg", "failed to get meta superset range", "err", err, "superset", superset) + return nil, errors.Wrap(err, "failed to get meta supseret range") + } + + level.Debug(logger).Log( + "msg", "found superset metas", + "metas", len(metas), + "fresh_metas", len(supersetMetas), + "delta", len(supersetMetas)-len(metas), + ) + + return supersetMetas, nil +} + +func (s *SimpleBloomController) findOutdatedGaps( + ctx context.Context, + tenant string, + table config.DayTable, + ownershipRange v1.FingerprintBounds, + metas []bloomshipper.Meta, + logger log.Logger, +) ([]blockPlan, error) { + // Resolve TSDBs + tsdbs, err := s.tsdbStore.ResolveTSDBs(ctx, table, tenant) + if err != nil { + level.Error(logger).Log("msg", "failed to resolve tsdbs", "err", err) + return nil, errors.Wrap(err, "failed to resolve tsdbs") + } + + if len(tsdbs) == 0 { + return nil, nil + } + + // Determine which TSDBs have gaps in the ownership range and need to + // be processed. + tsdbsWithGaps, err := gapsBetweenTSDBsAndMetas(ownershipRange, tsdbs, metas) + if err != nil { + level.Error(logger).Log("msg", "failed to find gaps", "err", err) + return nil, errors.Wrap(err, "failed to find gaps") + } + + if len(tsdbsWithGaps) == 0 { + level.Debug(logger).Log("msg", "blooms exist for all tsdbs") + return nil, nil + } + + work, err := blockPlansForGaps(tsdbsWithGaps, metas) + if err != nil { + level.Error(logger).Log("msg", "failed to create plan", "err", err) + return nil, errors.Wrap(err, "failed to create plan") + } + + return work, nil +} + +func (s *SimpleBloomController) loadWorkForGap( + ctx context.Context, + table config.DayTable, + tenant string, + id tsdb.Identifier, + gap gapWithBlocks, +) (v1.CloseableIterator[*v1.Series], v1.CloseableResettableIterator[*v1.SeriesWithBloom], error) { + // load a series iterator for the gap + seriesItr, err := s.tsdbStore.LoadTSDB(ctx, table, tenant, id, gap.bounds) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to load tsdb") + } + + // load a blocks iterator for the gap + fetcher, err := s.bloomStore.Fetcher(table.ModelTime()) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to get fetcher") + } + + f := FetchFunc[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier](fetcher.FetchBlocks) + blocksIter := newBlockLoadingIter(ctx, gap.blocks, f, 10) + + return seriesItr, blocksIter, nil +} + +func (s *SimpleBloomController) buildGaps( + ctx context.Context, + tenant string, + table config.DayTable, + client bloomshipper.Client, + work []blockPlan, + logger log.Logger, +) ([]bloomshipper.Meta, error) { + // Generate Blooms + // Now that we have the gaps, we will generate a bloom block for each gap. + // We can accelerate this by using existing blocks which may already contain + // needed chunks in their blooms, for instance after a new TSDB version is generated + // but contains many of the same chunk references from the previous version. + // To do this, we'll need to take the metas we've already resolved and find blocks + // overlapping the ownership ranges we've identified as needing updates. + // With these in hand, we can download the old blocks and use them to + // accelerate bloom generation for the new blocks. + + var ( + blockCt int + tsdbCt = len(work) + nGramSize = uint64(s.limits.BloomNGramLength(tenant)) + nGramSkip = uint64(s.limits.BloomNGramSkip(tenant)) + maxBlockSize = uint64(s.limits.BloomCompactorMaxBlockSize(tenant)) + blockOpts = v1.NewBlockOptions(nGramSize, nGramSkip, maxBlockSize) + created []bloomshipper.Meta + totalSeries uint64 + ) + + for _, plan := range work { + + for i := range plan.gaps { + gap := plan.gaps[i] + logger := log.With(logger, "gap", gap.bounds.String(), "tsdb", plan.tsdb.Name()) + + meta := bloomshipper.Meta{ + MetaRef: bloomshipper.MetaRef{ + Ref: bloomshipper.Ref{ + TenantID: tenant, + TableName: table.Addr(), + Bounds: gap.bounds, + }, + }, + Sources: []tsdb.SingleTenantTSDBIdentifier{plan.tsdb}, + } + + // Fetch blocks that aren't up to date but are in the desired fingerprint range + // to try and accelerate bloom creation + level.Debug(logger).Log("msg", "loading series and blocks for gap", "blocks", len(gap.blocks)) + seriesItr, blocksIter, err := s.loadWorkForGap(ctx, table, tenant, plan.tsdb, gap) + if err != nil { + level.Error(logger).Log("msg", "failed to get series and blocks", "err", err) + return nil, errors.Wrap(err, "failed to get series and blocks") + } + + // Blocks are built consuming the series iterator. For observability, we wrap the series iterator + // with a counter iterator to count the number of times Next() is called on it. + // This is used to observe the number of series that are being processed. + seriesItrWithCounter := v1.NewCounterIter[*v1.Series](seriesItr) + + gen := NewSimpleBloomGenerator( + tenant, + blockOpts, + seriesItrWithCounter, + s.chunkLoader, + blocksIter, + s.rwFn, + s.metrics, + logger, + ) + + level.Debug(logger).Log("msg", "generating blocks", "overlapping_blocks", len(gap.blocks)) + + newBlocks := gen.Generate(ctx) + if err != nil { + level.Error(logger).Log("msg", "failed to generate bloom", "err", err) + blocksIter.Close() + return nil, errors.Wrap(err, "failed to generate bloom") + } + + for newBlocks.Next() && newBlocks.Err() == nil { + blockCt++ + blk := newBlocks.At() + + built, err := bloomshipper.BlockFrom(tenant, table.Addr(), blk) + if err != nil { + level.Error(logger).Log("msg", "failed to build block", "err", err) + blocksIter.Close() + return nil, errors.Wrap(err, "failed to build block") + } + + if err := client.PutBlock( + ctx, + built, + ); err != nil { + level.Error(logger).Log("msg", "failed to write block", "err", err) + blocksIter.Close() + return nil, errors.Wrap(err, "failed to write block") + } + s.metrics.blocksCreated.Inc() + + totalGapKeyspace := (gap.bounds.Max - gap.bounds.Min) + progress := (built.Bounds.Max - gap.bounds.Min) + pct := float64(progress) / float64(totalGapKeyspace) * 100 + level.Debug(logger).Log( + "msg", "uploaded block", + "block", built.BlockRef.String(), + "progress_pct", fmt.Sprintf("%.2f", pct), + ) + + meta.Blocks = append(meta.Blocks, built.BlockRef) + } + + if err := newBlocks.Err(); err != nil { + level.Error(logger).Log("msg", "failed to generate bloom", "err", err) + return nil, errors.Wrap(err, "failed to generate bloom") + } + + // Close pre-existing blocks + blocksIter.Close() + + // Write the new meta + // TODO(owen-d): put total size in log, total time in metrics+log + ref, err := bloomshipper.MetaRefFrom(tenant, table.Addr(), gap.bounds, meta.Sources, meta.Blocks) + if err != nil { + level.Error(logger).Log("msg", "failed to checksum meta", "err", err) + return nil, errors.Wrap(err, "failed to checksum meta") + } + meta.MetaRef = ref + + if err := client.PutMeta(ctx, meta); err != nil { + level.Error(logger).Log("msg", "failed to write meta", "err", err) + return nil, errors.Wrap(err, "failed to write meta") + } + s.metrics.metasCreated.Inc() + level.Debug(logger).Log("msg", "uploaded meta", "meta", meta.MetaRef.String()) + + created = append(created, meta) + totalSeries += uint64(seriesItrWithCounter.Count()) + + s.metrics.blocksReused.Add(float64(len(gap.blocks))) + } + } + + s.metrics.tenantsSeries.Observe(float64(totalSeries)) + level.Debug(logger).Log("msg", "finished bloom generation", "blocks", blockCt, "tsdbs", tsdbCt) + return created, nil +} + +// outdatedMetas returns metas that are outdated and need to be removed, +// determined by if their entire ownership range is covered by other metas with newer +// TSDBs +func outdatedMetas(metas []bloomshipper.Meta) (outdated []bloomshipper.Meta) { + // first, ensure data is sorted so we can take advantage of that + sort.Slice(metas, func(i, j int) bool { + return metas[i].Bounds.Less(metas[j].Bounds) + }) + + // NB(owen-d): time complexity shouldn't be a problem + // given the number of metas should be low (famous last words, i know). + for i := range metas { + a := metas[i] + + var overlaps []v1.FingerprintBounds + + for j := range metas { + if j == i { + continue + } + + b := metas[j] + intersection := a.Bounds.Intersection(b.Bounds) + if intersection == nil { + if a.Bounds.Cmp(b.Bounds.Min) == v1.After { + // All subsequent metas will be newer, so we can break + break + } + // otherwise, just check the next meta + continue + } + + // we can only remove older data, not data which may be newer + if !tsdbsStrictlyNewer(b.Sources, a.Sources) { + continue + } + + // because we've sorted the metas, we only have to test overlaps against the last + // overlap we found (if any) + if len(overlaps) == 0 { + overlaps = append(overlaps, *intersection) + continue + } + + // best effort at merging overlaps first pass + last := overlaps[len(overlaps)-1] + overlaps = append(overlaps[:len(overlaps)-1], last.Union(*intersection)...) + + } + + if coversFullRange(a.Bounds, overlaps) { + outdated = append(outdated, a) + } + } + return +} + +func coversFullRange(bounds v1.FingerprintBounds, overlaps []v1.FingerprintBounds) bool { + // if there are no overlaps, the range is not covered + if len(overlaps) == 0 { + return false + } + + // keep track of bounds which need to be filled in order + // for the overlaps to cover the full range + missing := []v1.FingerprintBounds{bounds} + ignores := make(map[int]bool) + for _, overlap := range overlaps { + var i int + for { + if i >= len(missing) { + break + } + + if ignores[i] { + i++ + continue + } + + remaining := missing[i].Unless(overlap) + switch len(remaining) { + case 0: + // this range is covered, ignore it + ignores[i] = true + case 1: + // this range is partially covered, updated it + missing[i] = remaining[0] + case 2: + // this range has been partially covered in the middle, + // split it into two ranges and append + ignores[i] = true + missing = append(missing, remaining...) + } + i++ + } + + } + + return len(ignores) == len(missing) +} + +// tsdbStrictlyNewer returns if all of the tsdbs in a are newer than all of the tsdbs in b +func tsdbsStrictlyNewer(as, bs []tsdb.SingleTenantTSDBIdentifier) bool { + for _, a := range as { + for _, b := range bs { + if a.TS.Before(b.TS) { + return false + } + } + } + return true +} + +type gapWithBlocks struct { + bounds v1.FingerprintBounds + blocks []bloomshipper.BlockRef +} + +// blockPlan is a plan for all the work needed to build a meta.json +// It includes: +// - the tsdb (source of truth) which contains all the series+chunks +// we need to ensure are indexed in bloom blocks +// - a list of gaps that are out of date and need to be checked+built +// - within each gap, a list of block refs which overlap the gap are included +// so we can use them to accelerate bloom generation. They likely contain many +// of the same chunks we need to ensure are indexed, just from previous tsdb iterations. +// This is a performance optimization to avoid expensive re-reindexing +type blockPlan struct { + tsdb tsdb.SingleTenantTSDBIdentifier + gaps []gapWithBlocks +} + +// blockPlansForGaps groups tsdb gaps we wish to fill with overlapping but out of date blocks. +// This allows us to expedite bloom generation by using existing blocks to fill in the gaps +// since many will contain the same chunks. +func blockPlansForGaps(tsdbs []tsdbGaps, metas []bloomshipper.Meta) ([]blockPlan, error) { + plans := make([]blockPlan, 0, len(tsdbs)) + + for _, idx := range tsdbs { + plan := blockPlan{ + tsdb: idx.tsdb, + gaps: make([]gapWithBlocks, 0, len(idx.gaps)), + } + + for _, gap := range idx.gaps { + planGap := gapWithBlocks{ + bounds: gap, + } + + for _, meta := range metas { + + if meta.Bounds.Intersection(gap) == nil { + // this meta doesn't overlap the gap, skip + continue + } + + for _, block := range meta.Blocks { + if block.Bounds.Intersection(gap) == nil { + // this block doesn't overlap the gap, skip + continue + } + // this block overlaps the gap, add it to the plan + // for this gap + planGap.blocks = append(planGap.blocks, block) + } + } + + // ensure we sort blocks so deduping iterator works as expected + sort.Slice(planGap.blocks, func(i, j int) bool { + return planGap.blocks[i].Bounds.Less(planGap.blocks[j].Bounds) + }) + + peekingBlocks := v1.NewPeekingIter[bloomshipper.BlockRef]( + v1.NewSliceIter[bloomshipper.BlockRef]( + planGap.blocks, + ), + ) + // dedupe blocks which could be in multiple metas + itr := v1.NewDedupingIter[bloomshipper.BlockRef, bloomshipper.BlockRef]( + func(a, b bloomshipper.BlockRef) bool { + return a == b + }, + v1.Identity[bloomshipper.BlockRef], + func(a, _ bloomshipper.BlockRef) bloomshipper.BlockRef { + return a + }, + peekingBlocks, + ) + + deduped, err := v1.Collect[bloomshipper.BlockRef](itr) + if err != nil { + return nil, errors.Wrap(err, "failed to dedupe blocks") + } + planGap.blocks = deduped + + plan.gaps = append(plan.gaps, planGap) + } + + plans = append(plans, plan) + } + + return plans, nil +} + +// Used to signal the gaps that need to be populated for a tsdb +type tsdbGaps struct { + tsdb tsdb.SingleTenantTSDBIdentifier + gaps []v1.FingerprintBounds +} + +// tsdbsUpToDate returns if the metas are up to date with the tsdbs. This is determined by asserting +// that for each TSDB, there are metas covering the entire ownership range which were generated from that specific TSDB. +func gapsBetweenTSDBsAndMetas( + ownershipRange v1.FingerprintBounds, + tsdbs []tsdb.SingleTenantTSDBIdentifier, + metas []bloomshipper.Meta, +) (res []tsdbGaps, err error) { + for _, db := range tsdbs { + id := db.Name() + + relevantMetas := make([]v1.FingerprintBounds, 0, len(metas)) + for _, meta := range metas { + for _, s := range meta.Sources { + if s.Name() == id { + relevantMetas = append(relevantMetas, meta.Bounds) + } + } + } + + gaps, err := findGaps(ownershipRange, relevantMetas) + if err != nil { + return nil, err + } + + if len(gaps) > 0 { + res = append(res, tsdbGaps{ + tsdb: db, + gaps: gaps, + }) + } + } + + return res, err +} + +func findGaps(ownershipRange v1.FingerprintBounds, metas []v1.FingerprintBounds) (gaps []v1.FingerprintBounds, err error) { + if len(metas) == 0 { + return []v1.FingerprintBounds{ownershipRange}, nil + } + + // turn the available metas into a list of non-overlapping metas + // for easier processing + var nonOverlapping []v1.FingerprintBounds + // First, we reduce the metas into a smaller set by combining overlaps. They must be sorted. + var cur *v1.FingerprintBounds + for i := 0; i < len(metas); i++ { + j := i + 1 + + // first iteration (i == 0), set the current meta + if cur == nil { + cur = &metas[i] + } + + if j >= len(metas) { + // We've reached the end of the list. Add the last meta to the non-overlapping set. + nonOverlapping = append(nonOverlapping, *cur) + break + } + + combined := cur.Union(metas[j]) + if len(combined) == 1 { + // There was an overlap between the two tested ranges. Combine them and keep going. + cur = &combined[0] + continue + } + + // There was no overlap between the two tested ranges. Add the first to the non-overlapping set. + // and keep the second for the next iteration. + nonOverlapping = append(nonOverlapping, combined[0]) + cur = &combined[1] + } + + // Now, detect gaps between the non-overlapping metas and the ownership range. + // The left bound of the ownership range will be adjusted as we go. + leftBound := ownershipRange.Min + for _, meta := range nonOverlapping { + + clippedMeta := meta.Intersection(ownershipRange) + // should never happen as long as we are only combining metas + // that intersect with the ownership range + if clippedMeta == nil { + return nil, fmt.Errorf("meta is not within ownership range: %v", meta) + } + + searchRange := ownershipRange.Slice(leftBound, clippedMeta.Max) + // update the left bound for the next iteration + leftBound = min(clippedMeta.Max+1, ownershipRange.Max+1) + + // since we've already ensured that the meta is within the ownership range, + // we know the xor will be of length zero (when the meta is equal to the ownership range) + // or 1 (when the meta is a subset of the ownership range) + xors := searchRange.Unless(*clippedMeta) + if len(xors) == 0 { + // meta is equal to the ownership range. This means the meta + // covers this entire section of the ownership range. + continue + } + + gaps = append(gaps, xors[0]) + } + + if leftBound <= ownershipRange.Max { + // There is a gap between the last meta and the end of the ownership range. + gaps = append(gaps, v1.NewBounds(leftBound, ownershipRange.Max)) + } + + return gaps, nil +} diff --git a/pkg/bloomcompactor/controller_test.go b/pkg/bloomcompactor/controller_test.go new file mode 100644 index 000000000000..72653c292b18 --- /dev/null +++ b/pkg/bloomcompactor/controller_test.go @@ -0,0 +1,564 @@ +package bloomcompactor + +import ( + "testing" + "time" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" +) + +func Test_findGaps(t *testing.T) { + for _, tc := range []struct { + desc string + err bool + exp []v1.FingerprintBounds + ownershipRange v1.FingerprintBounds + metas []v1.FingerprintBounds + }{ + { + desc: "error nonoverlapping metas", + err: true, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{v1.NewBounds(11, 20)}, + }, + { + desc: "one meta with entire ownership range", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{v1.NewBounds(0, 10)}, + }, + { + desc: "two non-overlapping metas with entire ownership range", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + v1.NewBounds(6, 10), + }, + }, + { + desc: "two overlapping metas with entire ownership range", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 6), + v1.NewBounds(4, 10), + }, + }, + { + desc: "one meta with partial ownership range", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(6, 10), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + }, + }, + { + desc: "smaller subsequent meta with partial ownership range", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(8, 10), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 7), + v1.NewBounds(3, 4), + }, + }, + { + desc: "hole in the middle", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(4, 5), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 3), + v1.NewBounds(6, 10), + }, + }, + { + desc: "holes on either end", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(0, 2), + v1.NewBounds(8, 10), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(3, 5), + v1.NewBounds(6, 7), + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + gaps, err := findGaps(tc.ownershipRange, tc.metas) + if tc.err { + require.Error(t, err) + return + } + require.Equal(t, tc.exp, gaps) + }) + } +} + +func tsdbID(n int) tsdb.SingleTenantTSDBIdentifier { + return tsdb.SingleTenantTSDBIdentifier{ + TS: time.Unix(int64(n), 0), + } +} + +func genMeta(min, max model.Fingerprint, sources []int, blocks []bloomshipper.BlockRef) bloomshipper.Meta { + m := bloomshipper.Meta{ + MetaRef: bloomshipper.MetaRef{ + Ref: bloomshipper.Ref{ + Bounds: v1.NewBounds(min, max), + }, + }, + Blocks: blocks, + } + for _, source := range sources { + m.Sources = append(m.Sources, tsdbID(source)) + } + return m +} + +func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { + + for _, tc := range []struct { + desc string + err bool + exp []tsdbGaps + ownershipRange v1.FingerprintBounds + tsdbs []tsdb.SingleTenantTSDBIdentifier + metas []bloomshipper.Meta + }{ + { + desc: "non-overlapping tsdbs and metas", + err: true, + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(11, 20, []int{0}, nil), + }, + }, + { + desc: "single tsdb", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(4, 8, []int{0}, nil), + }, + exp: []tsdbGaps{ + { + tsdb: tsdbID(0), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 3), + v1.NewBounds(9, 10), + }, + }, + }, + }, + { + desc: "multiple tsdbs with separate blocks", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, + metas: []bloomshipper.Meta{ + genMeta(0, 5, []int{0}, nil), + genMeta(6, 10, []int{1}, nil), + }, + exp: []tsdbGaps{ + { + tsdb: tsdbID(0), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(6, 10), + }, + }, + { + tsdb: tsdbID(1), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + }, + }, + }, + }, + { + desc: "multiple tsdbs with the same blocks", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, + metas: []bloomshipper.Meta{ + genMeta(0, 5, []int{0, 1}, nil), + genMeta(6, 8, []int{1}, nil), + }, + exp: []tsdbGaps{ + { + tsdb: tsdbID(0), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(6, 10), + }, + }, + { + tsdb: tsdbID(1), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(9, 10), + }, + }, + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + gaps, err := gapsBetweenTSDBsAndMetas(tc.ownershipRange, tc.tsdbs, tc.metas) + if tc.err { + require.Error(t, err) + return + } + require.Equal(t, tc.exp, gaps) + }) + } +} + +func genBlockRef(min, max model.Fingerprint) bloomshipper.BlockRef { + bounds := v1.NewBounds(min, max) + return bloomshipper.BlockRef{ + Ref: bloomshipper.Ref{ + Bounds: bounds, + }, + } +} + +func Test_blockPlansForGaps(t *testing.T) { + for _, tc := range []struct { + desc string + ownershipRange v1.FingerprintBounds + tsdbs []tsdb.SingleTenantTSDBIdentifier + metas []bloomshipper.Meta + err bool + exp []blockPlan + }{ + { + desc: "single overlapping meta+no overlapping block", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(5, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(11, 20)}), + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []gapWithBlocks{ + { + bounds: v1.NewBounds(0, 10), + }, + }, + }, + }, + }, + { + desc: "single overlapping meta+one overlapping block", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(5, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []gapWithBlocks{ + { + bounds: v1.NewBounds(0, 10), + blocks: []bloomshipper.BlockRef{genBlockRef(9, 20)}, + }, + }, + }, + }, + }, + { + // the range which needs to be generated doesn't overlap with existing blocks + // from other tsdb versions since theres an up to date tsdb version block, + // but we can trim the range needing generation + desc: "trims up to date area", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(9, 20, []int{0}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), // block for same tsdb + genMeta(9, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), // block for different tsdb + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []gapWithBlocks{ + { + bounds: v1.NewBounds(0, 8), + }, + }, + }, + }, + }, + { + desc: "uses old block for overlapping range", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(9, 20, []int{0}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), // block for same tsdb + genMeta(5, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(5, 20)}), // block for different tsdb + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []gapWithBlocks{ + { + bounds: v1.NewBounds(0, 8), + blocks: []bloomshipper.BlockRef{genBlockRef(5, 20)}, + }, + }, + }, + }, + }, + { + desc: "multi case", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, // generate for both tsdbs + metas: []bloomshipper.Meta{ + genMeta(0, 2, []int{0}, []bloomshipper.BlockRef{ + genBlockRef(0, 1), + genBlockRef(1, 2), + }), // tsdb_0 + genMeta(6, 8, []int{0}, []bloomshipper.BlockRef{genBlockRef(6, 8)}), // tsdb_0 + + genMeta(3, 5, []int{1}, []bloomshipper.BlockRef{genBlockRef(3, 5)}), // tsdb_1 + genMeta(8, 10, []int{1}, []bloomshipper.BlockRef{genBlockRef(8, 10)}), // tsdb_1 + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []gapWithBlocks{ + // tsdb (id=0) can source chunks from the blocks built from tsdb (id=1) + { + bounds: v1.NewBounds(3, 5), + blocks: []bloomshipper.BlockRef{genBlockRef(3, 5)}, + }, + { + bounds: v1.NewBounds(9, 10), + blocks: []bloomshipper.BlockRef{genBlockRef(8, 10)}, + }, + }, + }, + // tsdb (id=1) can source chunks from the blocks built from tsdb (id=0) + { + tsdb: tsdbID(1), + gaps: []gapWithBlocks{ + { + bounds: v1.NewBounds(0, 2), + blocks: []bloomshipper.BlockRef{ + genBlockRef(0, 1), + genBlockRef(1, 2), + }, + }, + { + bounds: v1.NewBounds(6, 7), + blocks: []bloomshipper.BlockRef{genBlockRef(6, 8)}, + }, + }, + }, + }, + }, + { + desc: "dedupes block refs", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(9, 20, []int{1}, []bloomshipper.BlockRef{ + genBlockRef(1, 4), + genBlockRef(9, 20), + }), // blocks for first diff tsdb + genMeta(5, 20, []int{2}, []bloomshipper.BlockRef{ + genBlockRef(5, 10), + genBlockRef(9, 20), // same block references in prior meta (will be deduped) + }), // block for second diff tsdb + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []gapWithBlocks{ + { + bounds: v1.NewBounds(0, 10), + blocks: []bloomshipper.BlockRef{ + genBlockRef(1, 4), + genBlockRef(5, 10), + genBlockRef(9, 20), + }, + }, + }, + }, + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + // we reuse the gapsBetweenTSDBsAndMetas function to generate the gaps as this function is tested + // separately and it's used to generate input in our regular code path (easier to write tests this way). + gaps, err := gapsBetweenTSDBsAndMetas(tc.ownershipRange, tc.tsdbs, tc.metas) + require.NoError(t, err) + + plans, err := blockPlansForGaps(gaps, tc.metas) + if tc.err { + require.Error(t, err) + return + } + require.Equal(t, tc.exp, plans) + + }) + } +} + +func Test_coversFullRange(t *testing.T) { + for _, tc := range []struct { + desc string + src v1.FingerprintBounds + overlaps []v1.FingerprintBounds + exp bool + }{ + { + desc: "empty", + src: v1.NewBounds(0, 10), + overlaps: []v1.FingerprintBounds{}, + exp: false, + }, + { + desc: "single_full_range", + src: v1.NewBounds(0, 10), + overlaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 10), + }, + exp: true, + }, + { + desc: "single_partial_range", + src: v1.NewBounds(0, 10), + overlaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + }, + exp: false, + }, + { + desc: "multiple_full_ranges", + src: v1.NewBounds(0, 10), + overlaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + v1.NewBounds(6, 10), + }, + exp: true, + }, + { + desc: "multiple_partial_ranges", + src: v1.NewBounds(0, 10), + overlaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + v1.NewBounds(7, 8), + }, + exp: false, + }, + { + desc: "wraps_partial_range", + src: v1.NewBounds(10, 20), + overlaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 12), + v1.NewBounds(13, 15), + v1.NewBounds(19, 21), + }, + exp: false, + }, + { + desc: "wraps_full_range", + src: v1.NewBounds(10, 20), + overlaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 12), + v1.NewBounds(13, 15), + v1.NewBounds(16, 25), + }, + exp: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + require.Equal(t, tc.exp, coversFullRange(tc.src, tc.overlaps)) + }) + } +} + +func Test_OutdatedMetas(t *testing.T) { + gen := func(bounds v1.FingerprintBounds, tsdbTimes ...model.Time) (meta bloomshipper.Meta) { + for _, tsdbTime := range tsdbTimes { + meta.Sources = append(meta.Sources, tsdb.SingleTenantTSDBIdentifier{TS: tsdbTime.Time()}) + } + meta.Bounds = bounds + return meta + } + + for _, tc := range []struct { + desc string + metas []bloomshipper.Meta + exp []bloomshipper.Meta + }{ + { + desc: "no metas", + metas: nil, + exp: nil, + }, + { + desc: "single meta", + metas: []bloomshipper.Meta{ + gen(v1.NewBounds(0, 10), 0), + }, + exp: nil, + }, + { + desc: "single outdated meta", + metas: []bloomshipper.Meta{ + gen(v1.NewBounds(0, 10), 0), + gen(v1.NewBounds(0, 10), 1), + }, + exp: []bloomshipper.Meta{ + gen(v1.NewBounds(0, 10), 0), + }, + }, + { + desc: "single outdated via partitions", + metas: []bloomshipper.Meta{ + gen(v1.NewBounds(0, 5), 0), + gen(v1.NewBounds(6, 10), 0), + gen(v1.NewBounds(0, 10), 1), + }, + exp: []bloomshipper.Meta{ + gen(v1.NewBounds(0, 5), 0), + gen(v1.NewBounds(6, 10), 0), + }, + }, + { + desc: "multi tsdbs", + metas: []bloomshipper.Meta{ + gen(v1.NewBounds(0, 5), 0, 1), + gen(v1.NewBounds(6, 10), 0, 1), + gen(v1.NewBounds(0, 10), 2, 3), + }, + exp: []bloomshipper.Meta{ + gen(v1.NewBounds(0, 5), 0, 1), + gen(v1.NewBounds(6, 10), 0, 1), + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + require.Equal(t, tc.exp, outdatedMetas(tc.metas)) + }) + } +} diff --git a/pkg/bloomcompactor/job.go b/pkg/bloomcompactor/job.go deleted file mode 100644 index bd43293c73cb..000000000000 --- a/pkg/bloomcompactor/job.go +++ /dev/null @@ -1,85 +0,0 @@ -package bloomcompactor - -import ( - "math" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" -) - -type seriesMeta struct { - seriesFP model.Fingerprint - seriesLbs labels.Labels - chunkRefs []index.ChunkMeta -} - -type Job struct { - tableName, tenantID, indexPath string - seriesMetas []seriesMeta - - // We compute them lazily. Unset value is 0. - from, through model.Time - minFp, maxFp model.Fingerprint -} - -// NewJob returns a new compaction Job. -func NewJob( - tenantID string, - tableName string, - indexPath string, - seriesMetas []seriesMeta, -) Job { - j := Job{ - tenantID: tenantID, - tableName: tableName, - indexPath: indexPath, - seriesMetas: seriesMetas, - } - j.computeBounds() - return j -} - -func (j *Job) String() string { - return j.tableName + "_" + j.tenantID + "_" -} - -func (j *Job) computeBounds() { - if len(j.seriesMetas) == 0 { - return - } - - minFrom := model.Latest - maxThrough := model.Earliest - - minFp := model.Fingerprint(math.MaxInt64) - maxFp := model.Fingerprint(0) - - for _, seriesMeta := range j.seriesMetas { - // calculate timestamp boundaries - for _, chunkRef := range seriesMeta.chunkRefs { - from, through := chunkRef.Bounds() - if minFrom > from { - minFrom = from - } - if maxThrough < through { - maxThrough = through - } - } - - // calculate fingerprint boundaries - if minFp > seriesMeta.seriesFP { - minFp = seriesMeta.seriesFP - } - if maxFp < seriesMeta.seriesFP { - maxFp = seriesMeta.seriesFP - } - } - - j.from = minFrom - j.through = maxThrough - - j.minFp = minFp - j.maxFp = maxFp -} diff --git a/pkg/bloomcompactor/mergecompactor.go b/pkg/bloomcompactor/mergecompactor.go deleted file mode 100644 index 3486e40846b8..000000000000 --- a/pkg/bloomcompactor/mergecompactor.go +++ /dev/null @@ -1,150 +0,0 @@ -package bloomcompactor - -import ( - "context" - - "github.com/grafana/dskit/concurrency" - - "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/storage/chunk" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" -) - -func makeSeriesIterFromSeriesMeta(job Job) *v1.SliceIter[*v1.Series] { - // Satisfy types for series - seriesFromSeriesMeta := make([]*v1.Series, len(job.seriesMetas)) - - for i, s := range job.seriesMetas { - crefs := make([]v1.ChunkRef, len(s.chunkRefs)) - for j, chk := range s.chunkRefs { - crefs[j] = v1.ChunkRef{ - Start: chk.From(), - End: chk.Through(), - Checksum: chk.Checksum, - } - } - seriesFromSeriesMeta[i] = &v1.Series{ - Fingerprint: s.seriesFP, - Chunks: crefs, - } - } - return v1.NewSliceIter(seriesFromSeriesMeta) -} - -func makeBlockIterFromBlocks(ctx context.Context, logger log.Logger, - bloomShipperClient bloomshipper.Client, blocksToUpdate []bloomshipper.BlockRef, - workingDir string) ([]v1.PeekingIterator[*v1.SeriesWithBloom], []string, error) { - - // Download existing blocks that needs compaction - blockIters := make([]v1.PeekingIterator[*v1.SeriesWithBloom], len(blocksToUpdate)) - blockPaths := make([]string, len(blocksToUpdate)) - - err := concurrency.ForEachJob(ctx, len(blocksToUpdate), len(blocksToUpdate), func(ctx context.Context, i int) error { - b := blocksToUpdate[i] - - lazyBlock, err := bloomShipperClient.GetBlock(ctx, b) - if err != nil { - level.Error(logger).Log("msg", "failed downloading block", "err", err) - return err - } - - blockPath, err := bloomshipper.UncompressBloomBlock(&lazyBlock, workingDir, logger) - if err != nil { - level.Error(logger).Log("msg", "failed extracting block", "err", err) - return err - } - blockPaths[i] = blockPath - - reader := v1.NewDirectoryBlockReader(blockPath) - block := v1.NewBlock(reader) - blockQuerier := v1.NewBlockQuerier(block) - - blockIters[i] = v1.NewPeekingIter[*v1.SeriesWithBloom](blockQuerier) - return nil - }) - - if err != nil { - return nil, nil, err - } - return blockIters, blockPaths, nil -} - -func createPopulateFunc(_ context.Context, job Job, _ storeClient, bt *v1.BloomTokenizer, _ Limits) func(series *v1.Series, bloom *v1.Bloom) error { - return func(series *v1.Series, bloom *v1.Bloom) error { - bloomForChks := v1.SeriesWithBloom{ - Series: series, - Bloom: bloom, - } - - // Satisfy types for chunks - chunkRefs := make([]chunk.Chunk, len(series.Chunks)) - for i, chk := range series.Chunks { - chunkRefs[i] = chunk.Chunk{ - ChunkRef: logproto.ChunkRef{ - Fingerprint: uint64(series.Fingerprint), - UserID: job.tenantID, - From: chk.Start, - Through: chk.End, - Checksum: chk.Checksum, - }, - } - } - - // batchesIterator, err := newChunkBatchesIterator(ctx, storeClient.chunk, chunkRefs, limits.BloomCompactorChunksBatchSize(job.tenantID)) - // if err != nil { - // return fmt.Errorf("error creating chunks batches iterator: %w", err) - // } - // NB(owen-d): this panics/etc, but the code is being refactored and will be removed. - // I've replaced `batchesIterator` with `emptyIter` to pass compiler checks while keeping this code around as reference - err := bt.Populate(&bloomForChks, v1.NewEmptyIter[v1.ChunkRefWithIter]()) - if err != nil { - return err - } - return nil - } -} - -func mergeCompactChunks(logger log.Logger, - populate func(*v1.Series, *v1.Bloom) error, - mergeBlockBuilder *PersistentBlockBuilder, - blockIters []v1.PeekingIterator[*v1.SeriesWithBloom], seriesIter *v1.SliceIter[*v1.Series], - job Job) (bloomshipper.Block, error) { - - mergeBuilder := v1.NewMergeBuilder( - blockIters, - seriesIter, - populate) - - checksum, err := mergeBlockBuilder.mergeBuild(mergeBuilder) - if err != nil { - level.Error(logger).Log("msg", "failed merging the blooms", "err", err) - return bloomshipper.Block{}, err - } - data, err := mergeBlockBuilder.Data() - if err != nil { - level.Error(logger).Log("msg", "failed reading bloom data", "err", err) - return bloomshipper.Block{}, err - } - - mergedBlock := bloomshipper.Block{ - BlockRef: bloomshipper.BlockRef{ - Ref: bloomshipper.Ref{ - TenantID: job.tenantID, - TableName: job.tableName, - MinFingerprint: uint64(job.minFp), - MaxFingerprint: uint64(job.maxFp), - StartTimestamp: job.from, - EndTimestamp: job.through, - Checksum: checksum, - }, - IndexPath: job.indexPath, - }, - Data: data, - } - return mergedBlock, nil -} diff --git a/pkg/bloomcompactor/metrics.go b/pkg/bloomcompactor/metrics.go index ee2f1630ab5e..9f844f0e40f7 100644 --- a/pkg/bloomcompactor/metrics.go +++ b/pkg/bloomcompactor/metrics.go @@ -3,6 +3,8 @@ package bloomcompactor import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" ) const ( @@ -13,97 +15,142 @@ const ( statusFailure = "failure" ) -type metrics struct { - compactionRunsStarted prometheus.Counter - compactionRunsCompleted *prometheus.CounterVec - compactionRunTime *prometheus.HistogramVec - compactionRunDiscoveredTenants prometheus.Counter - compactionRunSkippedTenants prometheus.Counter - compactionRunTenantsCompleted *prometheus.CounterVec - compactionRunTenantsTime *prometheus.HistogramVec - compactionRunJobStarted prometheus.Counter - compactionRunJobCompleted *prometheus.CounterVec - compactionRunJobTime *prometheus.HistogramVec - compactionRunInterval prometheus.Gauge - compactorRunning prometheus.Gauge +type Metrics struct { + bloomMetrics *v1.Metrics + compactorRunning prometheus.Gauge + chunkSize prometheus.Histogram // uncompressed size of all chunks summed per series + + compactionsStarted prometheus.Counter + compactionCompleted *prometheus.CounterVec + compactionTime *prometheus.HistogramVec + + tenantsDiscovered prometheus.Counter + tenantsOwned prometheus.Counter + tenantsSkipped prometheus.Counter + tenantsStarted prometheus.Counter + tenantsCompleted *prometheus.CounterVec + tenantsCompletedTime *prometheus.HistogramVec + tenantsSeries prometheus.Histogram + + blocksReused prometheus.Counter + + blocksCreated prometheus.Counter + blocksDeleted prometheus.Counter + metasCreated prometheus.Counter + metasDeleted prometheus.Counter } -func newMetrics(r prometheus.Registerer) *metrics { - m := metrics{ - compactionRunsStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ +func NewMetrics(r prometheus.Registerer, bloomMetrics *v1.Metrics) *Metrics { + m := Metrics{ + bloomMetrics: bloomMetrics, + compactorRunning: promauto.With(r).NewGauge(prometheus.GaugeOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "runs_started_total", + Name: "running", + Help: "Value will be 1 if compactor is currently running on this instance", + }), + chunkSize: promauto.With(r).NewHistogram(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "chunk_series_size", + Help: "Uncompressed size of chunks in a series", + Buckets: prometheus.ExponentialBucketsRange(1024, 1073741824, 10), + }), + + compactionsStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "compactions_started_total", Help: "Total number of compactions started", }), - compactionRunsCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + compactionCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "runs_completed_total", - Help: "Total number of compactions completed successfully", + Name: "compactions_completed_total", + Help: "Total number of compactions completed", }, []string{"status"}), - compactionRunTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + compactionTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "runs_time_seconds", + Name: "compactions_time_seconds", Help: "Time spent during a compaction cycle.", Buckets: prometheus.DefBuckets, }, []string{"status"}), - compactionRunDiscoveredTenants: promauto.With(r).NewCounter(prometheus.CounterOpts{ + + tenantsDiscovered: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "tenants_discovered", + Name: "tenants_discovered_total", Help: "Number of tenants discovered during the current compaction run", }), - compactionRunSkippedTenants: promauto.With(r).NewCounter(prometheus.CounterOpts{ + tenantsOwned: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "tenants_skipped", - Help: "Number of tenants skipped during the current compaction run", + Name: "tenants_owned", + Help: "Number of tenants owned by this instance", }), - compactionRunTenantsCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + tenantsSkipped: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "tenants_completed", + Name: "tenants_skipped_total", + Help: "Number of tenants skipped since they are not owned by this instance", + }), + tenantsStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "tenants_started_total", + Help: "Number of tenants started to process during the current compaction run", + }), + tenantsCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "tenants_completed_total", Help: "Number of tenants successfully processed during the current compaction run", }, []string{"status"}), - compactionRunTenantsTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + tenantsCompletedTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, Name: "tenants_time_seconds", Help: "Time spent processing tenants.", Buckets: prometheus.DefBuckets, }, []string{"status"}), - compactionRunJobStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ + tenantsSeries: promauto.With(r).NewHistogram(prometheus.HistogramOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "job_started", - Help: "Number of jobs started processing during the current compaction run", + Name: "tenants_series", + Help: "Number of series processed per tenant in the owned fingerprint-range.", + // Up to 10M series per tenant, way more than what we expect given our max_global_streams_per_user limits + Buckets: prometheus.ExponentialBucketsRange(1, 10000000, 10), }), - compactionRunJobCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + blocksReused: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "job_completed", - Help: "Number of jobs successfully processed during the current compaction run", - }, []string{"status"}), - compactionRunJobTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + Name: "blocks_reused_total", + Help: "Number of overlapping bloom blocks reused when creating new blocks", + }), + blocksCreated: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "job_time_seconds", - Help: "Time spent processing jobs.", - Buckets: prometheus.DefBuckets, - }, []string{"status"}), - compactionRunInterval: promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Name: "blocks_created_total", + Help: "Number of blocks created", + }), + blocksDeleted: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "compaction_interval_seconds", - Help: "The configured interval on which compaction is run in seconds", + Name: "blocks_deleted_total", + Help: "Number of blocks deleted", }), - compactorRunning: promauto.With(r).NewGauge(prometheus.GaugeOpts{ + metasCreated: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, - Name: "running", - Help: "Value will be 1 if compactor is currently running on this instance", + Name: "metas_created_total", + Help: "Number of metas created", + }), + metasDeleted: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "metas_deleted_total", + Help: "Number of metas deleted", }), } diff --git a/pkg/bloomcompactor/sharding.go b/pkg/bloomcompactor/sharding.go deleted file mode 100644 index 9b3009bd5065..000000000000 --- a/pkg/bloomcompactor/sharding.go +++ /dev/null @@ -1,58 +0,0 @@ -package bloomcompactor - -import ( - "github.com/grafana/dskit/ring" - - util_ring "github.com/grafana/loki/pkg/util/ring" -) - -var ( - // TODO: Should we include LEAVING instances in the replication set? - RingOp = ring.NewOp([]ring.InstanceState{ring.JOINING, ring.ACTIVE}, nil) -) - -// ShardingStrategy describes whether compactor "owns" given user or job. -type ShardingStrategy interface { - util_ring.TenantSharding - OwnsFingerprint(tenantID string, fp uint64) (bool, error) -} - -type ShuffleShardingStrategy struct { - util_ring.TenantSharding - ringLifeCycler *ring.BasicLifecycler -} - -func NewShuffleShardingStrategy(r *ring.Ring, ringLifecycler *ring.BasicLifecycler, limits Limits) *ShuffleShardingStrategy { - s := ShuffleShardingStrategy{ - TenantSharding: util_ring.NewTenantShuffleSharding(r, ringLifecycler, limits.BloomCompactorShardSize), - ringLifeCycler: ringLifecycler, - } - - return &s -} - -// OwnsFingerprint makes sure only a single compactor processes the fingerprint. -func (s *ShuffleShardingStrategy) OwnsFingerprint(tenantID string, fp uint64) (bool, error) { - if !s.OwnsTenant(tenantID) { - return false, nil - } - - tenantRing := s.GetTenantSubRing(tenantID) - fpSharding := util_ring.NewFingerprintShuffleSharding(tenantRing, s.ringLifeCycler, RingOp) - return fpSharding.OwnsFingerprint(fp) -} - -// NoopStrategy is an implementation of the ShardingStrategy that does not -// filter anything. -type NoopStrategy struct { - util_ring.NoopStrategy -} - -// OwnsFingerprint implements TenantShuffleSharding. -func (s *NoopStrategy) OwnsFingerprint(_ string, _ uint64) (bool, error) { - return true, nil -} - -func NewNoopStrategy() *NoopStrategy { - return &NoopStrategy{NoopStrategy: util_ring.NoopStrategy{}} -} diff --git a/pkg/bloomcompactor/sharding_test.go b/pkg/bloomcompactor/sharding_test.go deleted file mode 100644 index 4e79752279fb..000000000000 --- a/pkg/bloomcompactor/sharding_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package bloomcompactor - -import ( - "context" - "flag" - "fmt" - "testing" - "time" - - "github.com/grafana/dskit/services" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - - util_log "github.com/grafana/loki/pkg/util/log" - lokiring "github.com/grafana/loki/pkg/util/ring" - "github.com/grafana/loki/pkg/validation" -) - -func TestShuffleSharding(t *testing.T) { - const shardSize = 2 - const rings = 4 - const tenants = 2000 - const jobsPerTenant = 200 - - var limits validation.Limits - limits.RegisterFlags(flag.NewFlagSet("limits", flag.PanicOnError)) - overrides, err := validation.NewOverrides(limits, nil) - require.NoError(t, err) - - var ringManagers []*lokiring.RingManager - var shards []*ShuffleShardingStrategy - for i := 0; i < rings; i++ { - var ringCfg lokiring.RingConfig - ringCfg.RegisterFlagsWithPrefix("", "", flag.NewFlagSet("ring", flag.PanicOnError)) - ringCfg.KVStore.Store = "inmemory" - ringCfg.InstanceID = fmt.Sprintf("bloom-compactor-%d", i) - ringCfg.InstanceAddr = fmt.Sprintf("localhost-%d", i) - - ringManager, err := lokiring.NewRingManager("bloom-compactor", lokiring.ServerMode, ringCfg, 1, 1, util_log.Logger, prometheus.NewRegistry()) - require.NoError(t, err) - require.NoError(t, ringManager.StartAsync(context.Background())) - - sharding := NewShuffleShardingStrategy(ringManager.Ring, ringManager.RingLifecycler, mockLimits{ - Overrides: overrides, - bloomCompactorShardSize: shardSize, - }) - - ringManagers = append(ringManagers, ringManager) - shards = append(shards, sharding) - } - - // Wait for all rings to see each other. - for i := 0; i < rings; i++ { - require.Eventually(t, func() bool { - running := ringManagers[i].State() == services.Running - discovered := ringManagers[i].Ring.InstancesCount() == rings - return running && discovered - }, 1*time.Minute, 100*time.Millisecond) - } - - // This is kind of an un-deterministic test, because sharding is random - // and the seed is initialized by the ring lib. - // Here we'll generate a bunch of tenants and test that if the sharding doesn't own the tenant, - // that's because the tenant is owned by other ring instances. - shard := shards[0] - otherShards := shards[1:] - var ownedTenants, ownedJobs int - for i := 0; i < tenants; i++ { - tenant := fmt.Sprintf("tenant-%d", i) - ownsTenant := shard.OwnsTenant(tenant) - - var tenantOwnedByOther int - for _, other := range otherShards { - otherOwns := other.OwnsTenant(tenant) - if otherOwns { - tenantOwnedByOther++ - } - } - - // If this shard owns the tenant, shardSize-1 other members should also own the tenant. - // Otherwise, shardSize other members should own the tenant. - if ownsTenant { - require.Equal(t, shardSize-1, tenantOwnedByOther) - ownedTenants++ - } else { - require.Equal(t, shardSize, tenantOwnedByOther) - } - - for j := 0; j < jobsPerTenant; j++ { - lbls := labels.FromStrings("namespace", fmt.Sprintf("namespace-%d", j)) - fp := model.Fingerprint(lbls.Hash()) - ownsFingerprint, err := shard.OwnsFingerprint(tenant, uint64(fp)) - require.NoError(t, err) - - var jobOwnedByOther int - for _, other := range otherShards { - otherOwns, err := other.OwnsFingerprint(tenant, uint64(fp)) - require.NoError(t, err) - if otherOwns { - jobOwnedByOther++ - } - } - - // If this shard owns the job, no one else should own the job. - // And if this shard doesn't own the job, only one of the other shards should own the job. - if ownsFingerprint { - require.Equal(t, 0, jobOwnedByOther) - ownedJobs++ - } else { - require.Equal(t, 1, jobOwnedByOther) - } - } - } - - t.Logf("owned tenants: %d (out of %d)", ownedTenants, tenants) - t.Logf("owned jobs: %d (out of %d)", ownedJobs, tenants*jobsPerTenant) - - // Stop all rings and wait for them to stop. - for i := 0; i < rings; i++ { - ringManagers[i].StopAsync() - require.Eventually(t, func() bool { - return ringManagers[i].State() == services.Terminated - }, 1*time.Minute, 100*time.Millisecond) - } -} - -type mockLimits struct { - *validation.Overrides - bloomCompactorShardSize int - chunksDownloadingBatchSize int - fpRate float64 -} - -func (m mockLimits) BloomFalsePositiveRate(_ string) float64 { - return m.fpRate -} - -func (m mockLimits) BloomCompactorShardSize(_ string) int { - return m.bloomCompactorShardSize -} - -func (m mockLimits) BloomCompactorChunksBatchSize(_ string) int { - if m.chunksDownloadingBatchSize != 0 { - return m.chunksDownloadingBatchSize - } - return 1 -} diff --git a/pkg/bloomcompactor/spec.go b/pkg/bloomcompactor/spec.go new file mode 100644 index 000000000000..13707186a183 --- /dev/null +++ b/pkg/bloomcompactor/spec.go @@ -0,0 +1,279 @@ +package bloomcompactor + +import ( + "context" + "fmt" + "io" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/common/model" + + "github.com/grafana/loki/pkg/logproto" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/chunk" + "github.com/grafana/loki/pkg/storage/chunk/fetcher" + "github.com/grafana/loki/pkg/storage/stores" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" +) + +// inclusive range +type Keyspace struct { + min, max model.Fingerprint +} + +func (k Keyspace) Cmp(other Keyspace) v1.BoundsCheck { + if other.max < k.min { + return v1.Before + } else if other.min > k.max { + return v1.After + } + return v1.Overlap +} + +// Store is likely bound within. This allows specifying impls like ShardedStore +// to only request the shard-range needed from the existing store. +type BloomGenerator interface { + Generate(ctx context.Context) (skippedBlocks []v1.BlockMetadata, toClose []io.Closer, results v1.Iterator[*v1.Block], err error) +} + +// Simple implementation of a BloomGenerator. +type SimpleBloomGenerator struct { + userID string + store v1.Iterator[*v1.Series] + chunkLoader ChunkLoader + blocksIter v1.ResettableIterator[*v1.SeriesWithBloom] + + // options to build blocks with + opts v1.BlockOptions + + metrics *Metrics + logger log.Logger + + readWriterFn func() (v1.BlockWriter, v1.BlockReader) + + tokenizer *v1.BloomTokenizer +} + +// SimpleBloomGenerator is a foundational implementation of BloomGenerator. +// It mainly wires up a few different components to generate bloom filters for a set of blocks +// and handles schema compatibility: +// Blocks which are incompatible with the schema are skipped and will have their chunks reindexed +func NewSimpleBloomGenerator( + userID string, + opts v1.BlockOptions, + store v1.Iterator[*v1.Series], + chunkLoader ChunkLoader, + blocksIter v1.ResettableIterator[*v1.SeriesWithBloom], + readWriterFn func() (v1.BlockWriter, v1.BlockReader), + metrics *Metrics, + logger log.Logger, +) *SimpleBloomGenerator { + return &SimpleBloomGenerator{ + userID: userID, + opts: opts, + store: store, + chunkLoader: chunkLoader, + blocksIter: blocksIter, + logger: log.With(logger, "component", "bloom_generator"), + readWriterFn: readWriterFn, + metrics: metrics, + + tokenizer: v1.NewBloomTokenizer(opts.Schema.NGramLen(), opts.Schema.NGramSkip(), metrics.bloomMetrics), + } +} + +func (s *SimpleBloomGenerator) populator(ctx context.Context) func(series *v1.Series, bloom *v1.Bloom) error { + return func(series *v1.Series, bloom *v1.Bloom) error { + chunkItersWithFP, err := s.chunkLoader.Load(ctx, s.userID, series) + if err != nil { + return errors.Wrapf(err, "failed to load chunks for series: %+v", series) + } + + return s.tokenizer.Populate( + &v1.SeriesWithBloom{ + Series: series, + Bloom: bloom, + }, + chunkItersWithFP.itr, + ) + } + +} + +func (s *SimpleBloomGenerator) Generate(ctx context.Context) v1.Iterator[*v1.Block] { + level.Debug(s.logger).Log("msg", "generating bloom filters for blocks", "schema", fmt.Sprintf("%+v", s.opts.Schema)) + + series := v1.NewPeekingIter(s.store) + + // TODO: Use interface + impl, ok := s.blocksIter.(*blockLoadingIter) + if ok { + impl.Filter( + func(bq *bloomshipper.CloseableBlockQuerier) bool { + + logger := log.With(s.logger, "block", bq.BlockRef) + md, err := bq.Metadata() + schema := md.Options.Schema + if err != nil { + level.Warn(logger).Log("msg", "failed to get schema for block", "err", err) + bq.Close() // close unused querier + return false + } + + if !s.opts.Schema.Compatible(schema) { + level.Warn(logger).Log("msg", "block schema incompatible with options", "generator_schema", fmt.Sprintf("%+v", s.opts.Schema), "block_schema", fmt.Sprintf("%+v", schema)) + bq.Close() // close unused querier + return false + } + + level.Debug(logger).Log("msg", "adding compatible block to bloom generation inputs") + return true + }, + ) + } + + return NewLazyBlockBuilderIterator(ctx, s.opts, s.metrics, s.populator(ctx), s.readWriterFn, series, s.blocksIter) +} + +// LazyBlockBuilderIterator is a lazy iterator over blocks that builds +// each block by adding series to them until they are full. +type LazyBlockBuilderIterator struct { + ctx context.Context + opts v1.BlockOptions + metrics *Metrics + populate func(*v1.Series, *v1.Bloom) error + readWriterFn func() (v1.BlockWriter, v1.BlockReader) + series v1.PeekingIterator[*v1.Series] + blocks v1.ResettableIterator[*v1.SeriesWithBloom] + + curr *v1.Block + err error +} + +func NewLazyBlockBuilderIterator( + ctx context.Context, + opts v1.BlockOptions, + metrics *Metrics, + populate func(*v1.Series, *v1.Bloom) error, + readWriterFn func() (v1.BlockWriter, v1.BlockReader), + series v1.PeekingIterator[*v1.Series], + blocks v1.ResettableIterator[*v1.SeriesWithBloom], +) *LazyBlockBuilderIterator { + return &LazyBlockBuilderIterator{ + ctx: ctx, + opts: opts, + metrics: metrics, + populate: populate, + readWriterFn: readWriterFn, + series: series, + blocks: blocks, + } +} + +func (b *LazyBlockBuilderIterator) Next() bool { + // No more series to process + if _, hasNext := b.series.Peek(); !hasNext { + return false + } + + if err := b.ctx.Err(); err != nil { + b.err = errors.Wrap(err, "context canceled") + return false + } + + if err := b.blocks.Reset(); err != nil { + b.err = errors.Wrap(err, "reset blocks iterator") + return false + } + + mergeBuilder := v1.NewMergeBuilder(b.blocks, b.series, b.populate, b.metrics.bloomMetrics) + writer, reader := b.readWriterFn() + blockBuilder, err := v1.NewBlockBuilder(b.opts, writer) + if err != nil { + b.err = errors.Wrap(err, "failed to create bloom block builder") + return false + } + _, err = mergeBuilder.Build(blockBuilder) + if err != nil { + b.err = errors.Wrap(err, "failed to build bloom block") + return false + } + + b.curr = v1.NewBlock(reader) + return true +} + +func (b *LazyBlockBuilderIterator) At() *v1.Block { + return b.curr +} + +func (b *LazyBlockBuilderIterator) Err() error { + return b.err +} + +// IndexLoader loads an index. This helps us do things like +// load TSDBs for a specific period excluding multitenant (pre-compacted) indices +type indexLoader interface { + Index() (tsdb.Index, error) +} + +// ChunkItersByFingerprint models the chunks belonging to a fingerprint +type ChunkItersByFingerprint struct { + fp model.Fingerprint + itr v1.Iterator[v1.ChunkRefWithIter] +} + +// ChunkLoader loads chunks from a store +type ChunkLoader interface { + Load(ctx context.Context, userID string, series *v1.Series) (*ChunkItersByFingerprint, error) +} + +// StoreChunkLoader loads chunks from a store +type StoreChunkLoader struct { + fetcherProvider stores.ChunkFetcherProvider + metrics *Metrics +} + +func NewStoreChunkLoader(fetcherProvider stores.ChunkFetcherProvider, metrics *Metrics) *StoreChunkLoader { + return &StoreChunkLoader{ + fetcherProvider: fetcherProvider, + metrics: metrics, + } +} + +func (s *StoreChunkLoader) Load(ctx context.Context, userID string, series *v1.Series) (*ChunkItersByFingerprint, error) { + // NB(owen-d): This is probably unnecessary as we should only have one fetcher + // because we'll only be working on a single index period at a time, but this should protect + // us in the case of refactoring/changing this and likely isn't a perf bottleneck. + chksByFetcher := make(map[*fetcher.Fetcher][]chunk.Chunk) + for _, chk := range series.Chunks { + fetcher := s.fetcherProvider.GetChunkFetcher(chk.Start) + chksByFetcher[fetcher] = append(chksByFetcher[fetcher], chunk.Chunk{ + ChunkRef: logproto.ChunkRef{ + Fingerprint: uint64(series.Fingerprint), + UserID: userID, + From: chk.Start, + Through: chk.End, + Checksum: chk.Checksum, + }, + }) + } + + var ( + fetchers = make([]Fetcher[chunk.Chunk, chunk.Chunk], 0, len(chksByFetcher)) + inputs = make([][]chunk.Chunk, 0, len(chksByFetcher)) + ) + for fetcher, chks := range chksByFetcher { + fn := FetchFunc[chunk.Chunk, chunk.Chunk](fetcher.FetchChunks) + fetchers = append(fetchers, fn) + inputs = append(inputs, chks) + } + + return &ChunkItersByFingerprint{ + fp: series.Fingerprint, + itr: newBatchedChunkLoader(ctx, fetchers, inputs, s.metrics, batchedLoaderDefaultBatchSize), + }, nil +} diff --git a/pkg/bloomcompactor/spec_test.go b/pkg/bloomcompactor/spec_test.go new file mode 100644 index 000000000000..d5a4502a0f17 --- /dev/null +++ b/pkg/bloomcompactor/spec_test.go @@ -0,0 +1,162 @@ +package bloomcompactor + +import ( + "bytes" + "context" + "testing" + + "github.com/go-kit/log" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" +) + +func blocksFromSchema(t *testing.T, n int, options v1.BlockOptions) (res []*v1.Block, data []v1.SeriesWithBloom, refs []bloomshipper.BlockRef) { + return blocksFromSchemaWithRange(t, n, options, 0, 0xffff) +} + +// splits 100 series across `n` non-overlapping blocks. +// uses options to build blocks with. +func blocksFromSchemaWithRange(t *testing.T, n int, options v1.BlockOptions, fromFP, throughFp model.Fingerprint) (res []*v1.Block, data []v1.SeriesWithBloom, refs []bloomshipper.BlockRef) { + if 100%n != 0 { + panic("100 series must be evenly divisible by n") + } + + numSeries := 100 + data, _ = v1.MkBasicSeriesWithBlooms(numSeries, 0, fromFP, throughFp, 0, 10000) + + seriesPerBlock := numSeries / n + + for i := 0; i < n; i++ { + // references for linking in memory reader+writer + indexBuf := bytes.NewBuffer(nil) + bloomsBuf := bytes.NewBuffer(nil) + writer := v1.NewMemoryBlockWriter(indexBuf, bloomsBuf) + reader := v1.NewByteReader(indexBuf, bloomsBuf) + + builder, err := v1.NewBlockBuilder( + options, + writer, + ) + require.Nil(t, err) + + minIdx, maxIdx := i*seriesPerBlock, (i+1)*seriesPerBlock + + itr := v1.NewSliceIter[v1.SeriesWithBloom](data[minIdx:maxIdx]) + _, err = builder.BuildFrom(itr) + require.Nil(t, err) + + res = append(res, v1.NewBlock(reader)) + ref := genBlockRef(data[minIdx].Series.Fingerprint, data[maxIdx-1].Series.Fingerprint) + t.Log("create block", ref) + refs = append(refs, ref) + } + + return res, data, refs +} + +// doesn't actually load any chunks +type dummyChunkLoader struct{} + +func (dummyChunkLoader) Load(_ context.Context, _ string, series *v1.Series) (*ChunkItersByFingerprint, error) { + return &ChunkItersByFingerprint{ + fp: series.Fingerprint, + itr: v1.NewEmptyIter[v1.ChunkRefWithIter](), + }, nil +} + +func dummyBloomGen(t *testing.T, opts v1.BlockOptions, store v1.Iterator[*v1.Series], blocks []*v1.Block, refs []bloomshipper.BlockRef) *SimpleBloomGenerator { + bqs := make([]*bloomshipper.CloseableBlockQuerier, 0, len(blocks)) + for i, b := range blocks { + bqs = append(bqs, &bloomshipper.CloseableBlockQuerier{ + BlockRef: refs[i], + BlockQuerier: v1.NewBlockQuerier(b), + }) + } + + fetcher := func(_ context.Context, refs []bloomshipper.BlockRef) ([]*bloomshipper.CloseableBlockQuerier, error) { + res := make([]*bloomshipper.CloseableBlockQuerier, 0, len(refs)) + for _, ref := range refs { + for _, bq := range bqs { + if ref.Bounds.Equal(bq.Bounds) { + res = append(res, bq) + } + } + } + t.Log("req", refs) + t.Log("res", res) + return res, nil + } + + blocksIter := newBlockLoadingIter(context.Background(), refs, FetchFunc[bloomshipper.BlockRef, *bloomshipper.CloseableBlockQuerier](fetcher), 1) + + return NewSimpleBloomGenerator( + "fake", + opts, + store, + dummyChunkLoader{}, + blocksIter, + func() (v1.BlockWriter, v1.BlockReader) { + indexBuf := bytes.NewBuffer(nil) + bloomsBuf := bytes.NewBuffer(nil) + return v1.NewMemoryBlockWriter(indexBuf, bloomsBuf), v1.NewByteReader(indexBuf, bloomsBuf) + }, + NewMetrics(nil, v1.NewMetrics(nil)), + log.NewNopLogger(), + ) +} + +func TestSimpleBloomGenerator(t *testing.T) { + const maxBlockSize = 100 << 20 // 100MB + for _, tc := range []struct { + desc string + fromSchema, toSchema v1.BlockOptions + overlapping bool + }{ + { + desc: "SkipsIncompatibleSchemas", + fromSchema: v1.NewBlockOptions(3, 0, maxBlockSize), + toSchema: v1.NewBlockOptions(4, 0, maxBlockSize), + }, + { + desc: "CombinesBlocks", + fromSchema: v1.NewBlockOptions(4, 0, maxBlockSize), + toSchema: v1.NewBlockOptions(4, 0, maxBlockSize), + }, + } { + t.Run(tc.desc, func(t *testing.T) { + sourceBlocks, data, refs := blocksFromSchemaWithRange(t, 2, tc.fromSchema, 0x00000, 0x6ffff) + storeItr := v1.NewMapIter[v1.SeriesWithBloom, *v1.Series]( + v1.NewSliceIter[v1.SeriesWithBloom](data), + func(swb v1.SeriesWithBloom) *v1.Series { + return swb.Series + }, + ) + + gen := dummyBloomGen(t, tc.toSchema, storeItr, sourceBlocks, refs) + results := gen.Generate(context.Background()) + + var outputBlocks []*v1.Block + for results.Next() { + outputBlocks = append(outputBlocks, results.At()) + } + // require.Equal(t, tc.outputBlocks, len(outputBlocks)) + + // Check all the input series are present in the output blocks. + expectedRefs := v1.PointerSlice(data) + outputRefs := make([]*v1.SeriesWithBloom, 0, len(data)) + for _, block := range outputBlocks { + bq := block.Querier() + for bq.Next() { + outputRefs = append(outputRefs, bq.At()) + } + } + require.Equal(t, len(expectedRefs), len(outputRefs)) + for i := range expectedRefs { + require.Equal(t, expectedRefs[i].Series, outputRefs[i].Series) + } + }) + } +} diff --git a/pkg/bloomcompactor/table_utils.go b/pkg/bloomcompactor/table_utils.go deleted file mode 100644 index 91940f4cfd45..000000000000 --- a/pkg/bloomcompactor/table_utils.go +++ /dev/null @@ -1,37 +0,0 @@ -package bloomcompactor - -import ( - "sort" - - "github.com/prometheus/common/model" - - "github.com/grafana/loki/pkg/compactor/retention" - "github.com/grafana/loki/pkg/storage/config" -) - -func getIntervalsForTables(tables []string) map[string]model.Interval { - tablesIntervals := make(map[string]model.Interval, len(tables)) - for _, table := range tables { - tablesIntervals[table] = retention.ExtractIntervalFromTableName(table) - } - - return tablesIntervals -} - -func sortTablesByRange(tables []string, intervals map[string]model.Interval) { - sort.Slice(tables, func(i, j int) bool { - // less than if start time is after produces a most recent first sort order - return intervals[tables[i]].Start.After(intervals[tables[j]].Start) - }) -} - -// TODO: comes from pkg/compactor/compactor.go -func schemaPeriodForTable(cfg config.SchemaConfig, tableName string) (config.PeriodConfig, bool) { - tableInterval := retention.ExtractIntervalFromTableName(tableName) - schemaCfg, err := cfg.SchemaForTime(tableInterval.Start) - if err != nil || schemaCfg.IndexTables.TableFor(tableInterval.Start) != tableName { - return config.PeriodConfig{}, false - } - - return schemaCfg, true -} diff --git a/pkg/bloomcompactor/tsdb.go b/pkg/bloomcompactor/tsdb.go new file mode 100644 index 000000000000..7f5ec5eab81a --- /dev/null +++ b/pkg/bloomcompactor/tsdb.go @@ -0,0 +1,305 @@ +package bloomcompactor + +import ( + "context" + "fmt" + "io" + "math" + "path" + "strings" + + "github.com/pkg/errors" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + + "github.com/grafana/loki/pkg/chunkenc" + baseStore "github.com/grafana/loki/pkg/storage" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/config" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" +) + +const ( + gzipExtension = ".gz" +) + +type TSDBStore interface { + UsersForPeriod(ctx context.Context, table config.DayTable) ([]string, error) + ResolveTSDBs(ctx context.Context, table config.DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) + LoadTSDB( + ctx context.Context, + table config.DayTable, + tenant string, + id tsdb.Identifier, + bounds v1.FingerprintBounds, + ) (v1.CloseableIterator[*v1.Series], error) +} + +// BloomTSDBStore is a wrapper around the storage.Client interface which +// implements the TSDBStore interface for this pkg. +type BloomTSDBStore struct { + storage storage.Client +} + +func NewBloomTSDBStore(storage storage.Client) *BloomTSDBStore { + return &BloomTSDBStore{ + storage: storage, + } +} + +func (b *BloomTSDBStore) UsersForPeriod(ctx context.Context, table config.DayTable) ([]string, error) { + _, users, err := b.storage.ListFiles(ctx, table.Addr(), true) // bypass cache for ease of testing + return users, err +} + +func (b *BloomTSDBStore) ResolveTSDBs(ctx context.Context, table config.DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) { + indices, err := b.storage.ListUserFiles(ctx, table.Addr(), tenant, true) // bypass cache for ease of testing + if err != nil { + return nil, errors.Wrap(err, "failed to list user files") + } + + ids := make([]tsdb.SingleTenantTSDBIdentifier, 0, len(indices)) + for _, index := range indices { + key := index.Name + if decompress := storage.IsCompressedFile(index.Name); decompress { + key = strings.TrimSuffix(key, gzipExtension) + } + + id, ok := tsdb.ParseSingleTenantTSDBPath(path.Base(key)) + if !ok { + return nil, errors.Errorf("failed to parse single tenant tsdb path: %s", key) + } + + ids = append(ids, id) + + } + return ids, nil +} + +func (b *BloomTSDBStore) LoadTSDB( + ctx context.Context, + table config.DayTable, + tenant string, + id tsdb.Identifier, + bounds v1.FingerprintBounds, +) (v1.CloseableIterator[*v1.Series], error) { + withCompression := id.Name() + gzipExtension + + data, err := b.storage.GetUserFile(ctx, table.Addr(), tenant, withCompression) + if err != nil { + return nil, errors.Wrap(err, "failed to get file") + } + defer data.Close() + + decompressorPool := chunkenc.GetReaderPool(chunkenc.EncGZIP) + decompressor, err := decompressorPool.GetReader(data) + if err != nil { + return nil, errors.Wrap(err, "failed to get decompressor") + } + defer decompressorPool.PutReader(decompressor) + + buf, err := io.ReadAll(decompressor) + if err != nil { + return nil, errors.Wrap(err, "failed to read file") + } + + reader, err := index.NewReader(index.RealByteSlice(buf)) + if err != nil { + return nil, errors.Wrap(err, "failed to create index reader") + } + + idx := tsdb.NewTSDBIndex(reader) + + return NewTSDBSeriesIter(ctx, idx, bounds), nil +} + +// TSDBStore is an interface for interacting with the TSDB, +// modeled off a relevant subset of the `tsdb.TSDBIndex` struct +type forSeries interface { + ForSeries( + ctx context.Context, + fpFilter index.FingerprintFilter, + from model.Time, + through model.Time, + fn func(labels.Labels, model.Fingerprint, []index.ChunkMeta), + matchers ...*labels.Matcher, + ) error + Close() error +} + +type TSDBSeriesIter struct { + f forSeries + bounds v1.FingerprintBounds + ctx context.Context + + ch chan *v1.Series + initialized bool + next *v1.Series + err error +} + +func NewTSDBSeriesIter(ctx context.Context, f forSeries, bounds v1.FingerprintBounds) *TSDBSeriesIter { + return &TSDBSeriesIter{ + f: f, + bounds: bounds, + ctx: ctx, + ch: make(chan *v1.Series), + } +} + +func (t *TSDBSeriesIter) Next() bool { + if !t.initialized { + t.initialized = true + t.background() + } + + select { + case <-t.ctx.Done(): + return false + case next, ok := <-t.ch: + t.next = next + return ok + } +} + +func (t *TSDBSeriesIter) At() *v1.Series { + return t.next +} + +func (t *TSDBSeriesIter) Err() error { + if t.err != nil { + return t.err + } + + return t.ctx.Err() +} + +func (t *TSDBSeriesIter) Close() error { + return t.f.Close() +} + +// background iterates over the tsdb file, populating the next +// value via a channel to handle backpressure +func (t *TSDBSeriesIter) background() { + go func() { + t.err = t.f.ForSeries( + t.ctx, + t.bounds, + 0, math.MaxInt64, + func(_ labels.Labels, fp model.Fingerprint, chks []index.ChunkMeta) { + + res := &v1.Series{ + Fingerprint: fp, + Chunks: make(v1.ChunkRefs, 0, len(chks)), + } + for _, chk := range chks { + res.Chunks = append(res.Chunks, v1.ChunkRef{ + Start: model.Time(chk.MinTime), + End: model.Time(chk.MaxTime), + Checksum: chk.Checksum, + }) + } + + select { + case <-t.ctx.Done(): + return + case t.ch <- res: + } + }, + labels.MustNewMatcher(labels.MatchEqual, "", ""), + ) + close(t.ch) + }() +} + +type TSDBStores struct { + schemaCfg config.SchemaConfig + stores []TSDBStore +} + +func NewTSDBStores( + schemaCfg config.SchemaConfig, + storeCfg baseStore.Config, + clientMetrics baseStore.ClientMetrics, +) (*TSDBStores, error) { + res := &TSDBStores{ + schemaCfg: schemaCfg, + stores: make([]TSDBStore, len(schemaCfg.Configs)), + } + + for i, cfg := range schemaCfg.Configs { + if cfg.IndexType == config.TSDBType { + + c, err := baseStore.NewObjectClient(cfg.ObjectType, storeCfg, clientMetrics) + if err != nil { + return nil, errors.Wrap(err, "failed to create object client") + } + res.stores[i] = NewBloomTSDBStore(storage.NewIndexStorageClient(c, cfg.IndexTables.PathPrefix)) + } + } + + return res, nil +} + +func (s *TSDBStores) storeForPeriod(table config.DayTime) (TSDBStore, error) { + for i := len(s.schemaCfg.Configs) - 1; i >= 0; i-- { + period := s.schemaCfg.Configs[i] + + if !table.Before(period.From) { + // we have the desired period config + + if s.stores[i] != nil { + // valid: it's of tsdb type + return s.stores[i], nil + } + + // invalid + return nil, errors.Errorf( + "store for period is not of TSDB type (%s) while looking up store for (%v)", + period.IndexType, + table, + ) + } + + } + + return nil, fmt.Errorf( + "there is no store matching no matching period found for table (%v) -- too early", + table, + ) +} + +func (s *TSDBStores) UsersForPeriod(ctx context.Context, table config.DayTable) ([]string, error) { + store, err := s.storeForPeriod(table.DayTime) + if err != nil { + return nil, err + } + + return store.UsersForPeriod(ctx, table) +} + +func (s *TSDBStores) ResolveTSDBs(ctx context.Context, table config.DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) { + store, err := s.storeForPeriod(table.DayTime) + if err != nil { + return nil, err + } + + return store.ResolveTSDBs(ctx, table, tenant) +} + +func (s *TSDBStores) LoadTSDB( + ctx context.Context, + table config.DayTable, + tenant string, + id tsdb.Identifier, + bounds v1.FingerprintBounds, +) (v1.CloseableIterator[*v1.Series], error) { + store, err := s.storeForPeriod(table.DayTime) + if err != nil { + return nil, err + } + + return store.LoadTSDB(ctx, table, tenant, id, bounds) +} diff --git a/pkg/bloomcompactor/tsdb_test.go b/pkg/bloomcompactor/tsdb_test.go new file mode 100644 index 000000000000..08f301758bf5 --- /dev/null +++ b/pkg/bloomcompactor/tsdb_test.go @@ -0,0 +1,86 @@ +package bloomcompactor + +import ( + "context" + "math" + "testing" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" +) + +type forSeriesTestImpl []*v1.Series + +func (f forSeriesTestImpl) ForSeries( + _ context.Context, + _ index.FingerprintFilter, + _ model.Time, + _ model.Time, + fn func(labels.Labels, model.Fingerprint, []index.ChunkMeta), + _ ...*labels.Matcher, +) error { + for i := range f { + unmapped := make([]index.ChunkMeta, 0, len(f[i].Chunks)) + for _, c := range f[i].Chunks { + unmapped = append(unmapped, index.ChunkMeta{ + MinTime: int64(c.Start), + MaxTime: int64(c.End), + Checksum: c.Checksum, + }) + } + + fn(nil, f[i].Fingerprint, unmapped) + } + return nil +} + +func (f forSeriesTestImpl) Close() error { + return nil +} + +func TestTSDBSeriesIter(t *testing.T) { + input := []*v1.Series{ + { + Fingerprint: 1, + Chunks: []v1.ChunkRef{ + { + Start: 0, + End: 1, + Checksum: 2, + }, + { + Start: 3, + End: 4, + Checksum: 5, + }, + }, + }, + } + srcItr := v1.NewSliceIter(input) + itr := NewTSDBSeriesIter(context.Background(), forSeriesTestImpl(input), v1.NewBounds(0, math.MaxUint64)) + + v1.EqualIterators[*v1.Series]( + t, + func(a, b *v1.Series) { + require.Equal(t, a, b) + }, + itr, + srcItr, + ) +} + +func TestTSDBSeriesIter_Expiry(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + itr := NewTSDBSeriesIter(ctx, forSeriesTestImpl{ + {}, // a single entry + }, v1.NewBounds(0, math.MaxUint64)) + + require.False(t, itr.Next()) + require.Error(t, itr.Err()) + +} diff --git a/pkg/bloomcompactor/utils.go b/pkg/bloomcompactor/utils.go deleted file mode 100644 index 4b9c3ff541fe..000000000000 --- a/pkg/bloomcompactor/utils.go +++ /dev/null @@ -1,37 +0,0 @@ -package bloomcompactor - -import "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" - -func matchingBlocks(metas []bloomshipper.Meta, job Job) ([]bloomshipper.Meta, []bloomshipper.BlockRef) { - var metasMatchingJob []bloomshipper.Meta - var blocksMatchingJob []bloomshipper.BlockRef - oldTombstonedBlockRefs := make(map[bloomshipper.BlockRef]struct{}) - - for _, meta := range metas { - if meta.TableName != job.tableName { - continue - } - metasMatchingJob = append(metasMatchingJob, meta) - - for _, tombstonedBlockRef := range meta.Tombstones { - oldTombstonedBlockRefs[tombstonedBlockRef] = struct{}{} - } - } - - for _, meta := range metasMatchingJob { - for _, blockRef := range meta.Blocks { - if _, ok := oldTombstonedBlockRefs[blockRef]; ok { - // skip any previously tombstoned blockRefs - continue - } - - if blockRef.IndexPath == job.indexPath { - // index has not changed, no compaction needed - continue - } - blocksMatchingJob = append(blocksMatchingJob, blockRef) - } - } - - return metasMatchingJob, blocksMatchingJob -} diff --git a/pkg/bloomcompactor/v2_meta.go b/pkg/bloomcompactor/v2_meta.go deleted file mode 100644 index 1be785c0934a..000000000000 --- a/pkg/bloomcompactor/v2_meta.go +++ /dev/null @@ -1,135 +0,0 @@ -package bloomcompactor - -import ( - "fmt" - "hash" - "path" - - "github.com/pkg/errors" - - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" - "github.com/grafana/loki/pkg/util/encoding" -) - -const ( - BloomPrefix = "bloom" - MetasPrefix = "metas" -) - -// TODO(owen-d): Probably want to integrate against the block shipper -// instead of defining here, but only (min,max,fp) should be required for -// the ref. Things like index-paths, etc are not needed and possibly harmful -// in the case we want to do migrations. It's easier to load a block-ref or similar -// within the context of a specific tenant+period+index path and not couple them. -type BlockRef struct { - OwnershipRange v1.FingerprintBounds - Checksum uint32 -} - -func (r BlockRef) Hash(h hash.Hash32) error { - if err := r.OwnershipRange.Hash(h); err != nil { - return err - } - - var enc encoding.Encbuf - enc.PutBE32(r.Checksum) - _, err := h.Write(enc.Get()) - return errors.Wrap(err, "writing BlockRef") -} - -type MetaRef struct { - OwnershipRange v1.FingerprintBounds - Checksum uint32 -} - -// `bloom///metas/--.json` -func (m MetaRef) Address(tenant string, period int) (string, error) { - joined := path.Join( - BloomPrefix, - fmt.Sprintf("%v", period), - tenant, - MetasPrefix, - fmt.Sprintf("%v-%v", m.OwnershipRange, m.Checksum), - ) - - return fmt.Sprintf("%s.json", joined), nil -} - -type Meta struct { - - // The fingerprint range of the block. This is the range _owned_ by the meta and - // is greater than or equal to the range of the actual data in the underlying blocks. - OwnershipRange v1.FingerprintBounds - - // Old blocks which can be deleted in the future. These should be from pervious compaction rounds. - Tombstones []BlockRef - - // The specific TSDB files used to generate the block. - Sources []tsdb.SingleTenantTSDBIdentifier - - // A list of blocks that were generated - Blocks []BlockRef -} - -// Generate MetaRef from Meta -func (m Meta) Ref() (MetaRef, error) { - checksum, err := m.Checksum() - if err != nil { - return MetaRef{}, errors.Wrap(err, "getting checksum") - } - return MetaRef{ - OwnershipRange: m.OwnershipRange, - Checksum: checksum, - }, nil -} - -func (m Meta) Checksum() (uint32, error) { - h := v1.Crc32HashPool.Get() - defer v1.Crc32HashPool.Put(h) - - _, err := h.Write([]byte(m.OwnershipRange.String())) - if err != nil { - return 0, errors.Wrap(err, "writing OwnershipRange") - } - - for _, tombstone := range m.Tombstones { - err = tombstone.Hash(h) - if err != nil { - return 0, errors.Wrap(err, "writing Tombstones") - } - } - - for _, source := range m.Sources { - err = source.Hash(h) - if err != nil { - return 0, errors.Wrap(err, "writing Sources") - } - } - - for _, block := range m.Blocks { - err = block.Hash(h) - if err != nil { - return 0, errors.Wrap(err, "writing Blocks") - } - } - - return h.Sum32(), nil - -} - -type TSDBStore interface { - ResolveTSDBs() ([]*tsdb.TSDBFile, error) -} - -type MetaStore interface { - GetMetas([]MetaRef) ([]Meta, error) - PutMeta(Meta) error - ResolveMetas(bounds v1.FingerprintBounds) ([]MetaRef, error) -} - -type BlockStore interface { - // TODO(owen-d): flesh out|integrate against bloomshipper.Client - GetBlocks([]BlockRef) ([]interface{}, error) - PutBlock(interface{}) error -} diff --git a/pkg/bloomcompactor/v2controller.go b/pkg/bloomcompactor/v2controller.go deleted file mode 100644 index 3fbcd04cd93d..000000000000 --- a/pkg/bloomcompactor/v2controller.go +++ /dev/null @@ -1,206 +0,0 @@ -package bloomcompactor - -import ( - "context" - "fmt" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" -) - -type SimpleBloomController struct { - ownershipRange v1.FingerprintBounds // ownership range of this controller - tsdbStore TSDBStore - metaStore MetaStore - blockStore BlockStore - - // TODO(owen-d): add metrics - logger log.Logger -} - -func NewSimpleBloomController( - ownershipRange v1.FingerprintBounds, - tsdbStore TSDBStore, - metaStore MetaStore, - blockStore BlockStore, - logger log.Logger, -) *SimpleBloomController { - return &SimpleBloomController{ - ownershipRange: ownershipRange, - tsdbStore: tsdbStore, - metaStore: metaStore, - blockStore: blockStore, - logger: log.With(logger, "ownership", ownershipRange), - } -} - -func (s *SimpleBloomController) do(_ context.Context) error { - // 1. Resolve TSDBs - tsdbs, err := s.tsdbStore.ResolveTSDBs() - if err != nil { - level.Error(s.logger).Log("msg", "failed to resolve tsdbs", "err", err) - return errors.Wrap(err, "failed to resolve tsdbs") - } - - // 2. Resolve Metas - metaRefs, err := s.metaStore.ResolveMetas(s.ownershipRange) - if err != nil { - level.Error(s.logger).Log("msg", "failed to resolve metas", "err", err) - return errors.Wrap(err, "failed to resolve metas") - } - - // 3. Fetch metas - metas, err := s.metaStore.GetMetas(metaRefs) - if err != nil { - level.Error(s.logger).Log("msg", "failed to get metas", "err", err) - return errors.Wrap(err, "failed to get metas") - } - - ids := make([]tsdb.Identifier, 0, len(tsdbs)) - for _, idx := range tsdbs { - ids = append(ids, idx.Identifier) - } - - // 4. Determine which TSDBs have gaps in the ownership range and need to - // be processed. - work, err := gapsBetweenTSDBsAndMetas(s.ownershipRange, ids, metas) - if err != nil { - level.Error(s.logger).Log("msg", "failed to find gaps", "err", err) - return errors.Wrap(err, "failed to find gaps") - } - - if len(work) == 0 { - level.Debug(s.logger).Log("msg", "blooms exist for all tsdbs") - return nil - } - - // TODO(owen-d): finish - panic("not implemented") - - // Now that we have the gaps, we will generate a bloom block for each gap. - // We can accelerate this by using existing blocks which may already contain - // needed chunks in their blooms, for instance after a new TSDB version is generated - // but contains many of the same chunk references from the previous version. - // To do this, we'll need to take the metas we've already resolved and find blocks - // overlapping the ownership ranges we've identified as needing updates. - // With these in hand, we can download the old blocks and use them to - // accelerate bloom generation for the new blocks. -} - -type tsdbGaps struct { - tsdb tsdb.Identifier - gaps []v1.FingerprintBounds -} - -// tsdbsUpToDate returns if the metas are up to date with the tsdbs. This is determined by asserting -// that for each TSDB, there are metas covering the entire ownership range which were generated from that specific TSDB. -func gapsBetweenTSDBsAndMetas( - ownershipRange v1.FingerprintBounds, - tsdbs []tsdb.Identifier, - metas []Meta, -) (res []tsdbGaps, err error) { - for _, db := range tsdbs { - id := db.Name() - - relevantMetas := make([]v1.FingerprintBounds, 0, len(metas)) - for _, meta := range metas { - for _, s := range meta.Sources { - if s.Name() == id { - relevantMetas = append(relevantMetas, meta.OwnershipRange) - } - } - } - - gaps, err := findGaps(ownershipRange, relevantMetas) - if err != nil { - return nil, err - } - - if len(gaps) > 0 { - res = append(res, tsdbGaps{ - tsdb: db, - gaps: gaps, - }) - } - } - - return res, err -} - -func findGaps(ownershipRange v1.FingerprintBounds, metas []v1.FingerprintBounds) (gaps []v1.FingerprintBounds, err error) { - if len(metas) == 0 { - return []v1.FingerprintBounds{ownershipRange}, nil - } - - // turn the available metas into a list of non-overlapping metas - // for easier processing - var nonOverlapping []v1.FingerprintBounds - // First, we reduce the metas into a smaller set by combining overlaps. They must be sorted. - var cur *v1.FingerprintBounds - for i := 0; i < len(metas); i++ { - j := i + 1 - - // first iteration (i == 0), set the current meta - if cur == nil { - cur = &metas[i] - } - - if j >= len(metas) { - // We've reached the end of the list. Add the last meta to the non-overlapping set. - nonOverlapping = append(nonOverlapping, *cur) - break - } - - combined := cur.Union(metas[j]) - if len(combined) == 1 { - // There was an overlap between the two tested ranges. Combine them and keep going. - cur = &combined[0] - continue - } - - // There was no overlap between the two tested ranges. Add the first to the non-overlapping set. - // and keep the second for the next iteration. - nonOverlapping = append(nonOverlapping, combined[0]) - cur = &combined[1] - } - - // Now, detect gaps between the non-overlapping metas and the ownership range. - // The left bound of the ownership range will be adjusted as we go. - leftBound := ownershipRange.Min - for _, meta := range nonOverlapping { - - clippedMeta := meta.Intersection(ownershipRange) - // should never happen as long as we are only combining metas - // that intersect with the ownership range - if clippedMeta == nil { - return nil, fmt.Errorf("meta is not within ownership range: %v", meta) - } - - searchRange := ownershipRange.Slice(leftBound, clippedMeta.Max) - // update the left bound for the next iteration - leftBound = min(clippedMeta.Max+1, ownershipRange.Max+1) - - // since we've already ensured that the meta is within the ownership range, - // we know the xor will be of length zero (when the meta is equal to the ownership range) - // or 1 (when the meta is a subset of the ownership range) - xors := searchRange.Unless(*clippedMeta) - if len(xors) == 0 { - // meta is equal to the ownership range. This means the meta - // covers this entire section of the ownership range. - continue - } - - gaps = append(gaps, xors[0]) - } - - if leftBound <= ownershipRange.Max { - // There is a gap between the last meta and the end of the ownership range. - gaps = append(gaps, v1.NewBounds(leftBound, ownershipRange.Max)) - } - - return gaps, nil -} diff --git a/pkg/bloomcompactor/v2controller_test.go b/pkg/bloomcompactor/v2controller_test.go deleted file mode 100644 index 0a99f26d3ce1..000000000000 --- a/pkg/bloomcompactor/v2controller_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package bloomcompactor - -import ( - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" -) - -func Test_findGaps(t *testing.T) { - for _, tc := range []struct { - desc string - err bool - exp []v1.FingerprintBounds - ownershipRange v1.FingerprintBounds - metas []v1.FingerprintBounds - }{ - { - desc: "error nonoverlapping metas", - err: true, - exp: nil, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{v1.NewBounds(11, 20)}, - }, - { - desc: "one meta with entire ownership range", - err: false, - exp: nil, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{v1.NewBounds(0, 10)}, - }, - { - desc: "two non-overlapping metas with entire ownership range", - err: false, - exp: nil, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{ - v1.NewBounds(0, 5), - v1.NewBounds(6, 10), - }, - }, - { - desc: "two overlapping metas with entire ownership range", - err: false, - exp: nil, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{ - v1.NewBounds(0, 6), - v1.NewBounds(4, 10), - }, - }, - { - desc: "one meta with partial ownership range", - err: false, - exp: []v1.FingerprintBounds{ - v1.NewBounds(6, 10), - }, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{ - v1.NewBounds(0, 5), - }, - }, - { - desc: "smaller subsequent meta with partial ownership range", - err: false, - exp: []v1.FingerprintBounds{ - v1.NewBounds(8, 10), - }, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{ - v1.NewBounds(0, 7), - v1.NewBounds(3, 4), - }, - }, - { - desc: "hole in the middle", - err: false, - exp: []v1.FingerprintBounds{ - v1.NewBounds(4, 5), - }, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{ - v1.NewBounds(0, 3), - v1.NewBounds(6, 10), - }, - }, - { - desc: "holes on either end", - err: false, - exp: []v1.FingerprintBounds{ - v1.NewBounds(0, 2), - v1.NewBounds(8, 10), - }, - ownershipRange: v1.NewBounds(0, 10), - metas: []v1.FingerprintBounds{ - v1.NewBounds(3, 5), - v1.NewBounds(6, 7), - }, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - gaps, err := findGaps(tc.ownershipRange, tc.metas) - if tc.err { - require.Error(t, err) - return - } - require.Equal(t, tc.exp, gaps) - }) - } -} - -func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { - id := func(n int) tsdb.SingleTenantTSDBIdentifier { - return tsdb.SingleTenantTSDBIdentifier{ - TS: time.Unix(int64(n), 0), - } - } - - meta := func(min, max model.Fingerprint, sources ...int) Meta { - m := Meta{ - OwnershipRange: v1.NewBounds(min, max), - } - for _, source := range sources { - m.Sources = append(m.Sources, id(source)) - } - return m - } - - for _, tc := range []struct { - desc string - err bool - exp []tsdbGaps - ownershipRange v1.FingerprintBounds - tsdbs []tsdb.Identifier - metas []Meta - }{ - { - desc: "non-overlapping tsdbs and metas", - err: true, - ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.Identifier{id(0)}, - metas: []Meta{ - meta(11, 20, 0), - }, - }, - { - desc: "single tsdb", - ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.Identifier{id(0)}, - metas: []Meta{ - meta(4, 8, 0), - }, - exp: []tsdbGaps{ - { - tsdb: id(0), - gaps: []v1.FingerprintBounds{ - v1.NewBounds(0, 3), - v1.NewBounds(9, 10), - }, - }, - }, - }, - { - desc: "multiple tsdbs with separate blocks", - ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.Identifier{id(0), id(1)}, - metas: []Meta{ - meta(0, 5, 0), - meta(6, 10, 1), - }, - exp: []tsdbGaps{ - { - tsdb: id(0), - gaps: []v1.FingerprintBounds{ - v1.NewBounds(6, 10), - }, - }, - { - tsdb: id(1), - gaps: []v1.FingerprintBounds{ - v1.NewBounds(0, 5), - }, - }, - }, - }, - { - desc: "multiple tsdbs with the same blocks", - ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.Identifier{id(0), id(1)}, - metas: []Meta{ - meta(0, 5, 0, 1), - meta(6, 8, 1), - }, - exp: []tsdbGaps{ - { - tsdb: id(0), - gaps: []v1.FingerprintBounds{ - v1.NewBounds(6, 10), - }, - }, - { - tsdb: id(1), - gaps: []v1.FingerprintBounds{ - v1.NewBounds(9, 10), - }, - }, - }, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - gaps, err := gapsBetweenTSDBsAndMetas(tc.ownershipRange, tc.tsdbs, tc.metas) - if tc.err { - require.Error(t, err) - return - } - require.Equal(t, tc.exp, gaps) - }) - } -} diff --git a/pkg/bloomcompactor/v2spec.go b/pkg/bloomcompactor/v2spec.go deleted file mode 100644 index 49e74a47188a..000000000000 --- a/pkg/bloomcompactor/v2spec.go +++ /dev/null @@ -1,333 +0,0 @@ -package bloomcompactor - -import ( - "context" - "fmt" - "math" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - - "github.com/grafana/loki/pkg/chunkenc" - "github.com/grafana/loki/pkg/logproto" - logql_log "github.com/grafana/loki/pkg/logql/log" - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" -) - -/* -This file maintains a number of things supporting bloom generation. Most notably, the `BloomGenerator` interface/implementation which builds bloom filters. - -- `BloomGenerator`: Builds blooms. Most other things in this file are supporting this in various ways. -- `SimpleBloomGenerator`: A foundational implementation of `BloomGenerator` which wires up a few different components to generate bloom filters for a set of blocks and handles schema compatibility: -- `chunkLoader`: Loads chunks w/ a specific fingerprint from the store, returns an iterator of chunk iterators. We return iterators rather than chunk implementations mainly for ease of testing. In practice, this will just be an iterator over `MemChunk`s. -*/ - -type Metrics struct { - bloomMetrics *v1.Metrics - chunkSize prometheus.Histogram // uncompressed size of all chunks summed per series -} - -func NewMetrics(r prometheus.Registerer, bloomMetrics *v1.Metrics) *Metrics { - return &Metrics{ - bloomMetrics: bloomMetrics, - chunkSize: promauto.With(r).NewHistogram(prometheus.HistogramOpts{ - Name: "bloom_chunk_series_size", - Help: "Uncompressed size of chunks in a series", - Buckets: prometheus.ExponentialBucketsRange(1024, 1073741824, 10), - }), - } -} - -// inclusive range -type Keyspace struct { - min, max model.Fingerprint -} - -func (k Keyspace) Cmp(other Keyspace) v1.BoundsCheck { - if other.max < k.min { - return v1.Before - } else if other.min > k.max { - return v1.After - } - return v1.Overlap -} - -// Store is likely bound within. This allows specifying impls like ShardedStore -// to only request the shard-range needed from the existing store. -type BloomGenerator interface { - Generate(ctx context.Context) (skippedBlocks []*v1.Block, results v1.Iterator[*v1.Block], err error) -} - -// Simple implementation of a BloomGenerator. -type SimpleBloomGenerator struct { - store v1.Iterator[*v1.Series] - chunkLoader ChunkLoader - // TODO(owen-d): blocks need not be all downloaded prior. Consider implementing - // as an iterator of iterators, where each iterator is a batch of overlapping blocks. - blocks []*v1.Block - - // options to build blocks with - opts v1.BlockOptions - - metrics *Metrics - logger log.Logger - - readWriterFn func() (v1.BlockWriter, v1.BlockReader) - - tokenizer *v1.BloomTokenizer -} - -// SimpleBloomGenerator is a foundational implementation of BloomGenerator. -// It mainly wires up a few different components to generate bloom filters for a set of blocks -// and handles schema compatibility: -// Blocks which are incompatible with the schema are skipped and will have their chunks reindexed -func NewSimpleBloomGenerator( - opts v1.BlockOptions, - store v1.Iterator[*v1.Series], - chunkLoader ChunkLoader, - blocks []*v1.Block, - readWriterFn func() (v1.BlockWriter, v1.BlockReader), - metrics *Metrics, - logger log.Logger, -) *SimpleBloomGenerator { - return &SimpleBloomGenerator{ - opts: opts, - // TODO(owen-d): implement Iterator[Series] against TSDB files to hook in here. - store: store, - chunkLoader: chunkLoader, - blocks: blocks, - logger: logger, - readWriterFn: readWriterFn, - metrics: metrics, - - tokenizer: v1.NewBloomTokenizer(opts.Schema.NGramLen(), opts.Schema.NGramSkip(), metrics.bloomMetrics), - } -} - -func (s *SimpleBloomGenerator) populator(ctx context.Context) func(series *v1.Series, bloom *v1.Bloom) error { - return func(series *v1.Series, bloom *v1.Bloom) error { - chunkItersWithFP, err := s.chunkLoader.Load(ctx, series) - if err != nil { - return errors.Wrapf(err, "failed to load chunks for series: %+v", series) - } - - return s.tokenizer.Populate( - &v1.SeriesWithBloom{ - Series: series, - Bloom: bloom, - }, - chunkItersWithFP.itr, - ) - } - -} - -func (s *SimpleBloomGenerator) Generate(ctx context.Context) (skippedBlocks []*v1.Block, results v1.Iterator[*v1.Block], err error) { - - blocksMatchingSchema := make([]v1.PeekingIterator[*v1.SeriesWithBloom], 0, len(s.blocks)) - for _, block := range s.blocks { - // TODO(owen-d): implement block naming so we can log the affected block in all these calls - logger := log.With(s.logger, "block", fmt.Sprintf("%+v", block)) - schema, err := block.Schema() - if err != nil { - level.Warn(logger).Log("msg", "failed to get schema for block", "err", err) - skippedBlocks = append(skippedBlocks, block) - } - - if !s.opts.Schema.Compatible(schema) { - level.Warn(logger).Log("msg", "block schema incompatible with options", "generator_schema", fmt.Sprintf("%+v", s.opts.Schema), "block_schema", fmt.Sprintf("%+v", schema)) - skippedBlocks = append(skippedBlocks, block) - } - - level.Debug(logger).Log("msg", "adding compatible block to bloom generation inputs") - itr := v1.NewPeekingIter[*v1.SeriesWithBloom](v1.NewBlockQuerier(block)) - blocksMatchingSchema = append(blocksMatchingSchema, itr) - } - - level.Debug(s.logger).Log("msg", "generating bloom filters for blocks", "num_blocks", len(blocksMatchingSchema), "skipped_blocks", len(skippedBlocks), "schema", fmt.Sprintf("%+v", s.opts.Schema)) - - // TODO(owen-d): implement bounded block sizes - - mergeBuilder := v1.NewMergeBuilder(blocksMatchingSchema, s.store, s.populator(ctx)) - writer, reader := s.readWriterFn() - blockBuilder, err := v1.NewBlockBuilder(v1.NewBlockOptionsFromSchema(s.opts.Schema), writer) - if err != nil { - return skippedBlocks, nil, errors.Wrap(err, "failed to create bloom block builder") - } - _, err = mergeBuilder.Build(blockBuilder) - if err != nil { - return skippedBlocks, nil, errors.Wrap(err, "failed to build bloom block") - } - - return skippedBlocks, v1.NewSliceIter[*v1.Block]([]*v1.Block{v1.NewBlock(reader)}), nil - -} - -// IndexLoader loads an index. This helps us do things like -// load TSDBs for a specific period excluding multitenant (pre-compacted) indices -type indexLoader interface { - Index() (tsdb.Index, error) -} - -// ChunkItersByFingerprint models the chunks belonging to a fingerprint -type ChunkItersByFingerprint struct { - fp model.Fingerprint - itr v1.Iterator[v1.ChunkRefWithIter] -} - -// ChunkLoader loads chunks from a store -type ChunkLoader interface { - Load(context.Context, *v1.Series) (*ChunkItersByFingerprint, error) -} - -// interface modeled from `pkg/storage/stores/composite_store.ChunkFetcherProvider` -type fetcherProvider interface { - GetChunkFetcher(model.Time) chunkFetcher -} - -// interface modeled from `pkg/storage/chunk/fetcher.Fetcher` -type chunkFetcher interface { - FetchChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error) -} - -// StoreChunkLoader loads chunks from a store -type StoreChunkLoader struct { - userID string - fetcherProvider fetcherProvider - metrics *Metrics -} - -func NewStoreChunkLoader(userID string, fetcherProvider fetcherProvider, metrics *Metrics) *StoreChunkLoader { - return &StoreChunkLoader{ - userID: userID, - fetcherProvider: fetcherProvider, - metrics: metrics, - } -} - -func (s *StoreChunkLoader) Load(ctx context.Context, series *v1.Series) (*ChunkItersByFingerprint, error) { - // TODO(owen-d): This is probalby unnecessary as we should only have one fetcher - // because we'll only be working on a single index period at a time, but this should protect - // us in the case of refactoring/changing this and likely isn't a perf bottleneck. - chksByFetcher := make(map[chunkFetcher][]chunk.Chunk) - for _, chk := range series.Chunks { - fetcher := s.fetcherProvider.GetChunkFetcher(chk.Start) - chksByFetcher[fetcher] = append(chksByFetcher[fetcher], chunk.Chunk{ - ChunkRef: logproto.ChunkRef{ - Fingerprint: uint64(series.Fingerprint), - UserID: s.userID, - From: chk.Start, - Through: chk.End, - Checksum: chk.Checksum, - }, - }) - } - - work := make([]chunkWork, 0, len(chksByFetcher)) - for fetcher, chks := range chksByFetcher { - work = append(work, chunkWork{ - fetcher: fetcher, - chks: chks, - }) - } - - return &ChunkItersByFingerprint{ - fp: series.Fingerprint, - itr: newBatchedLoader(ctx, work, batchedLoaderDefaultBatchSize, s.metrics), - }, nil -} - -type chunkWork struct { - fetcher chunkFetcher - chks []chunk.Chunk -} - -// batchedLoader implements `v1.Iterator[v1.ChunkRefWithIter]` in batches -// to ensure memory is bounded while loading chunks -// TODO(owen-d): testware -type batchedLoader struct { - metrics *Metrics - batchSize int - ctx context.Context - work []chunkWork - - cur v1.ChunkRefWithIter - batch []chunk.Chunk - err error -} - -const batchedLoaderDefaultBatchSize = 50 - -func newBatchedLoader(ctx context.Context, work []chunkWork, batchSize int, metrics *Metrics) *batchedLoader { - return &batchedLoader{ - metrics: metrics, - batchSize: batchSize, - ctx: ctx, - work: work, - } -} - -func (b *batchedLoader) Next() bool { - if len(b.batch) > 0 { - b.cur, b.err = b.format(b.batch[0]) - b.batch = b.batch[1:] - return b.err == nil - } - - if len(b.work) == 0 { - return false - } - - // setup next batch - next := b.work[0] - batchSize := min(b.batchSize, len(next.chks)) - toFetch := next.chks[:batchSize] - // update work - b.work[0].chks = next.chks[batchSize:] - if len(b.work[0].chks) == 0 { - b.work = b.work[1:] - } - - b.batch, b.err = next.fetcher.FetchChunks(b.ctx, toFetch) - return b.err == nil -} - -func (b *batchedLoader) format(c chunk.Chunk) (v1.ChunkRefWithIter, error) { - chk := c.Data.(*chunkenc.Facade).LokiChunk() - b.metrics.chunkSize.Observe(float64(chk.UncompressedSize())) - itr, err := chk.Iterator( - b.ctx, - time.Unix(0, 0), // TODO: Parameterize/better handle the timestamps? - time.Unix(0, math.MaxInt64), - logproto.FORWARD, - logql_log.NewNoopPipeline().ForStream(c.Metric), - ) - - if err != nil { - return v1.ChunkRefWithIter{}, err - } - - return v1.ChunkRefWithIter{ - Ref: v1.ChunkRef{ - Start: c.From, - End: c.Through, - Checksum: c.Checksum, - }, - Itr: itr, - }, nil -} - -func (b *batchedLoader) At() v1.ChunkRefWithIter { - return b.cur -} - -func (b *batchedLoader) Err() error { - return b.err -} diff --git a/pkg/bloomcompactor/v2spec_test.go b/pkg/bloomcompactor/v2spec_test.go deleted file mode 100644 index 08c722d06e5d..000000000000 --- a/pkg/bloomcompactor/v2spec_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package bloomcompactor - -import ( - "bytes" - "context" - "testing" - - "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" -) - -func blocksFromSchema(t *testing.T, n int, options v1.BlockOptions) (res []*v1.Block, data []v1.SeriesWithBloom) { - return blocksFromSchemaWithRange(t, n, options, 0, 0xffff) -} - -// splits 100 series across `n` non-overlapping blocks. -// uses options to build blocks with. -func blocksFromSchemaWithRange(t *testing.T, n int, options v1.BlockOptions, fromFP, throughFp model.Fingerprint) (res []*v1.Block, data []v1.SeriesWithBloom) { - if 100%n != 0 { - panic("100 series must be evenly divisible by n") - } - - numSeries := 100 - numKeysPerSeries := 10000 - data, _ = v1.MkBasicSeriesWithBlooms(numSeries, numKeysPerSeries, fromFP, throughFp, 0, 10000) - - seriesPerBlock := 100 / n - - for i := 0; i < n; i++ { - // references for linking in memory reader+writer - indexBuf := bytes.NewBuffer(nil) - bloomsBuf := bytes.NewBuffer(nil) - writer := v1.NewMemoryBlockWriter(indexBuf, bloomsBuf) - reader := v1.NewByteReader(indexBuf, bloomsBuf) - - builder, err := v1.NewBlockBuilder( - options, - writer, - ) - require.Nil(t, err) - - itr := v1.NewSliceIter[v1.SeriesWithBloom](data[i*seriesPerBlock : (i+1)*seriesPerBlock]) - _, err = builder.BuildFrom(itr) - require.Nil(t, err) - - res = append(res, v1.NewBlock(reader)) - } - - return res, data -} - -// doesn't actually load any chunks -type dummyChunkLoader struct{} - -func (dummyChunkLoader) Load(_ context.Context, series *v1.Series) (*ChunkItersByFingerprint, error) { - return &ChunkItersByFingerprint{ - fp: series.Fingerprint, - itr: v1.NewEmptyIter[v1.ChunkRefWithIter](), - }, nil -} - -func dummyBloomGen(opts v1.BlockOptions, store v1.Iterator[*v1.Series], blocks []*v1.Block) *SimpleBloomGenerator { - return NewSimpleBloomGenerator( - opts, - store, - dummyChunkLoader{}, - blocks, - func() (v1.BlockWriter, v1.BlockReader) { - indexBuf := bytes.NewBuffer(nil) - bloomsBuf := bytes.NewBuffer(nil) - return v1.NewMemoryBlockWriter(indexBuf, bloomsBuf), v1.NewByteReader(indexBuf, bloomsBuf) - }, - NewMetrics(nil, v1.NewMetrics(nil)), - log.NewNopLogger(), - ) -} - -func TestSimpleBloomGenerator(t *testing.T) { - for _, tc := range []struct { - desc string - fromSchema, toSchema v1.BlockOptions - sourceBlocks, numSkipped int - }{ - { - desc: "SkipsIncompatibleSchemas", - fromSchema: v1.NewBlockOptions(3, 0), - toSchema: v1.NewBlockOptions(4, 0), - sourceBlocks: 2, - numSkipped: 2, - }, - { - desc: "CombinesBlocks", - fromSchema: v1.NewBlockOptions(4, 0), - toSchema: v1.NewBlockOptions(4, 0), - sourceBlocks: 2, - numSkipped: 0, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - sourceBlocks, data := blocksFromSchema(t, tc.sourceBlocks, tc.fromSchema) - storeItr := v1.NewMapIter[v1.SeriesWithBloom, *v1.Series]( - v1.NewSliceIter[v1.SeriesWithBloom](data), - func(swb v1.SeriesWithBloom) *v1.Series { - return swb.Series - }, - ) - - gen := dummyBloomGen(tc.toSchema, storeItr, sourceBlocks) - skipped, results, err := gen.Generate(context.Background()) - require.Nil(t, err) - require.Equal(t, tc.numSkipped, len(skipped)) - - require.True(t, results.Next()) - block := results.At() - require.False(t, results.Next()) - - refs := v1.PointerSlice[v1.SeriesWithBloom](data) - - v1.EqualIterators[*v1.SeriesWithBloom]( - t, - func(a, b *v1.SeriesWithBloom) { - // TODO(owen-d): better equality check - // once chunk fetching is implemented - require.Equal(t, a.Series, b.Series) - }, - v1.NewSliceIter[*v1.SeriesWithBloom](refs), - block.Querier(), - ) - }) - } -} diff --git a/pkg/bloomgateway/bloomgateway.go b/pkg/bloomgateway/bloomgateway.go index afe8d646ae63..b80cc908f719 100644 --- a/pkg/bloomgateway/bloomgateway.go +++ b/pkg/bloomgateway/bloomgateway.go @@ -23,13 +23,15 @@ of line filter expressions. | bloomgateway.Gateway | - queue.RequestQueue + queue.RequestQueue | - bloomgateway.Worker + bloomgateway.Worker | - bloomshipper.Shipper + bloomgateway.Processor | - bloomshipper.BloomFileClient + bloomshipper.Store + | + bloomshipper.Client | ObjectClient | @@ -42,7 +44,6 @@ package bloomgateway import ( "context" "fmt" - "io" "sort" "sync" "time" @@ -57,11 +58,9 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/queue" - "github.com/grafana/loki/pkg/storage" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" - "github.com/grafana/loki/pkg/storage/chunk/cache" - "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/constants" @@ -70,8 +69,9 @@ import ( var errGatewayUnhealthy = errors.New("bloom-gateway is unhealthy in the ring") const ( - pendingTasksInitialCap = 1024 - metricsSubsystem = "bloom_gateway" + pendingTasksInitialCap = 1024 + metricsSubsystem = "bloom_gateway" + querierMetricsSubsystem = "bloom_gateway_querier" ) var ( @@ -80,10 +80,9 @@ var ( ) type metrics struct { - queueDuration prometheus.Histogram - inflightRequests prometheus.Summary - chunkRefsUnfiltered prometheus.Counter - chunkRefsFiltered prometheus.Counter + queueDuration prometheus.Histogram + inflightRequests prometheus.Summary + chunkRemovals *prometheus.CounterVec } func newMetrics(registerer prometheus.Registerer, namespace, subsystem string) *metrics { @@ -104,29 +103,15 @@ func newMetrics(registerer prometheus.Registerer, namespace, subsystem string) * MaxAge: time.Minute, AgeBuckets: 6, }), - chunkRefsUnfiltered: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + chunkRemovals: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "chunkrefs_pre_filtering", - Help: "Total amount of chunk refs pre filtering. Does not count chunk refs in failed requests.", - }), - chunkRefsFiltered: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "chunkrefs_post_filtering", - Help: "Total amount of chunk refs post filtering.", - }), + Name: "chunk_removals_total", + Help: "Total amount of removals received from the block querier partitioned by state. The state 'accepted' means that the removals are processed, the state 'dropped' means that the removals were received after the task context was done (e.g. client timeout, etc).", + }, []string{"state"}), } } -func (m *metrics) addUnfilteredCount(n int) { - m.chunkRefsUnfiltered.Add(float64(n)) -} - -func (m *metrics) addFilteredCount(n int) { - m.chunkRefsFiltered.Add(float64(n)) -} - // SyncMap is a map structure which can be synchronized using the RWMutex type SyncMap[k comparable, v any] struct { sync.RWMutex @@ -171,11 +156,9 @@ type Gateway struct { workerMetrics *workerMetrics queueMetrics *queue.Metrics - queue *queue.RequestQueue - activeUsers *util.ActiveUsersCleanupService - bloomShipper bloomshipper.Interface - - sharding ShardingStrategy + queue *queue.RequestQueue + activeUsers *util.ActiveUsersCleanupService + bloomStore bloomshipper.Store pendingTasks *pendingTasks @@ -194,39 +177,22 @@ func (l *fixedQueueLimits) MaxConsumers(_ string, _ int) int { } // New returns a new instance of the Bloom Gateway. -func New(cfg Config, schemaCfg config.SchemaConfig, storageCfg storage.Config, overrides Limits, shardingStrategy ShardingStrategy, cm storage.ClientMetrics, logger log.Logger, reg prometheus.Registerer) (*Gateway, error) { +func New(cfg Config, store bloomshipper.Store, logger log.Logger, reg prometheus.Registerer) (*Gateway, error) { g := &Gateway{ cfg: cfg, logger: logger, metrics: newMetrics(reg, constants.Loki, metricsSubsystem), - sharding: shardingStrategy, pendingTasks: makePendingTasks(pendingTasksInitialCap), workerConfig: workerConfig{ maxItems: 100, }, workerMetrics: newWorkerMetrics(reg, constants.Loki, metricsSubsystem), queueMetrics: queue.NewMetrics(reg, constants.Loki, metricsSubsystem), + bloomStore: store, } - g.queue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, time.Minute, &fixedQueueLimits{0}, g.queueMetrics) g.activeUsers = util.NewActiveUsersCleanupWithDefaultValues(g.queueMetrics.Cleanup) - // TODO(chaudum): Plug in cache - var metasCache cache.Cache - var blocksCache *cache.EmbeddedCache[string, io.ReadCloser] - store, err := bloomshipper.NewBloomStore(schemaCfg.Configs, storageCfg, cm, metasCache, blocksCache, logger) - if err != nil { - return nil, err - } - - bloomShipper, err := bloomshipper.NewShipper(store, storageCfg.BloomShipperConfig, overrides, logger, reg) - if err != nil { - return nil, err - } - - // We need to keep a reference to be able to call Stop() on shutdown of the gateway. - g.bloomShipper = bloomShipper - if err := g.initServices(); err != nil { return nil, err } @@ -240,7 +206,7 @@ func (g *Gateway) initServices() error { svcs := []services.Service{g.queue, g.activeUsers} for i := 0; i < g.cfg.WorkerConcurrency; i++ { id := fmt.Sprintf("bloom-query-worker-%d", i) - w := newWorker(id, g.workerConfig, g.queue, g.bloomShipper, g.pendingTasks, g.logger, g.workerMetrics) + w := newWorker(id, g.workerConfig, g.queue, g.bloomStore, g.pendingTasks, g.logger, g.workerMetrics) svcs = append(svcs, w) } g.serviceMngr, err = services.NewManager(svcs...) @@ -292,7 +258,6 @@ func (g *Gateway) running(ctx context.Context) error { } func (g *Gateway) stopping(_ error) error { - g.bloomShipper.Stop() return services.StopManagerAndAwaitStopped(context.Background(), g.serviceMngr) } @@ -317,12 +282,8 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk return nil, errors.New("from time must not be after through time") } - numChunksUnfiltered := len(req.Refs) - // Shortcut if request does not contain filters - if len(req.Filters) == 0 { - g.metrics.addUnfilteredCount(numChunksUnfiltered) - g.metrics.addFilteredCount(len(req.Refs)) + if len(syntax.ExtractLineFilters(req.Plan.AST)) == 0 { return &logproto.FilterChunkRefResponse{ ChunkRefs: req.Refs, }, nil @@ -343,14 +304,16 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk }, nil } + filters := syntax.ExtractLineFilters(req.Plan.AST) tasks := make([]Task, 0, len(seriesByDay)) - for _, seriesWithBounds := range seriesByDay { - task, err := NewTask(ctx, tenantID, seriesWithBounds, req.Filters) + for _, seriesForDay := range seriesByDay { + task, err := NewTask(ctx, tenantID, seriesForDay, filters) if err != nil { return nil, err } + level.Debug(g.logger).Log("msg", "creating task for day", "day", seriesForDay.day, "interval", seriesForDay.interval.String(), "task", task.ID) tasks = append(tasks, task) - numSeries += len(seriesWithBounds.series) + numSeries += len(seriesForDay.series) } g.activeUsers.UpdateUserTimestamp(tenantID, time.Now()) @@ -362,20 +325,19 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk tasksCh := make(chan Task, len(tasks)) for _, task := range tasks { task := task - level.Info(logger).Log("msg", "enqueue task", "task", task.ID, "day", task.day, "series", len(task.series)) + level.Info(logger).Log("msg", "enqueue task", "task", task.ID, "table", task.table, "series", len(task.series)) g.queue.Enqueue(tenantID, []string{}, task, func() { // When enqueuing, we also add the task to the pending tasks g.pendingTasks.Add(task.ID, task) }) - go consumeTask(ctx, task, tasksCh, logger) + go g.consumeTask(ctx, task, tasksCh) } responses := responsesPool.Get(numSeries) defer responsesPool.Put(responses) remaining := len(tasks) -outer: - for { + for remaining > 0 { select { case <-ctx.Done(): return nil, errors.Wrap(ctx.Err(), "request failed") @@ -386,23 +348,17 @@ outer: } responses = append(responses, task.responses...) remaining-- - if remaining == 0 { - break outer - } } } - for _, o := range responses { - if o.Removals.Len() == 0 { - continue - } - removeNotMatchingChunks(req, o, g.logger) - } + preFilterSeries := len(req.Refs) - g.metrics.addUnfilteredCount(numChunksUnfiltered) - g.metrics.addFilteredCount(len(req.Refs)) + // TODO(chaudum): Don't wait for all responses before starting to filter chunks. + filtered := g.processResponses(req, responses) - level.Info(logger).Log("msg", "return filtered chunk refs", "unfiltered", numChunksUnfiltered, "filtered", len(req.Refs)) + postFilterSeries := len(req.Refs) + + level.Info(logger).Log("msg", "return filtered chunk refs", "pre_filter_series", preFilterSeries, "post_filter_series", postFilterSeries, "filtered_chunks", filtered) return &logproto.FilterChunkRefResponse{ChunkRefs: req.Refs}, nil } @@ -412,16 +368,18 @@ outer: // task is closed by the worker. // Once the tasks is closed, it will send the task with the results from the // block querier to the supplied task channel. -func consumeTask(ctx context.Context, task Task, tasksCh chan<- Task, logger log.Logger) { - logger = log.With(logger, "task", task.ID) +func (g *Gateway) consumeTask(ctx context.Context, task Task, tasksCh chan<- Task) { + logger := log.With(g.logger, "task", task.ID) for res := range task.resCh { select { case <-ctx.Done(): level.Debug(logger).Log("msg", "drop partial result", "fp_int", uint64(res.Fp), "fp_hex", res.Fp, "chunks_to_remove", res.Removals.Len()) + g.metrics.chunkRemovals.WithLabelValues("dropped").Add(float64(res.Removals.Len())) default: level.Debug(logger).Log("msg", "accept partial result", "fp_int", uint64(res.Fp), "fp_hex", res.Fp, "chunks_to_remove", res.Removals.Len()) task.responses = append(task.responses, res) + g.metrics.chunkRemovals.WithLabelValues("accepted").Add(float64(res.Removals.Len())) } } @@ -434,7 +392,18 @@ func consumeTask(ctx context.Context, task Task, tasksCh chan<- Task, logger log } } -func removeNotMatchingChunks(req *logproto.FilterChunkRefRequest, res v1.Output, logger log.Logger) { +func (g *Gateway) processResponses(req *logproto.FilterChunkRefRequest, responses []v1.Output) (filtered int) { + for _, o := range responses { + if o.Removals.Len() == 0 { + continue + } + filtered += g.removeNotMatchingChunks(req, o) + } + return +} + +func (g *Gateway) removeNotMatchingChunks(req *logproto.FilterChunkRefRequest, res v1.Output) (filtered int) { + // binary search index of fingerprint idx := sort.Search(len(req.Refs), func(i int) bool { return req.Refs[i].Fingerprint >= uint64(res.Fp) @@ -442,13 +411,15 @@ func removeNotMatchingChunks(req *logproto.FilterChunkRefRequest, res v1.Output, // fingerprint not found if idx >= len(req.Refs) { - level.Error(logger).Log("msg", "index out of range", "idx", idx, "len", len(req.Refs), "fp", uint64(res.Fp)) + level.Error(g.logger).Log("msg", "index out of range", "idx", idx, "len", len(req.Refs), "fp", uint64(res.Fp)) return } // if all chunks of a fingerprint are are removed // then remove the whole group from the response if len(req.Refs[idx].Refs) == res.Removals.Len() { + filtered += len(req.Refs[idx].Refs) + req.Refs[idx] = nil // avoid leaking pointer req.Refs = append(req.Refs[:idx], req.Refs[idx+1:]...) return @@ -458,10 +429,13 @@ func removeNotMatchingChunks(req *logproto.FilterChunkRefRequest, res v1.Output, toRemove := res.Removals[i] for j := 0; j < len(req.Refs[idx].Refs); j++ { if toRemove.Checksum == req.Refs[idx].Refs[j].Checksum { + filtered += 1 + req.Refs[idx].Refs[j] = nil // avoid leaking pointer req.Refs[idx].Refs = append(req.Refs[idx].Refs[:j], req.Refs[idx].Refs[j+1:]...) j-- // since we removed the current item at index, we have to redo the same index } } } + return } diff --git a/pkg/bloomgateway/bloomgateway_test.go b/pkg/bloomgateway/bloomgateway_test.go index c8da44a7c719..449c8b17a538 100644 --- a/pkg/bloomgateway/bloomgateway_test.go +++ b/pkg/bloomgateway/bloomgateway_test.go @@ -3,7 +3,7 @@ package bloomgateway import ( "context" "fmt" - "os" + "math/rand" "testing" "time" @@ -16,15 +16,17 @@ import ( "github.com/grafana/dskit/user" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/storage" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" + bloomshipperconfig "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper/config" lokiring "github.com/grafana/loki/pkg/util/ring" "github.com/grafana/loki/pkg/validation" ) @@ -44,12 +46,8 @@ func newLimits() *validation.Overrides { return overrides } -func TestBloomGateway_StartStopService(t *testing.T) { - - ss := NewNoopStrategy() +func setupBloomStore(t *testing.T) *bloomshipper.BloomStore { logger := log.NewNopLogger() - reg := prometheus.NewRegistry() - limits := newLimits() cm := storage.NewClientMetrics() t.Cleanup(cm.Unregister) @@ -71,11 +69,25 @@ func TestBloomGateway_StartStopService(t *testing.T) { Configs: []config.PeriodConfig{p}, } storageCfg := storage.Config{ + BloomShipperConfig: bloomshipperconfig.Config{ + WorkingDirectory: t.TempDir(), + }, FSConfig: local.FSConfig{ Directory: t.TempDir(), }, } + store, err := bloomshipper.NewBloomStore(schemaCfg.Configs, storageCfg, cm, nil, nil, logger) + require.NoError(t, err) + t.Cleanup(store.Stop) + + return store +} + +func TestBloomGateway_StartStopService(t *testing.T) { + logger := log.NewNopLogger() + reg := prometheus.NewRegistry() + t.Run("start and stop bloom gateway", func(t *testing.T) { kvStore, closer := consul.NewInMemoryClient(ring.GetCodec(), logger, reg) t.Cleanup(func() { @@ -96,7 +108,8 @@ func TestBloomGateway_StartStopService(t *testing.T) { MaxOutstandingPerTenant: 1024, } - gw, err := New(cfg, schemaCfg, storageCfg, limits, ss, cm, logger, reg) + store := setupBloomStore(t) + gw, err := New(cfg, store, logger, reg) require.NoError(t, err) err = services.StartAndAwaitRunning(context.Background(), gw) @@ -114,35 +127,9 @@ func TestBloomGateway_StartStopService(t *testing.T) { func TestBloomGateway_FilterChunkRefs(t *testing.T) { tenantID := "test" - ss := NewNoopStrategy() - logger := log.NewLogfmtLogger(os.Stderr) + store := setupBloomStore(t) + logger := log.NewNopLogger() reg := prometheus.NewRegistry() - limits := newLimits() - - cm := storage.NewClientMetrics() - t.Cleanup(cm.Unregister) - - p := config.PeriodConfig{ - From: parseDayTime("2023-09-01"), - IndexTables: config.IndexPeriodicTableConfig{ - PeriodicTableConfig: config.PeriodicTableConfig{ - Prefix: "index_", - Period: 24 * time.Hour, - }, - }, - IndexType: config.TSDBType, - ObjectType: config.StorageTypeFileSystem, - Schema: "v13", - RowShards: 16, - } - schemaCfg := config.SchemaConfig{ - Configs: []config.PeriodConfig{p}, - } - storageCfg := storage.Config{ - FSConfig: local.FSConfig{ - Directory: t.TempDir(), - }, - } kvStore, closer := consul.NewInMemoryClient(ring.GetCodec(), logger, reg) t.Cleanup(func() { @@ -164,18 +151,14 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { } t.Run("shipper error is propagated", func(t *testing.T) { - reg := prometheus.NewRegistry() - gw, err := New(cfg, schemaCfg, storageCfg, limits, ss, cm, logger, reg) - require.NoError(t, err) - now := mktime("2023-10-03 10:00") - bqs, data := createBlockQueriers(t, 10, now.Add(-24*time.Hour), now, 0, 1000) - mockStore := newMockBloomStore(bqs) - mockStore.err = errors.New("failed to fetch block") - gw.bloomShipper = mockStore + _, metas, queriers, data := createBlocks(t, tenantID, 10, now.Add(-1*time.Hour), now, 0x0000, 0x0fff) + mockStore := newMockBloomStore(queriers, metas) + mockStore.err = errors.New("request failed") - err = gw.initServices() + reg := prometheus.NewRegistry() + gw, err := New(cfg, mockStore, logger, reg) require.NoError(t, err) err = services.StartAndAwaitRunning(context.Background(), gw) @@ -185,18 +168,19 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { require.NoError(t, err) }) - chunkRefs := createQueryInputFromBlockData(t, tenantID, data, 10) + chunkRefs := createQueryInputFromBlockData(t, tenantID, data, 100) // saturate workers // then send additional request for i := 0; i < gw.cfg.WorkerConcurrency+1; i++ { + expr, err := syntax.ParseExpr(`{foo="bar"} |= "does not match"`) + require.NoError(t, err) + req := &logproto.FilterChunkRefRequest{ From: now.Add(-24 * time.Hour), Through: now, Refs: groupRefs(t, chunkRefs), - Filters: []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: "does not match"}, - }, + Plan: plan.QueryPlan{AST: expr}, } ctx, cancelFn := context.WithTimeout(context.Background(), 10*time.Second) @@ -204,23 +188,20 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { t.Cleanup(cancelFn) res, err := gw.FilterChunkRefs(ctx, req) - require.ErrorContainsf(t, err, "request failed: failed to fetch block", "%+v", res) + require.ErrorContainsf(t, err, "request failed", "%+v", res) } }) t.Run("request cancellation does not result in channel locking", func(t *testing.T) { - reg := prometheus.NewRegistry() - gw, err := New(cfg, schemaCfg, storageCfg, limits, ss, cm, logger, reg) - require.NoError(t, err) - now := mktime("2024-01-25 10:00") - bqs, data := createBlockQueriers(t, 50, now.Add(-24*time.Hour), now, 0, 1024) - mockStore := newMockBloomStore(bqs) - mockStore.delay = 50 * time.Millisecond // delay for each block - 50x50=2500ms - gw.bloomShipper = mockStore + // replace store implementation and re-initialize workers and sub-services + _, metas, queriers, data := createBlocks(t, tenantID, 10, now.Add(-1*time.Hour), now, 0x0000, 0x0fff) + mockStore := newMockBloomStore(queriers, metas) + mockStore.delay = 2000 * time.Millisecond - err = gw.initServices() + reg := prometheus.NewRegistry() + gw, err := New(cfg, mockStore, logger, reg) require.NoError(t, err) err = services.StartAndAwaitRunning(context.Background(), gw) @@ -235,13 +216,14 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { // saturate workers // then send additional request for i := 0; i < gw.cfg.WorkerConcurrency+1; i++ { + expr, err := syntax.ParseExpr(`{foo="bar"} |= "does not match"`) + require.NoError(t, err) + req := &logproto.FilterChunkRefRequest{ From: now.Add(-24 * time.Hour), Through: now, Refs: groupRefs(t, chunkRefs), - Filters: []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: "does not match"}, - }, + Plan: plan.QueryPlan{AST: expr}, } ctx, cancelFn := context.WithTimeout(context.Background(), 500*time.Millisecond) @@ -254,8 +236,10 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { }) t.Run("returns unfiltered chunk refs if no filters provided", func(t *testing.T) { + now := mktime("2023-10-03 10:00") + reg := prometheus.NewRegistry() - gw, err := New(cfg, schemaCfg, storageCfg, limits, ss, cm, logger, reg) + gw, err := New(cfg, store, logger, reg) require.NoError(t, err) err = services.StartAndAwaitRunning(context.Background(), gw) @@ -265,8 +249,6 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { require.NoError(t, err) }) - now := mktime("2023-10-03 10:00") - chunkRefs := []*logproto.ChunkRef{ {Fingerprint: 3000, UserID: tenantID, From: now.Add(-24 * time.Hour), Through: now.Add(-23 * time.Hour), Checksum: 1}, {Fingerprint: 1000, UserID: tenantID, From: now.Add(-22 * time.Hour), Through: now.Add(-21 * time.Hour), Checksum: 2}, @@ -299,8 +281,10 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { }) t.Run("gateway tracks active users", func(t *testing.T) { + now := mktime("2023-10-03 10:00") + reg := prometheus.NewRegistry() - gw, err := New(cfg, schemaCfg, storageCfg, limits, ss, cm, logger, reg) + gw, err := New(cfg, store, logger, reg) require.NoError(t, err) err = services.StartAndAwaitRunning(context.Background(), gw) @@ -310,8 +294,6 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { require.NoError(t, err) }) - now := mktime("2023-10-03 10:00") - tenants := []string{"tenant-a", "tenant-b", "tenant-c"} for idx, tenantID := range tenants { chunkRefs := []*logproto.ChunkRef{ @@ -323,13 +305,13 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { Checksum: uint32(idx), }, } + expr, err := syntax.ParseExpr(`{foo="bar"} |= "foo"`) + require.NoError(t, err) req := &logproto.FilterChunkRefRequest{ From: now.Add(-24 * time.Hour), Through: now, Refs: groupRefs(t, chunkRefs), - Filters: []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: "foo"}, - }, + Plan: plan.QueryPlan{AST: expr}, } ctx := user.InjectOrgID(context.Background(), tenantID) _, err = gw.FilterChunkRefs(ctx, req) @@ -339,15 +321,16 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { }) t.Run("use fuse queriers to filter chunks", func(t *testing.T) { + now := mktime("2023-10-03 10:00") + reg := prometheus.NewRegistry() - gw, err := New(cfg, schemaCfg, storageCfg, limits, ss, cm, logger, reg) + gw, err := New(cfg, store, logger, reg) require.NoError(t, err) - now := mktime("2023-10-03 10:00") - // replace store implementation and re-initialize workers and sub-services - bqs, data := createBlockQueriers(t, 5, now.Add(-8*time.Hour), now, 0, 1024) - gw.bloomShipper = newMockBloomStore(bqs) + _, metas, queriers, data := createBlocks(t, tenantID, 10, now.Add(-1*time.Hour), now, 0x0000, 0x0fff) + + gw.bloomStore = newMockBloomStore(queriers, metas) err = gw.initServices() require.NoError(t, err) @@ -358,17 +341,17 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { require.NoError(t, err) }) - chunkRefs := createQueryInputFromBlockData(t, tenantID, data, 100) + chunkRefs := createQueryInputFromBlockData(t, tenantID, data, 10) t.Run("no match - return empty response", func(t *testing.T) { inputChunkRefs := groupRefs(t, chunkRefs) + expr, err := syntax.ParseExpr(`{foo="bar"} |= "does not match"`) + require.NoError(t, err) req := &logproto.FilterChunkRefRequest{ From: now.Add(-8 * time.Hour), Through: now, Refs: inputChunkRefs, - Filters: []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: "does not match"}, - }, + Plan: plan.QueryPlan{AST: expr}, } ctx := user.InjectOrgID(context.Background(), tenantID) res, err := gw.FilterChunkRefs(ctx, req) @@ -382,27 +365,38 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { t.Run("match - return filtered", func(t *testing.T) { inputChunkRefs := groupRefs(t, chunkRefs) - // hack to get indexed key for a specific series - // the indexed key range for a series is defined as - // i * keysPerSeries ... i * keysPerSeries + keysPerSeries - 1 - // where i is the nth series in a block - // fortunately, i is also used as Checksum for the single chunk of a series - // see mkBasicSeriesWithBlooms() in pkg/storage/bloom/v1/test_util.go - key := inputChunkRefs[0].Refs[0].Checksum*1000 + 500 + // Hack to get search string for a specific series + // see MkBasicSeriesWithBlooms() in pkg/storage/bloom/v1/test_util.go + // each series has 1 chunk + // each chunk has multiple strings, from int(fp) to int(nextFp)-1 + x := rand.Intn(len(inputChunkRefs)) + fp := inputChunkRefs[x].Fingerprint + chks := inputChunkRefs[x].Refs + line := fmt.Sprintf("%04x:%04x", int(fp), 0) // first line + + t.Log("x=", x, "fp=", fp, "line=", line) + + expr, err := syntax.ParseExpr(fmt.Sprintf(`{foo="bar"} |= "%s"`, line)) + require.NoError(t, err) req := &logproto.FilterChunkRefRequest{ From: now.Add(-8 * time.Hour), Through: now, Refs: inputChunkRefs, - Filters: []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: fmt.Sprintf("series %d", key)}, - }, + Plan: plan.QueryPlan{AST: expr}, } ctx := user.InjectOrgID(context.Background(), tenantID) res, err := gw.FilterChunkRefs(ctx, req) require.NoError(t, err) + expectedResponse := &logproto.FilterChunkRefResponse{ - ChunkRefs: inputChunkRefs[:1], + ChunkRefs: []*logproto.GroupedChunkRefs{ + { + Fingerprint: fp, + Refs: chks, + Tenant: tenantID, + }, + }, } require.Equal(t, expectedResponse, res) }) @@ -411,6 +405,9 @@ func TestBloomGateway_FilterChunkRefs(t *testing.T) { } func TestBloomGateway_RemoveNotMatchingChunks(t *testing.T) { + g := &Gateway{ + logger: log.NewNopLogger(), + } t.Run("removing chunks partially", func(t *testing.T) { req := &logproto.FilterChunkRefRequest{ Refs: []*logproto.GroupedChunkRefs{ @@ -438,7 +435,8 @@ func TestBloomGateway_RemoveNotMatchingChunks(t *testing.T) { }}, }, } - removeNotMatchingChunks(req, res, log.NewNopLogger()) + n := g.removeNotMatchingChunks(req, res) + require.Equal(t, 2, n) require.Equal(t, expected, req) }) @@ -462,7 +460,8 @@ func TestBloomGateway_RemoveNotMatchingChunks(t *testing.T) { expected := &logproto.FilterChunkRefRequest{ Refs: []*logproto.GroupedChunkRefs{}, } - removeNotMatchingChunks(req, res, log.NewNopLogger()) + n := g.removeNotMatchingChunks(req, res) + require.Equal(t, 3, n) require.Equal(t, expected, req) }) diff --git a/pkg/bloomgateway/cache.go b/pkg/bloomgateway/cache.go index fe40b87e9548..6c573cb47d6d 100644 --- a/pkg/bloomgateway/cache.go +++ b/pkg/bloomgateway/cache.go @@ -182,6 +182,7 @@ func NewBloomGatewayClientCacheMiddleware( }, cacheGen, retentionEnabled, + false, ) return &ClientCache{ diff --git a/pkg/bloomgateway/cache_test.go b/pkg/bloomgateway/cache_test.go index 3ae414cc43c6..bf1a8dbaa365 100644 --- a/pkg/bloomgateway/cache_test.go +++ b/pkg/bloomgateway/cache_test.go @@ -8,13 +8,13 @@ import ( "github.com/go-kit/log" "github.com/grafana/dskit/user" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" "google.golang.org/grpc" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/chunk/cache/resultscache" "github.com/grafana/loki/pkg/util/constants" @@ -382,13 +382,13 @@ func TestCache(t *testing.T) { Through: 3500, }, } + expr, err := syntax.ParseExpr(`{foo="bar"} |= "does not match"`) + require.NoError(t, err) req := &logproto.FilterChunkRefRequest{ From: model.Time(2000), Through: model.Time(3000), Refs: groupRefs(t, chunkRefs), - Filters: []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: "foo"}, - }, + Plan: plan.QueryPlan{AST: expr}, } expectedRes := &logproto.FilterChunkRefResponse{ ChunkRefs: groupRefs(t, chunkRefs), diff --git a/pkg/bloomgateway/client.go b/pkg/bloomgateway/client.go index 6453987b9168..d7328c3c8c31 100644 --- a/pkg/bloomgateway/client.go +++ b/pkg/bloomgateway/client.go @@ -7,7 +7,6 @@ import ( "io" "math" "math/rand" - "sort" "sync" "github.com/go-kit/log" @@ -20,14 +19,15 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" + "golang.org/x/exp/slices" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" "github.com/grafana/loki/pkg/bloomutils" "github.com/grafana/loki/pkg/distributor/clientpool" "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/queue" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/grafana/loki/pkg/storage/chunk/cache" @@ -36,6 +36,10 @@ import ( ) var ( + // BlocksOwnerRead is the operation used to check the authoritative owners of a block + // (replicas included) that are available for queries (a bloom gateway is available for + // queries only when ACTIVE). + BlocksOwnerRead = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, nil) // groupedChunksRefPool pooling slice of logproto.GroupedChunkRefs [64, 128, 256, ..., 65536] groupedChunksRefPool = queue.NewSlicePool[*logproto.GroupedChunkRefs](1<<6, 1<<16, 2) // ringGetBuffersPool pooling for ringGetBuffers to avoid calling ring.MakeBuffersForGet() for each request @@ -138,7 +142,7 @@ func (i *ClientConfig) Validate() error { } type Client interface { - FilterChunks(ctx context.Context, tenant string, from, through model.Time, groups []*logproto.GroupedChunkRefs, filters ...syntax.LineFilter) ([]*logproto.GroupedChunkRefs, error) + FilterChunks(ctx context.Context, tenant string, from, through model.Time, groups []*logproto.GroupedChunkRefs, plan plan.QueryPlan) ([]*logproto.GroupedChunkRefs, error) } type GatewayClient struct { @@ -220,34 +224,35 @@ func shuffleAddrs(addrs []string) []string { } // FilterChunkRefs implements Client -func (c *GatewayClient) FilterChunks(ctx context.Context, tenant string, from, through model.Time, groups []*logproto.GroupedChunkRefs, filters ...syntax.LineFilter) ([]*logproto.GroupedChunkRefs, error) { +func (c *GatewayClient) FilterChunks(ctx context.Context, tenant string, from, through model.Time, groups []*logproto.GroupedChunkRefs, plan plan.QueryPlan) ([]*logproto.GroupedChunkRefs, error) { if !c.limits.BloomGatewayEnabled(tenant) { return groups, nil } subRing := GetShuffleShardingSubring(c.ring, tenant, c.limits) - rs, err := subRing.GetAllHealthy(BlocksRead) + rs, err := subRing.GetAllHealthy(BlocksOwnerRead) if err != nil { return nil, errors.Wrap(err, "bloom gateway get healthy instances") } - streamsByInst, err := c.groupFingerprintsByServer(groups, subRing, rs.Instances) + servers, err := replicationSetsWithBounds(subRing, rs.Instances) if err != nil { - return nil, err + return nil, errors.Wrap(err, "bloom gateway get replication sets") } + servers = partitionByReplicationSet(groups, servers) filteredChunkRefs := groupedChunksRefPool.Get(len(groups)) defer groupedChunksRefPool.Put(filteredChunkRefs) - for _, item := range streamsByInst { + for _, rs := range servers { // randomize order of addresses so we don't hotspot the first server in the list - addrs := shuffleAddrs(item.instance.addrs) + addrs := shuffleAddrs(rs.rs.GetAddresses()) err := c.doForAddrs(addrs, func(client logproto.BloomGatewayClient) error { req := &logproto.FilterChunkRefRequest{ From: from, Through: through, - Refs: item.fingerprints, - Filters: filters, + Refs: rs.groups, + Plan: plan, } resp, err := client.FilterChunkRefs(ctx, req) if err != nil { @@ -286,127 +291,104 @@ func (c *GatewayClient) doForAddrs(addrs []string, fn func(logproto.BloomGateway return err } -func (c *GatewayClient) groupFingerprintsByServer(groups []*logproto.GroupedChunkRefs, subRing ring.ReadRing, instances []ring.InstanceDesc) ([]instanceWithFingerprints, error) { - servers, err := serverAddressesWithTokenRanges(subRing, instances) - if err != nil { - return nil, err - } - boundedFingerprints := partitionFingerprintsByAddresses(groups, servers) - return groupByInstance(boundedFingerprints), nil +func mapTokenRangeToFingerprintRange(r bloomutils.Range[uint32]) v1.FingerprintBounds { + minFp := uint64(r.Min) << 32 + maxFp := uint64(r.Max) << 32 + return v1.NewBounds( + model.Fingerprint(minFp), + model.Fingerprint(maxFp|math.MaxUint32), + ) +} + +type rsWithRanges struct { + rs ring.ReplicationSet + ranges []v1.FingerprintBounds + groups []*logproto.GroupedChunkRefs } -func serverAddressesWithTokenRanges(subRing ring.ReadRing, instances []ring.InstanceDesc) ([]addrsWithTokenRange, error) { +func replicationSetsWithBounds(subRing ring.ReadRing, instances []ring.InstanceDesc) ([]rsWithRanges, error) { bufDescs, bufHosts, bufZones := ring.MakeBuffersForGet() - servers := make([]addrsWithTokenRange, 0, len(instances)) - it := bloomutils.NewInstanceSortMergeIterator(instances) - for it.Next() { - // We can use on of the tokens from the token range - // to obtain all addresses for that token. - rs, err := subRing.Get(it.At().MaxToken, BlocksRead, bufDescs, bufHosts, bufZones) + servers := make([]rsWithRanges, 0, len(instances)) + for _, inst := range instances { + tr, err := bloomutils.TokenRangesForInstance(inst.Id, instances) if err != nil { return nil, errors.Wrap(err, "bloom gateway get ring") } - servers = append(servers, addrsWithTokenRange{ - id: it.At().Instance.Id, - addrs: rs.GetAddresses(), - minToken: it.At().MinToken, - maxToken: it.At().MaxToken, - }) - } - if len(servers) > 0 && servers[len(servers)-1].maxToken < math.MaxUint32 { - // append the instance for the token range between the greates token and MaxUint32 - servers = append(servers, addrsWithTokenRange{ - id: servers[0].id, - addrs: servers[0].addrs, - minToken: servers[len(servers)-1].maxToken + 1, - maxToken: math.MaxUint32, - }) - } - return servers, nil -} - -type instanceWithToken struct { - instance ring.InstanceDesc - token uint32 -} + rs, err := subRing.Get(tr[0], BlocksOwnerRead, bufDescs, bufHosts, bufZones) + if err != nil { + return nil, errors.Wrap(err, "bloom gateway get ring") + } -type addrsWithTokenRange struct { - id string - addrs []string - minToken, maxToken uint32 -} + bounds := make([]v1.FingerprintBounds, 0, len(tr)/2) + for i := 0; i < len(tr); i += 2 { + b := v1.NewBounds( + model.Fingerprint(uint64(tr[i])<<32), + model.Fingerprint(uint64(tr[i+1])<<32|math.MaxUint32), + ) + bounds = append(bounds, b) + } -func (s addrsWithTokenRange) cmp(token uint32) v1.BoundsCheck { - if token < s.minToken { - return v1.Before - } else if token > s.maxToken { - return v1.After + servers = append(servers, rsWithRanges{ + rs: rs, + ranges: bounds, + }) } - return v1.Overlap -} - -type instanceWithFingerprints struct { - instance addrsWithTokenRange - fingerprints []*logproto.GroupedChunkRefs + return servers, nil } -func partitionFingerprintsByAddresses(fingerprints []*logproto.GroupedChunkRefs, addresses []addrsWithTokenRange) (result []instanceWithFingerprints) { - for _, instance := range addresses { - - min := sort.Search(len(fingerprints), func(i int) bool { - return instance.cmp(uint32(fingerprints[i].Fingerprint)) > v1.Before - }) - - max := sort.Search(len(fingerprints), func(i int) bool { - return instance.cmp(uint32(fingerprints[i].Fingerprint)) == v1.After - }) +func partitionByReplicationSet(fingerprints []*logproto.GroupedChunkRefs, rs []rsWithRanges) (result []rsWithRanges) { + for _, inst := range rs { + for _, bounds := range inst.ranges { + min, _ := slices.BinarySearchFunc(fingerprints, bounds, func(g *logproto.GroupedChunkRefs, b v1.FingerprintBounds) int { + if g.Fingerprint < uint64(b.Min) { + return -1 + } else if g.Fingerprint > uint64(b.Min) { + return 1 + } + return 0 + }) + + max, _ := slices.BinarySearchFunc(fingerprints, bounds, func(g *logproto.GroupedChunkRefs, b v1.FingerprintBounds) int { + if g.Fingerprint <= uint64(b.Max) { + return -1 + } else if g.Fingerprint > uint64(b.Max) { + return 1 + } + return 0 + }) + + // fingerprint is out of boundaries + if min == len(fingerprints) || max == 0 { + continue + } - // fingerprint is out of boundaries - if min == len(fingerprints) || max == 0 { - continue + inst.groups = append(inst.groups, fingerprints[min:max]...) } - result = append(result, instanceWithFingerprints{instance: instance, fingerprints: fingerprints[min:max]}) + if len(inst.groups) > 0 { + result = append(result, inst) + } } return result } -// groupByInstance groups fingerprints by server instance -func groupByInstance(boundedFingerprints []instanceWithFingerprints) []instanceWithFingerprints { - if len(boundedFingerprints) == 0 { - return []instanceWithFingerprints{} +// GetShuffleShardingSubring returns the subring to be used for a given user. +// This function should be used both by index gateway servers and clients in +// order to guarantee the same logic is used. +func GetShuffleShardingSubring(ring ring.ReadRing, tenantID string, limits Limits) ring.ReadRing { + shardSize := limits.BloomGatewayShardSize(tenantID) + + // A shard size of 0 means shuffle sharding is disabled for this specific user, + // so we just return the full ring so that indexes will be sharded across all index gateways. + // Since we set the shard size to replication factor if shard size is 0, this + // can only happen if both the shard size and the replication factor are set + // to 0. + if shardSize <= 0 { + return ring } - result := make([]instanceWithFingerprints, 0, len(boundedFingerprints)) - pos := make(map[string]int, len(boundedFingerprints)) - - for _, cur := range boundedFingerprints { - if len(cur.fingerprints) == 0 { - continue - } - // Copy fingerprint slice, otherwise we mutate the original - // TODO(chaudum): Use SlicePool - tmp := make([]*logproto.GroupedChunkRefs, len(cur.fingerprints)) - _ = copy(tmp, cur.fingerprints) - - idx, ok := pos[cur.instance.id] - if ok { - result[idx].fingerprints = append(result[idx].fingerprints, tmp...) - continue - } - - pos[cur.instance.id] = len(result) - result = append(result, instanceWithFingerprints{ - instance: addrsWithTokenRange{ - id: cur.instance.id, - addrs: cur.instance.addrs, - }, - fingerprints: tmp, - }) - } - - return result + return ring.ShuffleShard(tenantID, shardSize) } diff --git a/pkg/bloomgateway/client_test.go b/pkg/bloomgateway/client_test.go index e59fff2306ab..e4b905c37b12 100644 --- a/pkg/bloomgateway/client_test.go +++ b/pkg/bloomgateway/client_test.go @@ -2,8 +2,8 @@ package bloomgateway import ( "context" + "fmt" "math" - "sort" "testing" "time" @@ -16,9 +16,21 @@ import ( "github.com/grafana/loki/pkg/bloomutils" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/plan" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/grafana/loki/pkg/validation" ) +func rs(id int, tokens ...uint32) ring.ReplicationSet { + inst := ring.InstanceDesc{ + Id: fmt.Sprintf("instance-%d", id), + Addr: fmt.Sprintf("10.0.0.%d", id), + Tokens: tokens, + } + return ring.ReplicationSet{Instances: []ring.InstanceDesc{inst}} +} + func TestBloomGatewayClient(t *testing.T) { logger := log.NewNopLogger() reg := prometheus.NewRegistry() @@ -32,316 +44,206 @@ func TestBloomGatewayClient(t *testing.T) { t.Run("FilterChunks returns response", func(t *testing.T) { c, err := NewClient(cfg, &mockRing{}, l, reg, logger, "loki", nil, false) require.NoError(t, err) - res, err := c.FilterChunks(context.Background(), "tenant", model.Now(), model.Now(), nil) + expr, err := syntax.ParseExpr(`{foo="bar"}`) + require.NoError(t, err) + res, err := c.FilterChunks(context.Background(), "tenant", model.Now(), model.Now(), nil, plan.QueryPlan{AST: expr}) require.NoError(t, err) require.Equal(t, []*logproto.GroupedChunkRefs{}, res) }) } -func TestBloomGatewayClient_PartitionFingerprintsByAddresses(t *testing.T) { +func TestBloomGatewayClient_ReplicationSetsWithBounds(t *testing.T) { + testCases := map[string]struct { + instances []ring.InstanceDesc + expected []rsWithRanges + }{ + "single instance covers full range": { + instances: []ring.InstanceDesc{ + {Id: "instance-1", Addr: "10.0.0.1", Tokens: []uint32{(1 << 31)}}, // 0x80000000 + }, + expected: []rsWithRanges{ + {rs: rs(1, (1 << 31)), ranges: []v1.FingerprintBounds{ + v1.NewBounds(0, math.MaxUint64), + }}, + }, + }, + "one token per instance": { + instances: []ring.InstanceDesc{ + {Id: "instance-1", Addr: "10.0.0.1", Tokens: []uint32{(1 << 30) * 1}}, // 0x40000000 + {Id: "instance-2", Addr: "10.0.0.2", Tokens: []uint32{(1 << 30) * 2}}, // 0x80000000 + {Id: "instance-3", Addr: "10.0.0.3", Tokens: []uint32{(1 << 30) * 3}}, // 0xc0000000 + }, + expected: []rsWithRanges{ + {rs: rs(1, (1<<30)*1), ranges: []v1.FingerprintBounds{ + v1.NewBounds(0, 4611686018427387903), + v1.NewBounds(13835058055282163712, 18446744073709551615), + }}, + {rs: rs(2, (1<<30)*2), ranges: []v1.FingerprintBounds{ + v1.NewBounds(4611686018427387904, 9223372036854775807), + }}, + {rs: rs(3, (1<<30)*3), ranges: []v1.FingerprintBounds{ + v1.NewBounds(9223372036854775808, 13835058055282163711), + }}, + }, + }, + "extreme tokens in ring": { + instances: []ring.InstanceDesc{ + {Id: "instance-1", Addr: "10.0.0.1", Tokens: []uint32{0}}, + {Id: "instance-2", Addr: "10.0.0.2", Tokens: []uint32{math.MaxUint32}}, + }, + expected: []rsWithRanges{ + {rs: rs(1, 0), ranges: []v1.FingerprintBounds{ + v1.NewBounds(math.MaxUint64-math.MaxUint32, math.MaxUint64), + }}, + {rs: rs(2, math.MaxUint32), ranges: []v1.FingerprintBounds{ + v1.NewBounds(0, math.MaxUint64-math.MaxUint32-1), + }}, + }, + }, + } + + for name, tc := range testCases { + tc := tc + t.Run(name, func(t *testing.T) { + subRing := newMockRing(t, tc.instances) + res, err := replicationSetsWithBounds(subRing, tc.instances) + require.NoError(t, err) + require.Equal(t, tc.expected, res) + }) + } +} + +func TestBloomGatewayClient_PartitionByReplicationSet(t *testing.T) { + // Create 10 fingerprints [0, 2, 4, ... 18] + groups := make([]*logproto.GroupedChunkRefs, 0, 10) + for i := 0; i < 20; i += 2 { + groups = append(groups, &logproto.GroupedChunkRefs{Fingerprint: uint64(i)}) + } + // instance token ranges do not overlap t.Run("non-overlapping", func(t *testing.T) { - groups := []*logproto.GroupedChunkRefs{ - {Fingerprint: 0}, - {Fingerprint: 100}, - {Fingerprint: 101}, - {Fingerprint: 200}, - {Fingerprint: 201}, - {Fingerprint: 300}, - {Fingerprint: 301}, - {Fingerprint: 400}, - {Fingerprint: 401}, // out of bounds, will be dismissed - } - servers := []addrsWithTokenRange{ - {id: "instance-1", addrs: []string{"10.0.0.1"}, minToken: 0, maxToken: 100}, - {id: "instance-2", addrs: []string{"10.0.0.2"}, minToken: 101, maxToken: 200}, - {id: "instance-3", addrs: []string{"10.0.0.3"}, minToken: 201, maxToken: 300}, - {id: "instance-2", addrs: []string{"10.0.0.2"}, minToken: 301, maxToken: 400}, + + servers := []rsWithRanges{ + {rs: rs(1), ranges: []v1.FingerprintBounds{v1.NewBounds(0, 4)}}, + {rs: rs(2), ranges: []v1.FingerprintBounds{v1.NewBounds(5, 9), v1.NewBounds(15, 19)}}, + {rs: rs(3), ranges: []v1.FingerprintBounds{v1.NewBounds(10, 14)}}, } // partition fingerprints - expected := []instanceWithFingerprints{ - { - instance: servers[0], - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 0}, - {Fingerprint: 100}, - }, - }, + expected := [][]*logproto.GroupedChunkRefs{ { - instance: servers[1], - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 101}, - {Fingerprint: 200}, - }, + {Fingerprint: 0}, + {Fingerprint: 2}, + {Fingerprint: 4}, }, { - instance: servers[2], - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 201}, - {Fingerprint: 300}, - }, + {Fingerprint: 6}, + {Fingerprint: 8}, + {Fingerprint: 16}, + {Fingerprint: 18}, }, { - instance: servers[3], - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 301}, - {Fingerprint: 400}, - }, + {Fingerprint: 10}, + {Fingerprint: 12}, + {Fingerprint: 14}, }, } - bounded := partitionFingerprintsByAddresses(groups, servers) - require.Equal(t, expected, bounded) + partitioned := partitionByReplicationSet(groups, servers) + for i := range partitioned { + require.Equal(t, expected[i], partitioned[i].groups) + } + }) + + // instance token ranges overlap -- this should not happen in a real ring, though + t.Run("overlapping", func(t *testing.T) { + servers := []rsWithRanges{ + {rs: rs(1), ranges: []v1.FingerprintBounds{v1.NewBounds(0, 9)}}, + {rs: rs(2), ranges: []v1.FingerprintBounds{v1.NewBounds(5, 14)}}, + {rs: rs(3), ranges: []v1.FingerprintBounds{v1.NewBounds(10, 19)}}, + } - // group fingerprints by instance + // partition fingerprints - expected = []instanceWithFingerprints{ + expected := [][]*logproto.GroupedChunkRefs{ { - instance: addrsWithTokenRange{id: "instance-1", addrs: []string{"10.0.0.1"}}, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 0}, - {Fingerprint: 100}, - }, + {Fingerprint: 0}, + {Fingerprint: 2}, + {Fingerprint: 4}, + {Fingerprint: 6}, + {Fingerprint: 8}, }, { - instance: addrsWithTokenRange{id: "instance-2", addrs: []string{"10.0.0.2"}}, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 101}, - {Fingerprint: 200}, - {Fingerprint: 301}, - {Fingerprint: 400}, - }, + {Fingerprint: 6}, + {Fingerprint: 8}, + {Fingerprint: 10}, + {Fingerprint: 12}, + {Fingerprint: 14}, }, { - instance: addrsWithTokenRange{id: "instance-3", addrs: []string{"10.0.0.3"}}, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 201}, - {Fingerprint: 300}, - }, + {Fingerprint: 10}, + {Fingerprint: 12}, + {Fingerprint: 14}, + {Fingerprint: 16}, + {Fingerprint: 18}, }, } - result := groupByInstance(bounded) - require.Equal(t, expected, result) - }) - // instance token ranges overlap - t.Run("overlapping", func(t *testing.T) { - groups := []*logproto.GroupedChunkRefs{ - {Fingerprint: 50}, - {Fingerprint: 150}, - {Fingerprint: 250}, - {Fingerprint: 350}, - } - servers := []addrsWithTokenRange{ - {id: "instance-1", addrs: []string{"10.0.0.1"}, minToken: 0, maxToken: 200}, - {id: "instance-2", addrs: []string{"10.0.0.2"}, minToken: 100, maxToken: 300}, - {id: "instance-3", addrs: []string{"10.0.0.3"}, minToken: 200, maxToken: 400}, + partitioned := partitionByReplicationSet(groups, servers) + for i := range partitioned { + require.Equal(t, expected[i], partitioned[i].groups) } - - // partition fingerprints - - expected := []instanceWithFingerprints{ - {instance: servers[0], fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 50}, - {Fingerprint: 150}, - }}, - {instance: servers[1], fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 150}, - {Fingerprint: 250}, - }}, - {instance: servers[2], fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 250}, - {Fingerprint: 350}, - }}, - } - - bounded := partitionFingerprintsByAddresses(groups, servers) - require.Equal(t, expected, bounded) }) } -func TestBloomGatewayClient_ServerAddressesWithTokenRanges(t *testing.T) { - testCases := map[string]struct { - instances []ring.InstanceDesc - expected []addrsWithTokenRange - }{ - "one token per instance": { - instances: []ring.InstanceDesc{ - {Id: "instance-1", Addr: "10.0.0.1", Tokens: []uint32{math.MaxUint32 / 6 * 1}}, - {Id: "instance-2", Addr: "10.0.0.2", Tokens: []uint32{math.MaxUint32 / 6 * 3}}, - {Id: "instance-3", Addr: "10.0.0.3", Tokens: []uint32{math.MaxUint32 / 6 * 5}}, - }, - expected: []addrsWithTokenRange{ - {id: "instance-1", addrs: []string{"10.0.0.1"}, minToken: 0, maxToken: math.MaxUint32 / 6 * 1}, - {id: "instance-2", addrs: []string{"10.0.0.2"}, minToken: math.MaxUint32/6*1 + 1, maxToken: math.MaxUint32 / 6 * 3}, - {id: "instance-3", addrs: []string{"10.0.0.3"}, minToken: math.MaxUint32/6*3 + 1, maxToken: math.MaxUint32 / 6 * 5}, - {id: "instance-1", addrs: []string{"10.0.0.1"}, minToken: math.MaxUint32/6*5 + 1, maxToken: math.MaxUint32}, - }, - }, - "MinUint32 and MaxUint32 are tokens in the ring": { - instances: []ring.InstanceDesc{ - {Id: "instance-1", Addr: "10.0.0.1", Tokens: []uint32{0, math.MaxUint32 / 3 * 2}}, - {Id: "instance-2", Addr: "10.0.0.2", Tokens: []uint32{math.MaxUint32 / 3 * 1, math.MaxUint32}}, - }, - expected: []addrsWithTokenRange{ - {id: "instance-1", addrs: []string{"10.0.0.1"}, minToken: 0, maxToken: 0}, - {id: "instance-2", addrs: []string{"10.0.0.2"}, minToken: 1, maxToken: math.MaxUint32 / 3}, - {id: "instance-1", addrs: []string{"10.0.0.1"}, minToken: math.MaxUint32/3*1 + 1, maxToken: math.MaxUint32 / 3 * 2}, - {id: "instance-2", addrs: []string{"10.0.0.2"}, minToken: math.MaxUint32/3*2 + 1, maxToken: math.MaxUint32}, - }, - }, - } +func BenchmarkPartitionFingerprintsByAddresses(b *testing.B) { + numFp := 100000 + fpStep := math.MaxUint64 / uint64(numFp) - for name, tc := range testCases { - tc := tc - t.Run(name, func(t *testing.T) { - subRing := newMockRing(tc.instances) - res, err := serverAddressesWithTokenRanges(subRing, tc.instances) - require.NoError(t, err) - require.Equal(t, tc.expected, res) - }) + groups := make([]*logproto.GroupedChunkRefs, 0, numFp) + for i := uint64(0); i < math.MaxUint64-fpStep; i += fpStep { + groups = append(groups, &logproto.GroupedChunkRefs{Fingerprint: i}) } -} - -func TestBloomGatewayClient_GroupFingerprintsByServer(t *testing.T) { - - logger := log.NewNopLogger() - reg := prometheus.NewRegistry() - - l, err := validation.NewOverrides(validation.Limits{BloomGatewayShardSize: 1}, nil) - require.NoError(t, err) - - cfg := ClientConfig{} - flagext.DefaultValues(&cfg) - - c, err := NewClient(cfg, nil, l, reg, logger, "loki", nil, false) - require.NoError(t, err) - - instances := []ring.InstanceDesc{ - {Id: "instance-1", Addr: "10.0.0.1", Tokens: []uint32{2146405214, 1029997044, 678878693}}, - {Id: "instance-2", Addr: "10.0.0.2", Tokens: []uint32{296463531, 1697323986, 800258284}}, - {Id: "instance-3", Addr: "10.0.0.3", Tokens: []uint32{2014002871, 315617625, 1036168527}}, + numServers := 100 + tokenStep := math.MaxUint32 / uint32(numServers) + servers := make([]rsWithRanges, 0, numServers) + for i := uint32(0); i < math.MaxUint32-tokenStep; i += tokenStep { + servers = append(servers, rsWithRanges{ + rs: rs(int(i)), + ranges: []v1.FingerprintBounds{ + v1.NewBounds(model.Fingerprint(i)<<32, model.Fingerprint(i+tokenStep)<<32), + }, + }) } - it := bloomutils.NewInstanceSortMergeIterator(instances) - for it.Next() { - t.Log(it.At().MaxToken, it.At().Instance.Addr) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = partitionByReplicationSet(groups, servers) } +} - testCases := []struct { - name string - chunks []*logproto.GroupedChunkRefs - expected []instanceWithFingerprints +func TestBloomGatewayClient_MapTokenRangeToFingerprintRange(t *testing.T) { + testCases := map[string]struct { + lshift int + inp bloomutils.Range[uint32] + exp v1.FingerprintBounds }{ - { - name: "empty input yields empty result", - chunks: []*logproto.GroupedChunkRefs{}, - expected: []instanceWithFingerprints{}, - }, - { - name: "fingerprints within a single token range are grouped", - chunks: []*logproto.GroupedChunkRefs{ - {Fingerprint: 1000000000, Refs: []*logproto.ShortRef{{Checksum: 1}}}, - {Fingerprint: 1000000001, Refs: []*logproto.ShortRef{{Checksum: 2}}}, - }, - expected: []instanceWithFingerprints{ - { - instance: addrsWithTokenRange{ - id: "instance-1", - addrs: []string{"10.0.0.1"}, - }, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 1000000000, Refs: []*logproto.ShortRef{{Checksum: 1}}}, - {Fingerprint: 1000000001, Refs: []*logproto.ShortRef{{Checksum: 2}}}, - }, - }, - }, - }, - { - name: "fingerprints within multiple token ranges of a single instance are grouped", - chunks: []*logproto.GroupedChunkRefs{ - {Fingerprint: 1000000000, Refs: []*logproto.ShortRef{{Checksum: 1}}}, - {Fingerprint: 2100000000, Refs: []*logproto.ShortRef{{Checksum: 2}}}, - }, - expected: []instanceWithFingerprints{ - { - instance: addrsWithTokenRange{ - id: "instance-1", - addrs: []string{"10.0.0.1"}, - }, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 1000000000, Refs: []*logproto.ShortRef{{Checksum: 1}}}, - {Fingerprint: 2100000000, Refs: []*logproto.ShortRef{{Checksum: 2}}}, - }, - }, - }, + "single token expands to multiple fingerprints": { + inp: bloomutils.NewTokenRange(0, 0), + exp: v1.NewBounds(0, 0xffffffff), }, - { - name: "fingerprints with token ranges of multiple instances are grouped", - chunks: []*logproto.GroupedChunkRefs{ - // instance 1 - {Fingerprint: 1000000000, Refs: []*logproto.ShortRef{{Checksum: 1}}}, - // instance 1 - {Fingerprint: 2100000000, Refs: []*logproto.ShortRef{{Checksum: 2}}}, - // instance 2 - {Fingerprint: 290000000, Refs: []*logproto.ShortRef{{Checksum: 3}}}, - // instance 2 (fingerprint equals instance token) - {Fingerprint: 800258284, Refs: []*logproto.ShortRef{{Checksum: 4}}}, - // instance 2 (fingerprint greater than greatest token) - {Fingerprint: 2147483648, Refs: []*logproto.ShortRef{{Checksum: 5}}}, - // instance 3 - {Fingerprint: 1029997045, Refs: []*logproto.ShortRef{{Checksum: 6}}}, - }, - expected: []instanceWithFingerprints{ - { - instance: addrsWithTokenRange{ - id: "instance-2", - addrs: []string{"10.0.0.2"}, - }, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 290000000, Refs: []*logproto.ShortRef{{Checksum: 3}}}, - {Fingerprint: 800258284, Refs: []*logproto.ShortRef{{Checksum: 4}}}, - {Fingerprint: 2147483648, Refs: []*logproto.ShortRef{{Checksum: 5}}}, - }, - }, - { - instance: addrsWithTokenRange{ - id: "instance-1", - addrs: []string{"10.0.0.1"}, - }, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 1000000000, Refs: []*logproto.ShortRef{{Checksum: 1}}}, - {Fingerprint: 2100000000, Refs: []*logproto.ShortRef{{Checksum: 2}}}, - }, - }, - { - instance: addrsWithTokenRange{ - id: "instance-3", - addrs: []string{"10.0.0.3"}, - }, - fingerprints: []*logproto.GroupedChunkRefs{ - {Fingerprint: 1029997045, Refs: []*logproto.ShortRef{{Checksum: 6}}}, - }, - }, - }, + "max value expands to max value of new range": { + inp: bloomutils.NewTokenRange((1 << 31), math.MaxUint32), + exp: v1.NewBounds((1 << 63), 0xffffffffffffffff), }, } - - subRing := newMockRing(instances) - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - // sort chunks here, to be able to write more human readable test input - sort.Slice(tc.chunks, func(i, j int) bool { - return tc.chunks[i].Fingerprint < tc.chunks[j].Fingerprint - }) - - res, err := c.groupFingerprintsByServer(tc.chunks, subRing, instances) - require.NoError(t, err) - require.Equal(t, tc.expected, res) + for desc, tc := range testCases { + t.Run(desc, func(t *testing.T) { + actual := mapTokenRangeToFingerprintRange(tc.inp) + require.Equal(t, tc.exp, actual) }) } } @@ -349,11 +251,14 @@ func TestBloomGatewayClient_GroupFingerprintsByServer(t *testing.T) { // make sure mockRing implements the ring.ReadRing interface var _ ring.ReadRing = &mockRing{} -func newMockRing(instances []ring.InstanceDesc) *mockRing { - it := bloomutils.NewInstanceSortMergeIterator(instances) - ranges := make([]bloomutils.InstanceWithTokenRange, 0) - for it.Next() { - ranges = append(ranges, it.At()) +func newMockRing(t *testing.T, instances []ring.InstanceDesc) *mockRing { + ranges := make([]ring.TokenRanges, 0) + for i := range instances { + tr, err := bloomutils.TokenRangesForInstance(instances[i].Id, instances) + if err != nil { + t.Fatal(err) + } + ranges = append(ranges, tr) } return &mockRing{ instances: instances, @@ -363,21 +268,17 @@ func newMockRing(instances []ring.InstanceDesc) *mockRing { type mockRing struct { instances []ring.InstanceDesc - ranges []bloomutils.InstanceWithTokenRange + ranges []ring.TokenRanges } // Get implements ring.ReadRing. -func (r *mockRing) Get(key uint32, _ ring.Operation, _ []ring.InstanceDesc, _ []string, _ []string) (ring.ReplicationSet, error) { - idx, _ := sort.Find(len(r.ranges), func(i int) int { - if r.ranges[i].MaxToken < key { - return 1 +func (r *mockRing) Get(key uint32, _ ring.Operation, _ []ring.InstanceDesc, _ []string, _ []string) (rs ring.ReplicationSet, err error) { + for i := range r.ranges { + if r.ranges[i].IncludesKey(key) { + rs.Instances = append(rs.Instances, r.instances[i]) } - if r.ranges[i].MaxToken > key { - return -1 - } - return 0 - }) - return ring.ReplicationSet{Instances: []ring.InstanceDesc{r.ranges[idx].Instance}}, nil + } + return } // GetAllHealthy implements ring.ReadRing. @@ -427,7 +328,6 @@ func (*mockRing) CleanupShuffleShardCache(_ string) { panic("unimplemented") } -func (r *mockRing) GetTokenRangesForInstance(_ string) (ring.TokenRanges, error) { - tr := ring.TokenRanges{0, math.MaxUint32} - return tr, nil +func (r *mockRing) GetTokenRangesForInstance(id string) (ring.TokenRanges, error) { + return bloomutils.TokenRangesForInstance(id, r.instances) } diff --git a/pkg/bloomgateway/multiplexing.go b/pkg/bloomgateway/multiplexing.go index 97c257194809..8486f6e6e7cf 100644 --- a/pkg/bloomgateway/multiplexing.go +++ b/pkg/bloomgateway/multiplexing.go @@ -12,6 +12,8 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/config" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" ) const ( @@ -56,26 +58,26 @@ type Task struct { // the last error of the task // needs to be a pointer so multiple copies of the task can modify its value err *wrappedError - // the respones received from the block queriers + // the responses received from the block queriers responses []v1.Output // series of the original request series []*logproto.GroupedChunkRefs // filters of the original request - filters []syntax.LineFilter + filters []syntax.LineFilterExpr // from..through date of the task's chunks - bounds model.Interval + interval bloomshipper.Interval // the context from the request ctx context.Context // TODO(chaudum): Investigate how to remove that. - day model.Time + table config.DayTime } // NewTask returns a new Task that can be enqueued to the task queue. // In addition, it returns a result and an error channel, as well // as an error if the instantiation fails. -func NewTask(ctx context.Context, tenantID string, refs seriesWithBounds, filters []syntax.LineFilter) (Task, error) { +func NewTask(ctx context.Context, tenantID string, refs seriesWithInterval, filters []syntax.LineFilterExpr) (Task, error) { key, err := ulid.New(ulid.Now(), entropy) if err != nil { return Task{}, err @@ -88,8 +90,8 @@ func NewTask(ctx context.Context, tenantID string, refs seriesWithBounds, filter resCh: make(chan v1.Output), filters: filters, series: refs.series, - bounds: refs.bounds, - day: refs.day, + interval: refs.interval, + table: refs.day, ctx: ctx, done: make(chan struct{}), responses: make([]v1.Output, 0, len(refs.series)), @@ -97,8 +99,10 @@ func NewTask(ctx context.Context, tenantID string, refs seriesWithBounds, filter return task, nil } +// Bounds implements Bounded +// see pkg/storage/stores/shipper/indexshipper/tsdb.Bounded func (t Task) Bounds() (model.Time, model.Time) { - return t.bounds.Start, t.bounds.End + return t.interval.Start, t.interval.End } func (t Task) Done() <-chan struct{} { @@ -128,8 +132,8 @@ func (t Task) Copy(series []*logproto.GroupedChunkRefs) Task { resCh: t.resCh, filters: t.filters, series: series, - bounds: t.bounds, - day: t.day, + interval: t.interval, + table: t.table, ctx: t.ctx, done: make(chan struct{}), responses: make([]v1.Output, 0, len(series)), @@ -138,20 +142,20 @@ func (t Task) Copy(series []*logproto.GroupedChunkRefs) Task { func (t Task) RequestIter(tokenizer *v1.NGramTokenizer) v1.Iterator[v1.Request] { return &requestIterator{ - series: v1.NewSliceIter(t.series), - searches: convertToSearches(t.filters, tokenizer), - channel: t.resCh, - curr: v1.Request{}, + series: v1.NewSliceIter(t.series), + search: v1.FiltersToBloomTest(tokenizer, t.filters...), + channel: t.resCh, + curr: v1.Request{}, } } var _ v1.Iterator[v1.Request] = &requestIterator{} type requestIterator struct { - series v1.Iterator[*logproto.GroupedChunkRefs] - searches [][]byte - channel chan<- v1.Output - curr v1.Request + series v1.Iterator[*logproto.GroupedChunkRefs] + search v1.BloomTest + channel chan<- v1.Output + curr v1.Request } // At implements v1.Iterator. @@ -174,7 +178,7 @@ func (it *requestIterator) Next() bool { it.curr = v1.Request{ Fp: model.Fingerprint(group.Fingerprint), Chks: convertToChunkRefs(group.Refs), - Searches: it.searches, + Search: it.search, Response: it.channel, } return true diff --git a/pkg/bloomgateway/multiplexing_test.go b/pkg/bloomgateway/multiplexing_test.go index 009c825a7e84..af79f37b358b 100644 --- a/pkg/bloomgateway/multiplexing_test.go +++ b/pkg/bloomgateway/multiplexing_test.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" ) func TestTask(t *testing.T) { @@ -58,11 +59,11 @@ func TestTask_RequestIterator(t *testing.T) { tokenizer := v1.NewNGramTokenizer(4, 0) t.Run("empty request yields empty iterator", func(t *testing.T) { - swb := seriesWithBounds{ - bounds: model.Interval{Start: 0, End: math.MaxInt64}, - series: []*logproto.GroupedChunkRefs{}, + swb := seriesWithInterval{ + interval: bloomshipper.Interval{Start: 0, End: math.MaxInt64}, + series: []*logproto.GroupedChunkRefs{}, } - task, _ := NewTask(context.Background(), tenant, swb, []syntax.LineFilter{}) + task, _ := NewTask(context.Background(), tenant, swb, []syntax.LineFilterExpr{}) it := task.RequestIter(tokenizer) // nothing to iterate over require.False(t, it.Next()) diff --git a/pkg/bloomgateway/processor.go b/pkg/bloomgateway/processor.go index 460ac3f44e03..687d60dedd13 100644 --- a/pkg/bloomgateway/processor.go +++ b/pkg/bloomgateway/processor.go @@ -3,42 +3,39 @@ package bloomgateway import ( "context" "math" - "sort" + "time" "github.com/go-kit/log" - "github.com/prometheus/common/model" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" ) -type tasksForBlock struct { - blockRef bloomshipper.BlockRef - tasks []Task -} - -type blockLoader interface { - LoadBlocks(context.Context, []bloomshipper.BlockRef) (v1.Iterator[bloomshipper.BlockQuerierWithFingerprintRange], error) -} - -type store interface { - blockLoader - bloomshipper.Store +func newProcessor(id string, store bloomshipper.Store, logger log.Logger, metrics *workerMetrics) *processor { + return &processor{ + id: id, + store: store, + logger: logger, + metrics: metrics, + } } type processor struct { - store store - logger log.Logger + id string + store bloomshipper.Store + logger log.Logger + metrics *workerMetrics } func (p *processor) run(ctx context.Context, tasks []Task) error { - for ts, tasks := range group(tasks, func(t Task) model.Time { return t.day }) { - interval := bloomshipper.Interval{ - Start: ts, - End: ts.Add(Day), - } + return p.runWithBounds(ctx, tasks, v1.MultiFingerprintBounds{{Min: 0, Max: math.MaxUint64}}) +} + +func (p *processor) runWithBounds(ctx context.Context, tasks []Task, bounds v1.MultiFingerprintBounds) error { + for ts, tasks := range group(tasks, func(t Task) config.DayTime { return t.table }) { tenant := tasks[0].Tenant - err := p.processTasks(ctx, tenant, interval, []bloomshipper.Keyspace{{Min: 0, Max: math.MaxUint64}}, tasks) + err := p.processTasks(ctx, tenant, ts, bounds, tasks) if err != nil { for _, task := range tasks { task.CloseWithError(err) @@ -52,38 +49,45 @@ func (p *processor) run(ctx context.Context, tasks []Task) error { return nil } -func (p *processor) processTasks(ctx context.Context, tenant string, interval bloomshipper.Interval, keyspaces []bloomshipper.Keyspace, tasks []Task) error { +func (p *processor) processTasks(ctx context.Context, tenant string, day config.DayTime, keyspaces v1.MultiFingerprintBounds, tasks []Task) error { minFpRange, maxFpRange := getFirstLast(keyspaces) + interval := bloomshipper.NewInterval(day.Bounds()) metaSearch := bloomshipper.MetaSearchParams{ TenantID: tenant, Interval: interval, - Keyspace: bloomshipper.Keyspace{Min: minFpRange.Min, Max: maxFpRange.Max}, + Keyspace: v1.NewBounds(minFpRange.Min, maxFpRange.Max), } metas, err := p.store.FetchMetas(ctx, metaSearch) if err != nil { return err } + p.metrics.metasFetched.WithLabelValues(p.id).Observe(float64(len(metas))) + blocksRefs := bloomshipper.BlocksForMetas(metas, interval, keyspaces) - return p.processBlocks(ctx, partition(tasks, blocksRefs)) + return p.processBlocks(ctx, partitionTasks(tasks, blocksRefs)) } -func (p *processor) processBlocks(ctx context.Context, data []tasksForBlock) error { +func (p *processor) processBlocks(ctx context.Context, data []blockWithTasks) error { refs := make([]bloomshipper.BlockRef, len(data)) for _, block := range data { - refs = append(refs, block.blockRef) + refs = append(refs, block.ref) } - blockIter, err := p.store.LoadBlocks(ctx, refs) + bqs, err := p.store.FetchBlocks(ctx, refs) if err != nil { return err } + p.metrics.blocksFetched.WithLabelValues(p.id).Observe(float64(len(bqs))) + + blockIter := v1.NewSliceIter(bqs) outer: for blockIter.Next() { bq := blockIter.At() for i, block := range data { - if block.blockRef.MinFingerprint == uint64(bq.MinFp) && block.blockRef.MaxFingerprint == uint64(bq.MaxFp) { + if block.ref.Bounds.Equal(bq.Bounds) { err := p.processBlock(ctx, bq.BlockQuerier, block.tasks) + bq.Close() if err != nil { return err } @@ -91,6 +95,8 @@ outer: continue outer } } + // should not happen, but close anyway + bq.Close() } return nil } @@ -109,7 +115,16 @@ func (p *processor) processBlock(_ context.Context, blockQuerier *v1.BlockQuerie } fq := blockQuerier.Fuse(iters) - return fq.Run() + + start := time.Now() + err = fq.Run() + if err != nil { + p.metrics.blockQueryLatency.WithLabelValues(p.id, labelFailure).Observe(time.Since(start).Seconds()) + } else { + p.metrics.blockQueryLatency.WithLabelValues(p.id, labelSuccess).Observe(time.Since(start).Seconds()) + } + + return err } // getFirstLast returns the first and last item of a fingerprint slice @@ -129,37 +144,3 @@ func group[K comparable, V any, S ~[]V](s S, f func(v V) K) map[K]S { } return m } - -func partition(tasks []Task, blocks []bloomshipper.BlockRef) []tasksForBlock { - result := make([]tasksForBlock, 0, len(blocks)) - - for _, block := range blocks { - bounded := tasksForBlock{ - blockRef: block, - } - - for _, task := range tasks { - refs := task.series - min := sort.Search(len(refs), func(i int) bool { - return block.Cmp(refs[i].Fingerprint) > v1.Before - }) - - max := sort.Search(len(refs), func(i int) bool { - return block.Cmp(refs[i].Fingerprint) == v1.After - }) - - // All fingerprints fall outside of the consumer's range - if min == len(refs) || max == 0 { - continue - } - - bounded.tasks = append(bounded.tasks, task.Copy(refs[min:max])) - } - - if len(bounded.tasks) > 0 { - result = append(result, bounded) - } - - } - return result -} diff --git a/pkg/bloomgateway/processor_test.go b/pkg/bloomgateway/processor_test.go index 62c6d42ae18b..0c586897064b 100644 --- a/pkg/bloomgateway/processor_test.go +++ b/pkg/bloomgateway/processor_test.go @@ -7,24 +7,41 @@ import ( "testing" "time" + "github.com/go-kit/log" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "go.uber.org/atomic" "github.com/grafana/loki/pkg/logql/syntax" - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/pkg/util/constants" ) -var _ store = &dummyStore{} +var _ bloomshipper.Store = &dummyStore{} + +func newMockBloomStore(bqs []*bloomshipper.CloseableBlockQuerier, metas []bloomshipper.Meta) *dummyStore { + return &dummyStore{ + querieres: bqs, + metas: metas, + } +} type dummyStore struct { metas []bloomshipper.Meta - blocks []bloomshipper.BlockRef - querieres []bloomshipper.BlockQuerierWithFingerprintRange + querieres []*bloomshipper.CloseableBlockQuerier + + // mock how long it takes to serve block queriers + delay time.Duration + // mock response error when serving block queriers in ForEach + err error } func (s *dummyStore) ResolveMetas(_ context.Context, _ bloomshipper.MetaSearchParams) ([][]bloomshipper.MetaRef, []*bloomshipper.Fetcher, error) { + time.Sleep(s.delay) + //TODO(chaudum) Filter metas based on search params refs := make([]bloomshipper.MetaRef, 0, len(s.metas)) for _, meta := range s.metas { @@ -38,19 +55,28 @@ func (s *dummyStore) FetchMetas(_ context.Context, _ bloomshipper.MetaSearchPara return s.metas, nil } -func (s *dummyStore) Fetcher(_ model.Time) *bloomshipper.Fetcher { - return nil +func (s *dummyStore) Fetcher(_ model.Time) (*bloomshipper.Fetcher, error) { + return nil, nil +} + +func (s *dummyStore) Client(_ model.Time) (bloomshipper.Client, error) { + return nil, nil } func (s *dummyStore) Stop() { } -func (s *dummyStore) LoadBlocks(_ context.Context, refs []bloomshipper.BlockRef) (v1.Iterator[bloomshipper.BlockQuerierWithFingerprintRange], error) { - result := make([]bloomshipper.BlockQuerierWithFingerprintRange, len(s.querieres)) +func (s *dummyStore) FetchBlocks(_ context.Context, refs []bloomshipper.BlockRef) ([]*bloomshipper.CloseableBlockQuerier, error) { + result := make([]*bloomshipper.CloseableBlockQuerier, 0, len(s.querieres)) + + if s.err != nil { + time.Sleep(s.delay) + return result, s.err + } for _, ref := range refs { for _, bq := range s.querieres { - if ref.MinFingerprint == uint64(bq.MinFp) && ref.MaxFingerprint == uint64(bq.MaxFp) { + if ref.Bounds.Equal(bq.Bounds) { result = append(result, bq) } } @@ -60,35 +86,39 @@ func (s *dummyStore) LoadBlocks(_ context.Context, refs []bloomshipper.BlockRef) result[i], result[j] = result[j], result[i] }) - return v1.NewSliceIter(result), nil + time.Sleep(s.delay) + + return result, nil } func TestProcessor(t *testing.T) { ctx := context.Background() tenant := "fake" now := mktime("2024-01-27 12:00") + metrics := newWorkerMetrics(prometheus.NewPedanticRegistry(), constants.Loki, "bloom_gatway") - t.Run("dummy", func(t *testing.T) { - blocks, metas, queriers, data := createBlocks(t, tenant, 10, now.Add(-1*time.Hour), now, 0x0000, 0x1000) - p := &processor{ - store: &dummyStore{ - querieres: queriers, - metas: metas, - blocks: blocks, - }, - } + t.Run("success case", func(t *testing.T) { + _, metas, queriers, data := createBlocks(t, tenant, 10, now.Add(-1*time.Hour), now, 0x0000, 0x0fff) + + mockStore := newMockBloomStore(queriers, metas) + p := newProcessor("worker", mockStore, log.NewNopLogger(), metrics) chunkRefs := createQueryInputFromBlockData(t, tenant, data, 10) - swb := seriesWithBounds{ + swb := seriesWithInterval{ series: groupRefs(t, chunkRefs), - bounds: model.Interval{ + interval: bloomshipper.Interval{ Start: now.Add(-1 * time.Hour), End: now, }, - day: truncateDay(now), + day: config.NewDayTime(truncateDay(now)), } - filters := []syntax.LineFilter{ - {Ty: 0, Match: "no match"}, + filters := []syntax.LineFilterExpr{ + { + LineFilter: syntax.LineFilter{ + Ty: 0, + Match: "no match", + }, + }, } t.Log("series", len(swb.series)) @@ -113,4 +143,53 @@ func TestProcessor(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(len(swb.series)), results.Load()) }) + + t.Run("failure case", func(t *testing.T) { + _, metas, queriers, data := createBlocks(t, tenant, 10, now.Add(-1*time.Hour), now, 0x0000, 0x0fff) + + mockStore := newMockBloomStore(queriers, metas) + mockStore.err = errors.New("store failed") + + p := newProcessor("worker", mockStore, log.NewNopLogger(), metrics) + + chunkRefs := createQueryInputFromBlockData(t, tenant, data, 10) + swb := seriesWithInterval{ + series: groupRefs(t, chunkRefs), + interval: bloomshipper.Interval{ + Start: now.Add(-1 * time.Hour), + End: now, + }, + day: config.NewDayTime(truncateDay(now)), + } + filters := []syntax.LineFilterExpr{ + { + LineFilter: syntax.LineFilter{ + Ty: 0, + Match: "no match", + }, + }, + } + + t.Log("series", len(swb.series)) + task, _ := NewTask(ctx, "fake", swb, filters) + tasks := []Task{task} + + results := atomic.NewInt64(0) + var wg sync.WaitGroup + for i := range tasks { + wg.Add(1) + go func(ta Task) { + defer wg.Done() + for range ta.resCh { + results.Inc() + } + t.Log("done", results.Load()) + }(tasks[i]) + } + + err := p.run(ctx, tasks) + wg.Wait() + require.Errorf(t, err, "store failed") + require.Equal(t, int64(0), results.Load()) + }) } diff --git a/pkg/bloomgateway/querier.go b/pkg/bloomgateway/querier.go index 4b2366e83f28..171936d9e39c 100644 --- a/pkg/bloomgateway/querier.go +++ b/pkg/bloomgateway/querier.go @@ -5,30 +5,75 @@ import ( "sort" "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/plan" + "github.com/grafana/loki/pkg/util/constants" ) +type querierMetrics struct { + chunksTotal prometheus.Counter + chunksFiltered prometheus.Counter + seriesTotal prometheus.Counter + seriesFiltered prometheus.Counter +} + +func newQuerierMetrics(registerer prometheus.Registerer, namespace, subsystem string) *querierMetrics { + return &querierMetrics{ + chunksTotal: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "chunks_total", + Help: "Total amount of chunks pre filtering. Does not count chunks in failed requests.", + }), + chunksFiltered: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "chunks_filtered_total", + Help: "Total amount of chunks that have been filtered out. Does not count chunks in failed requests.", + }), + seriesTotal: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "series_total", + Help: "Total amount of series pre filtering. Does not count series in failed requests.", + }), + seriesFiltered: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "series_filtered_total", + Help: "Total amount of series that have been filtered out. Does not count series in failed requests.", + }), + } +} + // BloomQuerier is a store-level abstraction on top of Client // It is used by the index gateway to filter ChunkRefs based on given line fiter expression. type BloomQuerier struct { - c Client - logger log.Logger + c Client + logger log.Logger + metrics *querierMetrics } -func NewQuerier(c Client, logger log.Logger) *BloomQuerier { - return &BloomQuerier{c: c, logger: logger} +func NewQuerier(c Client, r prometheus.Registerer, logger log.Logger) *BloomQuerier { + return &BloomQuerier{ + c: c, + metrics: newQuerierMetrics(r, constants.Loki, querierMetricsSubsystem), + logger: logger, + } } func convertToShortRef(ref *logproto.ChunkRef) *logproto.ShortRef { return &logproto.ShortRef{From: ref.From, Through: ref.Through, Checksum: ref.Checksum} } -func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from, through model.Time, chunkRefs []*logproto.ChunkRef, filters ...syntax.LineFilter) ([]*logproto.ChunkRef, error) { +func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from, through model.Time, chunkRefs []*logproto.ChunkRef, queryPlan plan.QueryPlan) ([]*logproto.ChunkRef, error) { // Shortcut that does not require any filtering - if len(chunkRefs) == 0 || len(filters) == 0 { + if len(chunkRefs) == 0 || len(syntax.ExtractLineFilters(queryPlan.AST)) == 0 { return chunkRefs, nil } @@ -37,7 +82,10 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from defer groupedChunksRefPool.Put(grouped) grouped = groupChunkRefs(chunkRefs, grouped) - refs, err := bq.c.FilterChunks(ctx, tenant, from, through, grouped, filters...) + preFilterChunks := len(chunkRefs) + preFilterSeries := len(grouped) + + refs, err := bq.c.FilterChunks(ctx, tenant, from, through, grouped, queryPlan) if err != nil { return nil, err } @@ -55,6 +103,15 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from }) } } + + postFilterChunks := len(result) + postFilterSeries := len(refs) + + bq.metrics.chunksTotal.Add(float64(preFilterChunks)) + bq.metrics.chunksFiltered.Add(float64(preFilterChunks - postFilterChunks)) + bq.metrics.seriesTotal.Add(float64(preFilterSeries)) + bq.metrics.seriesFiltered.Add(float64(preFilterSeries - postFilterSeries)) + return result, nil } diff --git a/pkg/bloomgateway/querier_test.go b/pkg/bloomgateway/querier_test.go index 1e7cfc30a53b..0d7872927cc4 100644 --- a/pkg/bloomgateway/querier_test.go +++ b/pkg/bloomgateway/querier_test.go @@ -8,11 +8,11 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/plan" ) type noopClient struct { @@ -21,7 +21,7 @@ type noopClient struct { } // FilterChunks implements Client. -func (c *noopClient) FilterChunks(ctx context.Context, tenant string, from, through model.Time, groups []*logproto.GroupedChunkRefs, filters ...syntax.LineFilter) ([]*logproto.GroupedChunkRefs, error) { // nolint:revive +func (c *noopClient) FilterChunks(ctx context.Context, tenant string, from, through model.Time, groups []*logproto.GroupedChunkRefs, plan plan.QueryPlan) ([]*logproto.GroupedChunkRefs, error) { // nolint:revive c.callCount++ return groups, c.err } @@ -32,7 +32,7 @@ func TestBloomQuerier(t *testing.T) { t.Run("client not called when filters are empty", func(t *testing.T) { c := &noopClient{} - bq := NewQuerier(c, logger) + bq := NewQuerier(c, nil, logger) ctx := context.Background() through := model.Now() @@ -42,8 +42,9 @@ func TestBloomQuerier(t *testing.T) { {Fingerprint: 1000, UserID: tenant, Checksum: 2}, {Fingerprint: 2000, UserID: tenant, Checksum: 3}, } - filters := []syntax.LineFilter{} - res, err := bq.FilterChunkRefs(ctx, tenant, from, through, chunkRefs, filters...) + expr, err := syntax.ParseExpr(`{foo="bar"}`) + require.NoError(t, err) + res, err := bq.FilterChunkRefs(ctx, tenant, from, through, chunkRefs, plan.QueryPlan{AST: expr}) require.NoError(t, err) require.Equal(t, chunkRefs, res) require.Equal(t, 0, c.callCount) @@ -51,16 +52,15 @@ func TestBloomQuerier(t *testing.T) { t.Run("client not called when chunkRefs are empty", func(t *testing.T) { c := &noopClient{} - bq := NewQuerier(c, logger) + bq := NewQuerier(c, nil, logger) ctx := context.Background() through := model.Now() from := through.Add(-12 * time.Hour) chunkRefs := []*logproto.ChunkRef{} - filters := []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: "uuid"}, - } - res, err := bq.FilterChunkRefs(ctx, tenant, from, through, chunkRefs, filters...) + expr, err := syntax.ParseExpr(`{foo="bar"} |= "uuid"`) + require.NoError(t, err) + res, err := bq.FilterChunkRefs(ctx, tenant, from, through, chunkRefs, plan.QueryPlan{AST: expr}) require.NoError(t, err) require.Equal(t, chunkRefs, res) require.Equal(t, 0, c.callCount) @@ -68,7 +68,7 @@ func TestBloomQuerier(t *testing.T) { t.Run("querier propagates error from client", func(t *testing.T) { c := &noopClient{err: errors.New("something went wrong")} - bq := NewQuerier(c, logger) + bq := NewQuerier(c, nil, logger) ctx := context.Background() through := model.Now() @@ -78,10 +78,9 @@ func TestBloomQuerier(t *testing.T) { {Fingerprint: 1000, UserID: tenant, Checksum: 2}, {Fingerprint: 2000, UserID: tenant, Checksum: 3}, } - filters := []syntax.LineFilter{ - {Ty: labels.MatchEqual, Match: "uuid"}, - } - res, err := bq.FilterChunkRefs(ctx, tenant, from, through, chunkRefs, filters...) + expr, err := syntax.ParseExpr(`{foo="bar"} |= "uuid"`) + require.NoError(t, err) + res, err := bq.FilterChunkRefs(ctx, tenant, from, through, chunkRefs, plan.QueryPlan{AST: expr}) require.Error(t, err) require.Nil(t, res) }) diff --git a/pkg/bloomgateway/sharding.go b/pkg/bloomgateway/sharding.go deleted file mode 100644 index 5dfb9f11732a..000000000000 --- a/pkg/bloomgateway/sharding.go +++ /dev/null @@ -1,156 +0,0 @@ -package bloomgateway - -import ( - "context" - - "github.com/go-kit/log" - "github.com/grafana/dskit/ring" - - util_ring "github.com/grafana/loki/pkg/util/ring" -) - -// TODO(chaudum): Replace this placeholder with actual BlockRef struct. -type BlockRef struct { - FromFp, ThroughFp uint64 - FromTs, ThroughTs int64 -} - -var ( - // BlocksOwnerSync is the operation used to check the authoritative owners of a block - // (replicas included). - BlocksOwnerSync = ring.NewOp([]ring.InstanceState{ring.JOINING, ring.ACTIVE, ring.LEAVING}, nil) - - // BlocksOwnerRead is the operation used to check the authoritative owners of a block - // (replicas included) that are available for queries (a bloom gateway is available for - // queries only when ACTIVE). - BlocksOwnerRead = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, nil) - - // BlocksRead is the operation run by the querier to query blocks via the bloom gateway. - BlocksRead = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, func(s ring.InstanceState) bool { - // Blocks can only be queried from ACTIVE instances. However, if the block belongs to - // a non-active instance, then we should extend the replication set and try to query it - // from the next ACTIVE instance in the ring (which is expected to have it because a - // bloom gateway keeps their previously owned blocks until new owners are ACTIVE). - return s != ring.ACTIVE - }) -) - -type ShardingStrategy interface { - // FilterTenants whose indexes should be loaded by the index gateway. - // Returns the list of user IDs that should be synced by the index gateway. - FilterTenants(ctx context.Context, tenantIDs []string) ([]string, error) - FilterBlocks(ctx context.Context, tenantID string, blockRefs []BlockRef) ([]BlockRef, error) -} - -type ShuffleShardingStrategy struct { - util_ring.TenantSharding - r ring.ReadRing - ringLifeCycler *ring.BasicLifecycler - logger log.Logger -} - -func NewShuffleShardingStrategy(r ring.ReadRing, ringLifecycler *ring.BasicLifecycler, limits Limits, logger log.Logger) *ShuffleShardingStrategy { - return &ShuffleShardingStrategy{ - TenantSharding: util_ring.NewTenantShuffleSharding(r, ringLifecycler, limits.BloomGatewayShardSize), - ringLifeCycler: ringLifecycler, - logger: logger, - } -} - -// FilterTenants implements ShardingStrategy. -func (s *ShuffleShardingStrategy) FilterTenants(_ context.Context, tenantIDs []string) ([]string, error) { - // As a protection, ensure the bloom gateway instance is healthy in the ring. It could also be missing - // in the ring if it was failing to heartbeat the ring and it got remove from another healthy bloom gateway - // instance, because of the auto-forget feature. - if set, err := s.r.GetAllHealthy(BlocksOwnerSync); err != nil { - return nil, err - } else if !set.Includes(s.ringLifeCycler.GetInstanceID()) { - return nil, errGatewayUnhealthy - } - - var filteredIDs []string - - for _, tenantID := range tenantIDs { - // Include the user only if it belongs to this bloom gateway shard. - if s.OwnsTenant(tenantID) { - filteredIDs = append(filteredIDs, tenantID) - } - } - - return filteredIDs, nil -} - -// nolint:revive -func getBucket(rangeMin, rangeMax, pos uint64) int { - return 0 -} - -// FilterBlocks implements ShardingStrategy. -func (s *ShuffleShardingStrategy) FilterBlocks(_ context.Context, tenantID string, blockRefs []BlockRef) ([]BlockRef, error) { - if !s.OwnsTenant(tenantID) { - return nil, nil - } - - filteredBlockRefs := make([]BlockRef, 0, len(blockRefs)) - - tenantRing := s.GetTenantSubRing(tenantID) - - fpSharding := util_ring.NewFingerprintShuffleSharding(tenantRing, s.ringLifeCycler, BlocksOwnerSync) - for _, blockRef := range blockRefs { - owns, err := fpSharding.OwnsFingerprint(blockRef.FromFp) - if err != nil { - return nil, err - } - if owns { - filteredBlockRefs = append(filteredBlockRefs, blockRef) - continue - } - - owns, err = fpSharding.OwnsFingerprint(blockRef.ThroughFp) - if err != nil { - return nil, err - } - if owns { - filteredBlockRefs = append(filteredBlockRefs, blockRef) - continue - } - } - - return filteredBlockRefs, nil -} - -// GetShuffleShardingSubring returns the subring to be used for a given user. -// This function should be used both by index gateway servers and clients in -// order to guarantee the same logic is used. -func GetShuffleShardingSubring(ring ring.ReadRing, tenantID string, limits Limits) ring.ReadRing { - shardSize := limits.BloomGatewayShardSize(tenantID) - - // A shard size of 0 means shuffle sharding is disabled for this specific user, - // so we just return the full ring so that indexes will be sharded across all index gateways. - // Since we set the shard size to replication factor if shard size is 0, this - // can only happen if both the shard size and the replication factor are set - // to 0. - if shardSize <= 0 { - return ring - } - - return ring.ShuffleShard(tenantID, shardSize) -} - -// NoopStrategy is an implementation of the ShardingStrategy that does not -// filter anything. -type NoopStrategy struct{} - -func NewNoopStrategy() *NoopStrategy { - return &NoopStrategy{} -} - -// FilterTenants implements ShardingStrategy. -func (s *NoopStrategy) FilterTenants(_ context.Context, tenantIDs []string) ([]string, error) { - return tenantIDs, nil -} - -// FilterBlocks implements ShardingStrategy. -func (s *NoopStrategy) FilterBlocks(_ context.Context, _ string, blockRefs []BlockRef) ([]BlockRef, error) { - return blockRefs, nil -} diff --git a/pkg/bloomgateway/util.go b/pkg/bloomgateway/util.go index cf72aec3b5b4..dca14b7be3e5 100644 --- a/pkg/bloomgateway/util.go +++ b/pkg/bloomgateway/util.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" + "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" ) @@ -47,9 +48,15 @@ func getFromThrough(refs []*logproto.ShortRef) (model.Time, model.Time) { // convertToSearches converts a list of line filter expressions to a list of // byte slices that can be used with the bloom filters. -func convertToSearches(filters []syntax.LineFilter, t *v1.NGramTokenizer) [][]byte { +func convertToSearches(t *v1.NGramTokenizer, filters ...syntax.LineFilterExpr) [][]byte { searches := make([][]byte, 0, (13-t.N)*len(filters)) for _, f := range filters { + if f.Left != nil { + searches = append(searches, convertToSearches(t, *f.Left)...) + } + if f.Or != nil { + searches = append(searches, convertToSearches(t, *f.Or)...) + } if f.Ty == labels.MatchEqual { it := t.Tokens(f.Match) for it.Next() { @@ -82,15 +89,17 @@ func convertToChunkRefs(refs []*logproto.ShortRef) v1.ChunkRefs { return result } -type boundedTasks struct { - blockRef bloomshipper.BlockRef - tasks []Task +type blockWithTasks struct { + ref bloomshipper.BlockRef + tasks []Task } -func partitionFingerprintRange(tasks []Task, blocks []bloomshipper.BlockRef) (result []boundedTasks) { +func partitionTasks(tasks []Task, blocks []bloomshipper.BlockRef) []blockWithTasks { + result := make([]blockWithTasks, 0, len(blocks)) + for _, block := range blocks { - bounded := boundedTasks{ - blockRef: block, + bounded := blockWithTasks{ + ref: block, } for _, task := range tasks { @@ -119,14 +128,14 @@ func partitionFingerprintRange(tasks []Task, blocks []bloomshipper.BlockRef) (re return result } -type seriesWithBounds struct { - bounds model.Interval - day model.Time - series []*logproto.GroupedChunkRefs +type seriesWithInterval struct { + day config.DayTime + series []*logproto.GroupedChunkRefs + interval bloomshipper.Interval } -func partitionRequest(req *logproto.FilterChunkRefRequest) []seriesWithBounds { - result := make([]seriesWithBounds, 0) +func partitionRequest(req *logproto.FilterChunkRefRequest) []seriesWithInterval { + result := make([]seriesWithInterval, 0) fromDay, throughDay := truncateDay(req.From), truncateDay(req.Through) @@ -168,12 +177,12 @@ func partitionRequest(req *logproto.FilterChunkRefRequest) []seriesWithBounds { } if len(res) > 0 { - result = append(result, seriesWithBounds{ - bounds: model.Interval{ + result = append(result, seriesWithInterval{ + interval: bloomshipper.Interval{ Start: minTs, End: maxTs, }, - day: day, + day: config.NewDayTime(day), series: res, }) } diff --git a/pkg/bloomgateway/util_test.go b/pkg/bloomgateway/util_test.go index 61825a8c677a..3e55a3ab55aa 100644 --- a/pkg/bloomgateway/util_test.go +++ b/pkg/bloomgateway/util_test.go @@ -1,9 +1,6 @@ package bloomgateway import ( - "context" - "fmt" - "math/rand" "testing" "time" @@ -71,13 +68,12 @@ func TestTruncateDay(t *testing.T) { func mkBlockRef(minFp, maxFp uint64) bloomshipper.BlockRef { return bloomshipper.BlockRef{ Ref: bloomshipper.Ref{ - MinFingerprint: minFp, - MaxFingerprint: maxFp, + Bounds: v1.NewBounds(model.Fingerprint(minFp), model.Fingerprint(maxFp)), }, } } -func TestPartitionFingerprintRange(t *testing.T) { +func TestPartitionTasks(t *testing.T) { t.Run("consecutive block ranges", func(t *testing.T) { bounds := []bloomshipper.BlockRef{ @@ -97,7 +93,7 @@ func TestPartitionFingerprintRange(t *testing.T) { tasks[i%nTasks].series = append(tasks[i%nTasks].series, &logproto.GroupedChunkRefs{Fingerprint: uint64(i)}) } - results := partitionFingerprintRange(tasks, bounds) + results := partitionTasks(tasks, bounds) require.Equal(t, 3, len(results)) // ensure we only return bounds in range actualFingerprints := make([]*logproto.GroupedChunkRefs, 0, nSeries) @@ -132,7 +128,7 @@ func TestPartitionFingerprintRange(t *testing.T) { task.series = append(task.series, &logproto.GroupedChunkRefs{Fingerprint: uint64(i)}) } - results := partitionFingerprintRange([]Task{task}, bounds) + results := partitionTasks([]Task{task}, bounds) require.Equal(t, 3, len(results)) // ensure we only return bounds in range for _, res := range results { // ensure we have the right number of tasks per bound @@ -147,7 +143,7 @@ func TestPartitionRequest(t *testing.T) { testCases := map[string]struct { inp *logproto.FilterChunkRefRequest - exp []seriesWithBounds + exp []seriesWithInterval }{ "empty": { @@ -155,7 +151,7 @@ func TestPartitionRequest(t *testing.T) { From: ts.Add(-24 * time.Hour), Through: ts, }, - exp: []seriesWithBounds{}, + exp: []seriesWithInterval{}, }, "all chunks within single day": { @@ -177,10 +173,10 @@ func TestPartitionRequest(t *testing.T) { }, }, }, - exp: []seriesWithBounds{ + exp: []seriesWithInterval{ { - bounds: model.Interval{Start: ts.Add(-60 * time.Minute), End: ts.Add(-45 * time.Minute)}, - day: mktime("2024-01-24 00:00"), + interval: bloomshipper.Interval{Start: ts.Add(-60 * time.Minute), End: ts.Add(-45 * time.Minute)}, + day: config.NewDayTime(mktime("2024-01-24 00:00")), series: []*logproto.GroupedChunkRefs{ { Fingerprint: 0x00, @@ -218,10 +214,10 @@ func TestPartitionRequest(t *testing.T) { }, }, }, - exp: []seriesWithBounds{ + exp: []seriesWithInterval{ { - bounds: model.Interval{Start: ts.Add(-23 * time.Hour), End: ts.Add(-22 * time.Hour)}, - day: mktime("2024-01-23 00:00"), + interval: bloomshipper.Interval{Start: ts.Add(-23 * time.Hour), End: ts.Add(-22 * time.Hour)}, + day: config.NewDayTime(mktime("2024-01-23 00:00")), series: []*logproto.GroupedChunkRefs{ { Fingerprint: 0x00, @@ -232,8 +228,8 @@ func TestPartitionRequest(t *testing.T) { }, }, { - bounds: model.Interval{Start: ts.Add(-2 * time.Hour), End: ts.Add(-1 * time.Hour)}, - day: mktime("2024-01-24 00:00"), + interval: bloomshipper.Interval{Start: ts.Add(-2 * time.Hour), End: ts.Add(-1 * time.Hour)}, + day: config.NewDayTime(mktime("2024-01-24 00:00")), series: []*logproto.GroupedChunkRefs{ { Fingerprint: 0x01, @@ -259,10 +255,10 @@ func TestPartitionRequest(t *testing.T) { }, }, }, - exp: []seriesWithBounds{ + exp: []seriesWithInterval{ { - bounds: model.Interval{Start: ts.Add(-13 * time.Hour), End: ts.Add(-11 * time.Hour)}, - day: mktime("2024-01-23 00:00"), + interval: bloomshipper.Interval{Start: ts.Add(-13 * time.Hour), End: ts.Add(-11 * time.Hour)}, + day: config.NewDayTime(mktime("2024-01-23 00:00")), series: []*logproto.GroupedChunkRefs{ { Fingerprint: 0x00, @@ -273,8 +269,8 @@ func TestPartitionRequest(t *testing.T) { }, }, { - bounds: model.Interval{Start: ts.Add(-13 * time.Hour), End: ts.Add(-11 * time.Hour)}, - day: mktime("2024-01-24 00:00"), + interval: bloomshipper.Interval{Start: ts.Add(-13 * time.Hour), End: ts.Add(-11 * time.Hour)}, + day: config.NewDayTime(mktime("2024-01-24 00:00")), series: []*logproto.GroupedChunkRefs{ { Fingerprint: 0x00, @@ -297,36 +293,12 @@ func TestPartitionRequest(t *testing.T) { } -func createBlockQueriers(t *testing.T, numBlocks int, from, through model.Time, minFp, maxFp model.Fingerprint) ([]bloomshipper.BlockQuerierWithFingerprintRange, [][]v1.SeriesWithBloom) { +func createBlocks(t *testing.T, tenant string, n int, from, through model.Time, minFp, maxFp model.Fingerprint) ([]bloomshipper.BlockRef, []bloomshipper.Meta, []*bloomshipper.CloseableBlockQuerier, [][]v1.SeriesWithBloom) { t.Helper() - step := (maxFp - minFp) / model.Fingerprint(numBlocks) - bqs := make([]bloomshipper.BlockQuerierWithFingerprintRange, 0, numBlocks) - series := make([][]v1.SeriesWithBloom, 0, numBlocks) - for i := 0; i < numBlocks; i++ { - fromFp := minFp + (step * model.Fingerprint(i)) - throughFp := fromFp + step - 1 - // last block needs to include maxFp - if i == numBlocks-1 { - throughFp = maxFp - } - blockQuerier, data := v1.MakeBlockQuerier(t, fromFp, throughFp, from, through) - bq := bloomshipper.BlockQuerierWithFingerprintRange{ - BlockQuerier: blockQuerier, - MinFp: fromFp, - MaxFp: throughFp, - } - bqs = append(bqs, bq) - series = append(series, data) - } - return bqs, series -} -func createBlocks(t *testing.T, tenant string, n int, from, through model.Time, minFp, maxFp model.Fingerprint) ([]bloomshipper.BlockRef, []bloomshipper.Meta, []bloomshipper.BlockQuerierWithFingerprintRange, [][]v1.SeriesWithBloom) { - t.Helper() - - blocks := make([]bloomshipper.BlockRef, 0, n) + blockRefs := make([]bloomshipper.BlockRef, 0, n) metas := make([]bloomshipper.Meta, 0, n) - queriers := make([]bloomshipper.BlockQuerierWithFingerprintRange, 0, n) + queriers := make([]*bloomshipper.CloseableBlockQuerier, 0, n) series := make([][]v1.SeriesWithBloom, 0, n) step := (maxFp - minFp) / model.Fingerprint(n) @@ -339,94 +311,38 @@ func createBlocks(t *testing.T, tenant string, n int, from, through model.Time, } ref := bloomshipper.Ref{ TenantID: tenant, - TableName: "table_0", - MinFingerprint: uint64(fromFp), - MaxFingerprint: uint64(throughFp), + TableName: config.NewDayTable(config.NewDayTime(truncateDay(from)), "").Addr(), + Bounds: v1.NewBounds(fromFp, throughFp), StartTimestamp: from, EndTimestamp: through, } - block := bloomshipper.BlockRef{ - Ref: ref, - IndexPath: "index.tsdb.gz", - BlockPath: fmt.Sprintf("block-%d", i), + blockRef := bloomshipper.BlockRef{ + Ref: ref, } meta := bloomshipper.Meta{ MetaRef: bloomshipper.MetaRef{ Ref: ref, }, - Tombstones: []bloomshipper.BlockRef{}, - Blocks: []bloomshipper.BlockRef{block}, + Blocks: []bloomshipper.BlockRef{blockRef}, } - blockQuerier, data := v1.MakeBlockQuerier(t, fromFp, throughFp, from, through) - querier := bloomshipper.BlockQuerierWithFingerprintRange{ - BlockQuerier: blockQuerier, - MinFp: fromFp, - MaxFp: throughFp, + block, data, _ := v1.MakeBlock(t, n, fromFp, throughFp, from, through) + // Printing fingerprints and the log lines of its chunks comes handy for debugging... + // for i := range keys { + // t.Log(data[i].Series.Fingerprint) + // for j := range keys[i] { + // t.Log(i, j, string(keys[i][j])) + // } + // } + querier := &bloomshipper.CloseableBlockQuerier{ + BlockQuerier: v1.NewBlockQuerier(block), + BlockRef: blockRef, } queriers = append(queriers, querier) metas = append(metas, meta) - blocks = append(blocks, block) + blockRefs = append(blockRefs, blockRef) series = append(series, data) } - return blocks, metas, queriers, series -} - -func newMockBloomStore(bqs []bloomshipper.BlockQuerierWithFingerprintRange) *mockBloomStore { - return &mockBloomStore{bqs: bqs} -} - -type mockBloomStore struct { - bqs []bloomshipper.BlockQuerierWithFingerprintRange - // mock how long it takes to serve block queriers - delay time.Duration - // mock response error when serving block queriers in ForEach - err error -} - -var _ bloomshipper.Interface = &mockBloomStore{} - -// GetBlockRefs implements bloomshipper.Interface -func (s *mockBloomStore) GetBlockRefs(_ context.Context, tenant string, _ bloomshipper.Interval) ([]bloomshipper.BlockRef, error) { - time.Sleep(s.delay) - blocks := make([]bloomshipper.BlockRef, 0, len(s.bqs)) - for i := range s.bqs { - blocks = append(blocks, bloomshipper.BlockRef{ - Ref: bloomshipper.Ref{ - MinFingerprint: uint64(s.bqs[i].MinFp), - MaxFingerprint: uint64(s.bqs[i].MaxFp), - TenantID: tenant, - }, - }) - } - return blocks, nil -} - -// Stop implements bloomshipper.Interface -func (s *mockBloomStore) Stop() {} - -// Fetch implements bloomshipper.Interface -func (s *mockBloomStore) Fetch(_ context.Context, _ string, _ []bloomshipper.BlockRef, callback bloomshipper.ForEachBlockCallback) error { - if s.err != nil { - time.Sleep(s.delay) - return s.err - } - - shuffled := make([]bloomshipper.BlockQuerierWithFingerprintRange, len(s.bqs)) - _ = copy(shuffled, s.bqs) - - rand.Shuffle(len(shuffled), func(i, j int) { - shuffled[i], shuffled[j] = shuffled[j], shuffled[i] - }) - - for _, bq := range shuffled { - // ignore errors in the mock - time.Sleep(s.delay) - err := callback(bq.BlockQuerier, uint64(bq.MinFp), uint64(bq.MaxFp)) - if err != nil { - return err - } - } - return nil + return blockRefs, metas, queriers, series } func createQueryInputFromBlockData(t *testing.T, tenant string, data [][]v1.SeriesWithBloom, nthSeries int) []*logproto.ChunkRef { @@ -451,7 +367,7 @@ func createQueryInputFromBlockData(t *testing.T, tenant string, data [][]v1.Seri return res } -func createBlockRefsFromBlockData(t *testing.T, tenant string, data []bloomshipper.BlockQuerierWithFingerprintRange) []bloomshipper.BlockRef { +func createBlockRefsFromBlockData(t *testing.T, tenant string, data []*bloomshipper.CloseableBlockQuerier) []bloomshipper.BlockRef { t.Helper() res := make([]bloomshipper.BlockRef, 0) for i := range data { @@ -459,14 +375,11 @@ func createBlockRefsFromBlockData(t *testing.T, tenant string, data []bloomshipp Ref: bloomshipper.Ref{ TenantID: tenant, TableName: "", - MinFingerprint: uint64(data[i].MinFp), - MaxFingerprint: uint64(data[i].MaxFp), + Bounds: v1.NewBounds(data[i].Bounds.Min, data[i].Bounds.Max), StartTimestamp: 0, EndTimestamp: 0, Checksum: 0, }, - IndexPath: fmt.Sprintf("index-%d", i), - BlockPath: fmt.Sprintf("block-%d", i), }) } return res diff --git a/pkg/bloomgateway/worker.go b/pkg/bloomgateway/worker.go index 3b16fe4fdd7c..af61cdc1a0bd 100644 --- a/pkg/bloomgateway/worker.go +++ b/pkg/bloomgateway/worker.go @@ -11,59 +11,77 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" - "golang.org/x/exp/slices" "github.com/grafana/loki/pkg/queue" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper" ) +const ( + labelSuccess = "success" + labelFailure = "failure" +) + type workerConfig struct { maxItems int } type workerMetrics struct { - dequeuedTasks *prometheus.CounterVec - dequeueErrors *prometheus.CounterVec - dequeueWaitTime *prometheus.SummaryVec - storeAccessLatency *prometheus.HistogramVec - bloomQueryLatency *prometheus.HistogramVec + dequeueDuration *prometheus.HistogramVec + processDuration *prometheus.HistogramVec + metasFetched *prometheus.HistogramVec + blocksFetched *prometheus.HistogramVec + tasksDequeued *prometheus.CounterVec + tasksProcessed *prometheus.CounterVec + blockQueryLatency *prometheus.HistogramVec } func newWorkerMetrics(registerer prometheus.Registerer, namespace, subsystem string) *workerMetrics { labels := []string{"worker"} + r := promauto.With(registerer) return &workerMetrics{ - dequeuedTasks: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ + dequeueDuration: r.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "dequeued_tasks_total", - Help: "Total amount of tasks that the worker dequeued from the bloom query queue", + Name: "dequeue_duration_seconds", + Help: "Time spent dequeuing tasks from queue in seconds", }, labels), - dequeueErrors: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ + processDuration: r.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "process_duration_seconds", + Help: "Time spent processing tasks in seconds", + }, append(labels, "status")), + metasFetched: r.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "dequeue_errors_total", - Help: "Total amount of failed dequeue operations", + Name: "metas_fetched", + Help: "Amount of metas fetched", }, labels), - dequeueWaitTime: promauto.With(registerer).NewSummaryVec(prometheus.SummaryOpts{ + blocksFetched: r.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "dequeue_wait_time", - Help: "Time spent waiting for dequeuing tasks from queue", + Name: "blocks_fetched", + Help: "Amount of blocks fetched", }, labels), - bloomQueryLatency: promauto.With(registerer).NewHistogramVec(prometheus.HistogramOpts{ + tasksDequeued: r.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "bloom_query_latency", - Help: "Latency in seconds of processing bloom blocks", + Name: "tasks_dequeued_total", + Help: "Total amount of tasks that the worker dequeued from the queue", }, append(labels, "status")), - // TODO(chaudum): Move this metric into the bloomshipper - storeAccessLatency: promauto.With(registerer).NewHistogramVec(prometheus.HistogramOpts{ + tasksProcessed: r.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "store_latency", - Help: "Latency in seconds of accessing the bloom store component", - }, append(labels, "operation")), + Name: "tasks_processed_total", + Help: "Total amount of tasks that the worker processed", + }, append(labels, "status")), + blockQueryLatency: r.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "block_query_latency", + Help: "Time spent running searches against a bloom block", + }, append(labels, "status")), } } @@ -78,18 +96,18 @@ type worker struct { id string cfg workerConfig queue *queue.RequestQueue - shipper bloomshipper.Interface + store bloomshipper.Store pending *pendingTasks logger log.Logger metrics *workerMetrics } -func newWorker(id string, cfg workerConfig, queue *queue.RequestQueue, shipper bloomshipper.Interface, pending *pendingTasks, logger log.Logger, metrics *workerMetrics) *worker { +func newWorker(id string, cfg workerConfig, queue *queue.RequestQueue, store bloomshipper.Store, pending *pendingTasks, logger log.Logger, metrics *workerMetrics) *worker { w := &worker{ id: id, cfg: cfg, queue: queue, - shipper: shipper, + store: store, pending: pending, logger: log.With(logger, "worker", id), metrics: metrics, @@ -107,17 +125,19 @@ func (w *worker) starting(_ context.Context) error { func (w *worker) running(_ context.Context) error { idx := queue.StartIndexWithLocalQueue + p := newProcessor(w.id, w.store, w.logger, w.metrics) + for st := w.State(); st == services.Running || st == services.Stopping; { taskCtx := context.Background() - dequeueStart := time.Now() + start := time.Now() items, newIdx, err := w.queue.DequeueMany(taskCtx, idx, w.id, w.cfg.maxItems) - w.metrics.dequeueWaitTime.WithLabelValues(w.id).Observe(time.Since(dequeueStart).Seconds()) + w.metrics.dequeueDuration.WithLabelValues(w.id).Observe(time.Since(start).Seconds()) if err != nil { // We only return an error if the queue is stopped and dequeuing did not yield any items if err == queue.ErrStopped && len(items) == 0 { return err } - w.metrics.dequeueErrors.WithLabelValues(w.id).Inc() + w.metrics.tasksDequeued.WithLabelValues(w.id, labelFailure).Inc() level.Error(w.logger).Log("msg", "failed to dequeue tasks", "err", err, "items", len(items)) } idx = newIdx @@ -126,10 +146,10 @@ func (w *worker) running(_ context.Context) error { w.queue.ReleaseRequests(items) continue } - w.metrics.dequeuedTasks.WithLabelValues(w.id).Add(float64(len(items))) - - tasksPerDay := make(map[model.Time][]Task) + w.metrics.tasksDequeued.WithLabelValues(w.id, labelSuccess).Add(float64(len(items))) + tasks := make([]Task, 0, len(items)) + var mb v1.MultiFingerprintBounds for _, item := range items { task, ok := item.(Task) if !ok { @@ -139,95 +159,22 @@ func (w *worker) running(_ context.Context) error { } level.Debug(w.logger).Log("msg", "dequeued task", "task", task.ID) w.pending.Delete(task.ID) + tasks = append(tasks, task) - tasksPerDay[task.day] = append(tasksPerDay[task.day], task) + first, last := getFirstLast(task.series) + mb = mb.Union(v1.NewBounds(model.Fingerprint(first.Fingerprint), model.Fingerprint(last.Fingerprint))) } - for day, tasks := range tasksPerDay { + start = time.Now() + err = p.runWithBounds(taskCtx, tasks, mb) - // Remove tasks that are already cancelled - tasks = slices.DeleteFunc(tasks, func(t Task) bool { - if res := t.ctx.Err(); res != nil { - t.CloseWithError(res) - return true - } - return false - }) - // no tasks to process - // continue with tasks of next day - if len(tasks) == 0 { - continue - } - - // interval is [Start, End) - interval := bloomshipper.Interval{ - Start: day, // inclusive - End: day.Add(Day), // non-inclusive - } - - logger := log.With(w.logger, "day", day.Time(), "tenant", tasks[0].Tenant) - level.Debug(logger).Log("msg", "process tasks", "tasks", len(tasks)) - - storeFetchStart := time.Now() - blockRefs, err := w.shipper.GetBlockRefs(taskCtx, tasks[0].Tenant, interval) - w.metrics.storeAccessLatency.WithLabelValues(w.id, "GetBlockRefs").Observe(time.Since(storeFetchStart).Seconds()) - if err != nil { - for _, t := range tasks { - t.CloseWithError(err) - } - // continue with tasks of next day - continue - } - if len(tasks) == 0 { - continue - } - - // No blocks found. - // Since there are no blocks for the given tasks, we need to return the - // unfiltered list of chunk refs. - if len(blockRefs) == 0 { - level.Warn(logger).Log("msg", "no blocks found") - for _, t := range tasks { - t.Close() - } - // continue with tasks of next day - continue - } - - // Remove tasks that are already cancelled - tasks = slices.DeleteFunc(tasks, func(t Task) bool { - if res := t.ctx.Err(); res != nil { - t.CloseWithError(res) - return true - } - return false - }) - // no tasks to process - // continue with tasks of next day - if len(tasks) == 0 { - continue - } - - tasksForBlocks := partitionFingerprintRange(tasks, blockRefs) - blockRefs = blockRefs[:0] - for _, b := range tasksForBlocks { - blockRefs = append(blockRefs, b.blockRef) - } - - err = w.processBlocksWithCallback(taskCtx, tasks[0].Tenant, blockRefs, tasksForBlocks) - if err != nil { - for _, t := range tasks { - t.CloseWithError(err) - } - // continue with tasks of next day - continue - } - - // all tasks for this day are done. - // close them to notify the request handler - for _, task := range tasks { - task.Close() - } + if err != nil { + w.metrics.processDuration.WithLabelValues(w.id, labelFailure).Observe(time.Since(start).Seconds()) + w.metrics.tasksProcessed.WithLabelValues(w.id, labelFailure).Add(float64(len(tasks))) + level.Error(w.logger).Log("msg", "failed to process tasks", "err", err) + } else { + w.metrics.processDuration.WithLabelValues(w.id, labelSuccess).Observe(time.Since(start).Seconds()) + w.metrics.tasksProcessed.WithLabelValues(w.id, labelSuccess).Add(float64(len(tasks))) } // return dequeued items back to the pool @@ -242,41 +189,3 @@ func (w *worker) stopping(err error) error { w.queue.UnregisterConsumerConnection(w.id) return nil } - -func (w *worker) processBlocksWithCallback(taskCtx context.Context, tenant string, blockRefs []bloomshipper.BlockRef, boundedRefs []boundedTasks) error { - return w.shipper.Fetch(taskCtx, tenant, blockRefs, func(bq *v1.BlockQuerier, minFp, maxFp uint64) error { - for _, b := range boundedRefs { - if b.blockRef.MinFingerprint == minFp && b.blockRef.MaxFingerprint == maxFp { - return w.processBlock(bq, b.tasks) - } - } - return nil - }) -} - -func (w *worker) processBlock(blockQuerier *v1.BlockQuerier, tasks []Task) error { - schema, err := blockQuerier.Schema() - if err != nil { - return err - } - - tokenizer := v1.NewNGramTokenizer(schema.NGramLen(), 0) - iters := make([]v1.PeekingIterator[v1.Request], 0, len(tasks)) - for _, task := range tasks { - it := v1.NewPeekingIter(task.RequestIter(tokenizer)) - iters = append(iters, it) - } - fq := blockQuerier.Fuse(iters) - - start := time.Now() - err = fq.Run() - duration := time.Since(start).Seconds() - - if err != nil { - w.metrics.bloomQueryLatency.WithLabelValues(w.id, "failure").Observe(duration) - return err - } - - w.metrics.bloomQueryLatency.WithLabelValues(w.id, "success").Observe(duration) - return nil -} diff --git a/pkg/bloomutils/iter.go b/pkg/bloomutils/iter.go deleted file mode 100644 index fdbe4a5e6258..000000000000 --- a/pkg/bloomutils/iter.go +++ /dev/null @@ -1,37 +0,0 @@ -package bloomutils - -import ( - "io" - - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" -) - -// sortMergeIterator implements v1.Iterator -type sortMergeIterator[T any, C comparable, R any] struct { - curr *R - heap *v1.HeapIterator[v1.IndexedValue[C]] - items []T - transform func(T, C, *R) *R - err error -} - -func (it *sortMergeIterator[T, C, R]) Next() bool { - ok := it.heap.Next() - if !ok { - it.err = io.EOF - return false - } - - group := it.heap.At() - it.curr = it.transform(it.items[group.Index()], group.Value(), it.curr) - - return true -} - -func (it *sortMergeIterator[T, C, R]) At() R { - return *it.curr -} - -func (it *sortMergeIterator[T, C, R]) Err() error { - return it.err -} diff --git a/pkg/bloomutils/ring.go b/pkg/bloomutils/ring.go index 08e62a13acb7..9858f63e6ba3 100644 --- a/pkg/bloomutils/ring.go +++ b/pkg/bloomutils/ring.go @@ -1,32 +1,67 @@ // This file contains a bunch of utility functions for bloom components. -// TODO: Find a better location for this package package bloomutils import ( + "errors" + "fmt" "math" "sort" "github.com/grafana/dskit/ring" + "github.com/prometheus/common/model" + "golang.org/x/exp/constraints" "golang.org/x/exp/slices" v1 "github.com/grafana/loki/pkg/storage/bloom/v1" ) -type InstanceWithTokenRange struct { - Instance ring.InstanceDesc - MinToken, MaxToken uint32 +var ( + Uint32Range = Range[uint32]{Min: 0, Max: math.MaxUint32} + Uint64Range = Range[uint64]{Min: 0, Max: math.MaxUint64} +) + +type Range[T constraints.Unsigned] struct { + Min, Max T } -func (i InstanceWithTokenRange) Cmp(token uint32) v1.BoundsCheck { - if token < i.MinToken { +func (r Range[T]) String() string { + return fmt.Sprintf("%016x-%016x", r.Min, r.Max) +} + +func (r Range[T]) Less(other Range[T]) bool { + if r.Min != other.Min { + return r.Min < other.Min + } + return r.Max <= other.Max +} + +func (r Range[T]) Cmp(t T) v1.BoundsCheck { + if t < r.Min { return v1.Before - } else if token > i.MaxToken { + } else if t > r.Max { return v1.After } return v1.Overlap } +func NewRange[T constraints.Unsigned](min, max T) Range[T] { + return Range[T]{Min: min, Max: max} +} + +func NewTokenRange(min, max uint32) Range[uint32] { + return Range[uint32]{Min: min, Max: max} +} + +type InstanceWithTokenRange struct { + Instance ring.InstanceDesc + TokenRange Range[uint32] +} + +func (i InstanceWithTokenRange) Cmp(token uint32) v1.BoundsCheck { + return i.TokenRange.Cmp(token) +} + type InstancesWithTokenRange []InstanceWithTokenRange func (i InstancesWithTokenRange) Contains(token uint32) bool { @@ -38,109 +73,106 @@ func (i InstancesWithTokenRange) Contains(token uint32) bool { return false } -// GetInstanceTokenRange calculates the token range for a specific instance -// with given id based on the first token in the ring. -// This assumes that each instance in the ring is configured with only a single -// token. -func GetInstanceWithTokenRange(id string, instances []ring.InstanceDesc) InstancesWithTokenRange { - - // Sorting the tokens of the instances would not be necessary if there is - // only a single token per instances, however, since we only assume one - // token, but don't enforce one token, we keep the sorting. - for _, inst := range instances { - sort.Slice(inst.Tokens, func(i, j int) bool { - return inst.Tokens[i] < inst.Tokens[j] +// TODO(owen-d): use https://github.com/grafana/loki/pull/11975 after merge +func KeyspacesFromTokenRanges(tokenRanges ring.TokenRanges) []v1.FingerprintBounds { + keyspaces := make([]v1.FingerprintBounds, 0, len(tokenRanges)/2) + for i := 0; i < len(tokenRanges)-1; i += 2 { + keyspaces = append(keyspaces, v1.FingerprintBounds{ + Min: model.Fingerprint(tokenRanges[i]) << 32, + Max: model.Fingerprint(tokenRanges[i+1])<<32 | model.Fingerprint(math.MaxUint32), }) } + return keyspaces +} + +func TokenRangesForInstance(id string, instances []ring.InstanceDesc) (ranges ring.TokenRanges, err error) { + var ownedTokens map[uint32]struct{} + + // lifted from grafana/dskit/ring/model.go <*Desc>.GetTokens() + toks := make([][]uint32, 0, len(instances)) + for _, instance := range instances { + if instance.Id == id { + ranges = make(ring.TokenRanges, 0, 2*(len(instance.Tokens)+1)) + ownedTokens = make(map[uint32]struct{}, len(instance.Tokens)) + for _, tok := range instance.Tokens { + ownedTokens[tok] = struct{}{} + } + } + + // Tokens may not be sorted for an older version which, so we enforce sorting here. + tokens := instance.Tokens + if !sort.IsSorted(ring.Tokens(tokens)) { + sort.Sort(ring.Tokens(tokens)) + } - // Sort instances - sort.Slice(instances, func(i, j int) bool { - return instances[i].Tokens[0] < instances[j].Tokens[0] - }) + toks = append(toks, tokens) + } - idx := slices.IndexFunc(instances, func(inst ring.InstanceDesc) bool { - return inst.Id == id - }) + if cap(ranges) == 0 { + return nil, fmt.Errorf("instance %s not found", id) + } - // instance with Id == id not found - if idx == -1 { - return InstancesWithTokenRange{} + allTokens := ring.MergeTokens(toks) + if len(allTokens) == 0 { + return nil, errors.New("no tokens in the ring") } - i := uint32(idx) - n := uint32(len(instances)) - step := math.MaxUint32 / n + // mostly lifted from grafana/dskit/ring/token_range.go <*Ring>.GetTokenRangesForInstance() - minToken := step * i - maxToken := step*i + step - 1 - if i == n-1 { - // extend the last token tange to MaxUint32 - maxToken = math.MaxUint32 + // non-zero value means we're now looking for start of the range. Zero value means we're looking for next end of range (ie. token owned by this instance). + rangeEnd := uint32(0) + + // if this instance claimed the first token, it owns the wrap-around range, which we'll break into two separate ranges + firstToken := allTokens[0] + _, ownsFirstToken := ownedTokens[firstToken] + + if ownsFirstToken { + // we'll start by looking for the beginning of the range that ends with math.MaxUint32 + rangeEnd = math.MaxUint32 } - return InstancesWithTokenRange{ - {MinToken: minToken, MaxToken: maxToken, Instance: instances[i]}, + // walk the ring backwards, alternating looking for ends and starts of ranges + for i := len(allTokens) - 1; i > 0; i-- { + token := allTokens[i] + _, owned := ownedTokens[token] + + if rangeEnd == 0 { + // we're looking for the end of the next range + if owned { + rangeEnd = token - 1 + } + } else { + // we have a range end, and are looking for the start of the range + if !owned { + ranges = append(ranges, rangeEnd, token) + rangeEnd = 0 + } + } } -} -// GetInstancesWithTokenRanges calculates the token ranges for a specific -// instance with given id based on all tokens in the ring. -// If the instances in the ring are configured with a single token, such as the -// bloom compactor, use GetInstanceWithTokenRange() instead. -func GetInstancesWithTokenRanges(id string, instances []ring.InstanceDesc) InstancesWithTokenRange { - servers := make([]InstanceWithTokenRange, 0, len(instances)) - it := NewInstanceSortMergeIterator(instances) - var firstInst ring.InstanceDesc - var lastToken uint32 - for it.Next() { - if firstInst.Id == "" { - firstInst = it.At().Instance + // finally look at the first token again + // - if we have a range end, check if we claimed token 0 + // - if we don't, we have our start + // - if we do, the start is 0 + // - if we don't have a range end, check if we claimed token 0 + // - if we don't, do nothing + // - if we do, add the range of [0, token-1] + // - BUT, if the token itself is 0, do nothing, because we don't own the tokens themselves (we should be covered by the already added range that ends with MaxUint32) + + if rangeEnd == 0 { + if ownsFirstToken && firstToken != 0 { + ranges = append(ranges, firstToken-1, 0) } - if it.At().Instance.Id == id { - servers = append(servers, it.At()) + } else { + if ownsFirstToken { + ranges = append(ranges, rangeEnd, 0) + } else { + ranges = append(ranges, rangeEnd, firstToken) } - lastToken = it.At().MaxToken } - // append token range from lastToken+1 to MaxUint32 - // only if the instance with the first token is the current one - if len(servers) > 0 && firstInst.Id == id { - servers = append(servers, InstanceWithTokenRange{ - MinToken: lastToken + 1, - MaxToken: math.MaxUint32, - Instance: servers[0].Instance, - }) - } - return servers -} -// NewInstanceSortMergeIterator creates an iterator that yields instanceWithToken elements -// where the token of the elements are sorted in ascending order. -func NewInstanceSortMergeIterator(instances []ring.InstanceDesc) v1.Iterator[InstanceWithTokenRange] { - it := &sortMergeIterator[ring.InstanceDesc, uint32, InstanceWithTokenRange]{ - items: instances, - transform: func(item ring.InstanceDesc, val uint32, prev *InstanceWithTokenRange) *InstanceWithTokenRange { - var prevToken uint32 - if prev != nil { - prevToken = prev.MaxToken + 1 - } - return &InstanceWithTokenRange{Instance: item, MinToken: prevToken, MaxToken: val} - }, - } - sequences := make([]v1.PeekingIterator[v1.IndexedValue[uint32]], 0, len(instances)) - for i := range instances { - sort.Slice(instances[i].Tokens, func(a, b int) bool { - return instances[i].Tokens[a] < instances[i].Tokens[b] - }) - iter := v1.NewIterWithIndex[uint32](v1.NewSliceIter(instances[i].Tokens), i) - sequences = append(sequences, v1.NewPeekingIter[v1.IndexedValue[uint32]](iter)) - } - it.heap = v1.NewHeapIterator( - func(i, j v1.IndexedValue[uint32]) bool { - return i.Value() < j.Value() - }, - sequences..., - ) - it.err = nil - - return it + // Ensure returned ranges are sorted. + slices.Sort(ranges) + + return ranges, nil } diff --git a/pkg/bloomutils/ring_test.go b/pkg/bloomutils/ring_test.go index 30da072021ed..a6ef7374f527 100644 --- a/pkg/bloomutils/ring_test.go +++ b/pkg/bloomutils/ring_test.go @@ -1,112 +1,48 @@ package bloomutils import ( + "fmt" "math" "testing" "github.com/grafana/dskit/ring" "github.com/stretchr/testify/require" -) - -func TestBloomGatewayClient_SortInstancesByToken(t *testing.T) { - input := []ring.InstanceDesc{ - {Id: "1", Tokens: []uint32{5, 9}}, - {Id: "2", Tokens: []uint32{3, 7}}, - {Id: "3", Tokens: []uint32{1}}, - } - expected := []InstanceWithTokenRange{ - {Instance: input[2], MinToken: 0, MaxToken: 1}, - {Instance: input[1], MinToken: 2, MaxToken: 3}, - {Instance: input[0], MinToken: 4, MaxToken: 5}, - {Instance: input[1], MinToken: 6, MaxToken: 7}, - {Instance: input[0], MinToken: 8, MaxToken: 9}, - } - - var i int - it := NewInstanceSortMergeIterator(input) - for it.Next() { - t.Log(expected[i], it.At()) - require.Equal(t, expected[i], it.At()) - i++ - } -} -func TestBloomGatewayClient_GetInstancesWithTokenRanges(t *testing.T) { - t.Run("instance does not own first token in the ring", func(t *testing.T) { - input := []ring.InstanceDesc{ - {Id: "1", Tokens: []uint32{5, 9}}, - {Id: "2", Tokens: []uint32{3, 7}}, - {Id: "3", Tokens: []uint32{1}}, - } - expected := InstancesWithTokenRange{ - {Instance: input[1], MinToken: 2, MaxToken: 3}, - {Instance: input[1], MinToken: 6, MaxToken: 7}, - } - - result := GetInstancesWithTokenRanges("2", input) - require.Equal(t, expected, result) - }) - - t.Run("instance owns first token in the ring", func(t *testing.T) { - input := []ring.InstanceDesc{ - {Id: "1", Tokens: []uint32{5, 9}}, - {Id: "2", Tokens: []uint32{3, 7}}, - {Id: "3", Tokens: []uint32{1}}, - } - expected := InstancesWithTokenRange{ - {Instance: input[2], MinToken: 0, MaxToken: 1}, - {Instance: input[2], MinToken: 10, MaxToken: math.MaxUint32}, - } + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" +) - result := GetInstancesWithTokenRanges("3", input) - require.Equal(t, expected, result) - }) +func uint64Range(min, max uint64) Range[uint64] { + return Range[uint64]{min, max} } -func TestBloomGatewayClient_GetInstanceWithTokenRange(t *testing.T) { - for name, tc := range map[string]struct { - id string - input []ring.InstanceDesc - expected InstancesWithTokenRange +func TestKeyspacesFromTokenRanges(t *testing.T) { + for i, tc := range []struct { + tokenRanges ring.TokenRanges + exp []v1.FingerprintBounds }{ - "first instance includes 0 token": { - id: "3", - input: []ring.InstanceDesc{ - {Id: "1", Tokens: []uint32{3}}, - {Id: "2", Tokens: []uint32{5}}, - {Id: "3", Tokens: []uint32{1}}, - }, - expected: InstancesWithTokenRange{ - {Instance: ring.InstanceDesc{Id: "3", Tokens: []uint32{1}}, MinToken: 0, MaxToken: math.MaxUint32/3 - 1}, - }, - }, - "middle instance": { - id: "1", - input: []ring.InstanceDesc{ - {Id: "1", Tokens: []uint32{3}}, - {Id: "2", Tokens: []uint32{5}}, - {Id: "3", Tokens: []uint32{1}}, + { + tokenRanges: ring.TokenRanges{ + 0, math.MaxUint32 / 2, + math.MaxUint32/2 + 1, math.MaxUint32, }, - expected: InstancesWithTokenRange{ - {Instance: ring.InstanceDesc{Id: "1", Tokens: []uint32{3}}, MinToken: math.MaxUint32 / 3, MaxToken: math.MaxUint32/3*2 - 1}, + exp: []v1.FingerprintBounds{ + v1.NewBounds(0, math.MaxUint64/2), + v1.NewBounds(math.MaxUint64/2+1, math.MaxUint64), }, }, - "last instance includes MaxUint32 token": { - id: "2", - input: []ring.InstanceDesc{ - {Id: "1", Tokens: []uint32{3}}, - {Id: "2", Tokens: []uint32{5}}, - {Id: "3", Tokens: []uint32{1}}, + { + tokenRanges: ring.TokenRanges{ + 0, math.MaxUint8, + math.MaxUint16, math.MaxUint16 << 1, }, - expected: InstancesWithTokenRange{ - {Instance: ring.InstanceDesc{Id: "2", Tokens: []uint32{5}}, MinToken: math.MaxUint32 / 3 * 2, MaxToken: math.MaxUint32}, + exp: []v1.FingerprintBounds{ + v1.NewBounds(0, 0xff00000000|math.MaxUint32), + v1.NewBounds(math.MaxUint16<<32, math.MaxUint16<<33|math.MaxUint32), }, }, } { - tc := tc - t.Run(name, func(t *testing.T) { - result := GetInstanceWithTokenRange(tc.id, tc.input) - require.Equal(t, tc.expected, result) + t.Run(fmt.Sprint(i), func(t *testing.T) { + require.Equal(t, tc.exp, KeyspacesFromTokenRanges(tc.tokenRanges)) }) } } diff --git a/pkg/chunkenc/pool.go b/pkg/chunkenc/pool.go index ebe1924e8b65..4b6cf7abb90b 100644 --- a/pkg/chunkenc/pool.go +++ b/pkg/chunkenc/pool.go @@ -349,6 +349,9 @@ func (pool *SnappyPool) GetReader(src io.Reader) (io.Reader, error) { // PutReader places back in the pool a CompressionReader func (pool *SnappyPool) PutReader(reader io.Reader) { + r := reader.(*snappy.Reader) + // Reset to free reference to the underlying reader + r.Reset(nil) pool.readers.Put(reader) } diff --git a/pkg/compactor/compactor.go b/pkg/compactor/compactor.go index ca8323863370..8e3fa5212692 100644 --- a/pkg/compactor/compactor.go +++ b/pkg/compactor/compactor.go @@ -578,7 +578,7 @@ func (c *Compactor) stopping(_ error) error { } func (c *Compactor) CompactTable(ctx context.Context, tableName string, applyRetention bool) error { - schemaCfg, ok := schemaPeriodForTable(c.schemaConfig, tableName) + schemaCfg, ok := SchemaPeriodForTable(c.schemaConfig, tableName) if !ok { level.Error(util_log.Logger).Log("msg", "skipping compaction since we can't find schema for table", "table", tableName) return nil @@ -720,7 +720,7 @@ func (c *Compactor) RunCompaction(ctx context.Context, applyRetention bool) (err } // process most recent tables first - sortTablesByRange(tables) + SortTablesByRange(tables) // apply passed in compaction limits if c.cfg.SkipLatestNTables <= len(tables) { @@ -866,7 +866,7 @@ func (c *Compactor) ServeHTTP(w http.ResponseWriter, req *http.Request) { c.ring.ServeHTTP(w, req) } -func sortTablesByRange(tables []string) { +func SortTablesByRange(tables []string) { tableRanges := make(map[string]model.Interval) for _, table := range tables { tableRanges[table] = retention.ExtractIntervalFromTableName(table) @@ -879,7 +879,7 @@ func sortTablesByRange(tables []string) { } -func schemaPeriodForTable(cfg config.SchemaConfig, tableName string) (config.PeriodConfig, bool) { +func SchemaPeriodForTable(cfg config.SchemaConfig, tableName string) (config.PeriodConfig, bool) { tableInterval := retention.ExtractIntervalFromTableName(tableName) schemaCfg, err := cfg.SchemaForTime(tableInterval.Start) if err != nil || schemaCfg.IndexTables.TableFor(tableInterval.Start) != tableName { diff --git a/pkg/compactor/compactor_test.go b/pkg/compactor/compactor_test.go index 9f3f23424f2d..cfcc55e456d0 100644 --- a/pkg/compactor/compactor_test.go +++ b/pkg/compactor/compactor_test.go @@ -286,7 +286,7 @@ func Test_schemaPeriodForTable(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual, actualFound := schemaPeriodForTable(tt.config, tt.tableName) + actual, actualFound := SchemaPeriodForTable(tt.config, tt.tableName) require.Equal(t, tt.expectedFound, actualFound) require.Equal(t, tt.expected, actual) }) @@ -300,7 +300,7 @@ func Test_tableSort(t *testing.T) { "index_19192", } - sortTablesByRange(intervals) + SortTablesByRange(intervals) require.Equal(t, []string{"index_19195", "index_19192", "index_19191"}, intervals) } diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index a5229b0ca149..53ff20ed9274 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -39,6 +39,7 @@ import ( "github.com/grafana/loki/pkg/distributor/writefailures" "github.com/grafana/loki/pkg/ingester" "github.com/grafana/loki/pkg/ingester/client" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/runtime" @@ -52,7 +53,7 @@ import ( const ( ringKey = "distributor" - ringAutoForgetUnhealthyPeriods = 10 + ringAutoForgetUnhealthyPeriods = 2 ) var ( @@ -126,6 +127,8 @@ type Distributor struct { ingesterAppendTimeouts *prometheus.CounterVec replicationFactor prometheus.Gauge streamShardCount prometheus.Counter + + usageTracker push.UsageTracker } // New a distributor creates. @@ -138,6 +141,7 @@ func New( registerer prometheus.Registerer, metricsNamespace string, tee Tee, + usageTracker push.UsageTracker, logger log.Logger, ) (*Distributor, error) { factory := cfg.factory @@ -153,7 +157,7 @@ func New( return client.New(internalCfg, addr) } - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, usageTracker) if err != nil { return nil, err } @@ -185,6 +189,7 @@ func New( healthyInstancesCount: atomic.NewUint32(0), rateLimitStrat: rateLimitStrat, tee: tee, + usageTracker: usageTracker, ingesterAppends: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_ingester_appends_total", @@ -337,7 +342,8 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log // Truncate first so subsequent steps have consistent line lengths d.truncateLines(validationContext, &stream) - stream.Labels, stream.Hash, err = d.parseStreamLabels(validationContext, stream.Labels, &stream) + var lbs labels.Labels + lbs, stream.Labels, stream.Hash, err = d.parseStreamLabels(validationContext, stream.Labels, &stream) if err != nil { d.writeFailuresManager.Log(tenantID, err) validationErrors.Add(err) @@ -354,7 +360,7 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log pushSize := 0 prevTs := stream.Entries[0].Timestamp for _, entry := range stream.Entries { - if err := d.validator.ValidateEntry(validationContext, stream.Labels, entry); err != nil { + if err := d.validator.ValidateEntry(validationContext, lbs, entry); err != nil { d.writeFailuresManager.Log(tenantID, err) validationErrors.Add(err) continue @@ -412,6 +418,24 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log validation.DiscardedSamples.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineCount)) validation.DiscardedBytes.WithLabelValues(validation.RateLimited, tenantID).Add(float64(validatedLineSize)) + if d.usageTracker != nil { + for _, stream := range req.Streams { + lbs, _, _, err := d.parseStreamLabels(validationContext, stream.Labels, &stream) + if err != nil { + continue + } + + discardedStreamBytes := 0 + for _, e := range stream.Entries { + discardedStreamBytes += len(e.Line) + } + + if d.usageTracker != nil { + d.usageTracker.DiscardedBytesAdd(tenantID, validation.RateLimited, lbs, float64(discardedStreamBytes)) + } + } + } + err = fmt.Errorf(validation.RateLimitedErrorMsg, tenantID, int(d.ingestionRateLimiter.Limit(now, tenantID)), validatedLineCount, validatedLineSize) d.writeFailuresManager.Log(tenantID, err) return nil, httpgrpc.Errorf(http.StatusTooManyRequests, err.Error()) @@ -684,30 +708,29 @@ func (d *Distributor) sendStreamsErr(ctx context.Context, ingester ring.Instance } type labelData struct { - labels string - hash uint64 + ls labels.Labels + hash uint64 } -func (d *Distributor) parseStreamLabels(vContext validationContext, key string, stream *logproto.Stream) (string, uint64, error) { +func (d *Distributor) parseStreamLabels(vContext validationContext, key string, stream *logproto.Stream) (labels.Labels, string, uint64, error) { if val, ok := d.labelCache.Get(key); ok { labelVal := val.(labelData) - return labelVal.labels, labelVal.hash, nil + return labelVal.ls, labelVal.ls.String(), labelVal.hash, nil } ls, err := syntax.ParseLabels(key) if err != nil { - return "", 0, fmt.Errorf(validation.InvalidLabelsErrorMsg, key, err) + return nil, "", 0, fmt.Errorf(validation.InvalidLabelsErrorMsg, key, err) } if err := d.validator.ValidateLabels(vContext, ls, *stream); err != nil { - return "", 0, err + return nil, "", 0, err } - lsVal := ls.String() lsHash := ls.Hash() - d.labelCache.Add(key, labelData{lsVal, lsHash}) - return lsVal, lsHash, nil + d.labelCache.Add(key, labelData{ls, lsHash}) + return ls, ls.String(), lsHash, nil } // shardCountFor returns the right number of shards to be used by the given stream. diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 71830b4be4d2..04747ffb7233 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -396,7 +396,8 @@ func Test_IncrementTimestamp(t *testing.T) { distributors, _ := prepare(t, 1, 3, testData.limits, func(addr string) (ring_client.PoolClient, error) { return ing, nil }) _, err := distributors[0].Push(ctx, testData.push) assert.NoError(t, err) - assert.Equal(t, testData.expectedPush, ing.pushed[0]) + topVal := ing.Peek() + assert.Equal(t, testData.expectedPush, topVal) }) } } @@ -433,6 +434,8 @@ func TestDistributorPushConcurrently(t *testing.T) { labels := make(map[string]int) for i := range ingesters { + ingesters[i].mu.Lock() + pushed := ingesters[i].pushed counter = counter + len(pushed) for _, pr := range pushed { @@ -440,6 +443,7 @@ func TestDistributorPushConcurrently(t *testing.T) { labels[st.Labels] = labels[st.Labels] + 1 } } + ingesters[i].mu.Unlock() } assert.Equal(t, numReq*3, counter) // RF=3 // each stream is present 3 times @@ -500,7 +504,8 @@ func Test_SortLabelsOnPush(t *testing.T) { request.Streams[0].Labels = `{buzz="f", a="b"}` _, err := distributors[0].Push(ctx, request) require.NoError(t, err) - require.Equal(t, `{a="b", buzz="f"}`, ingester.pushed[0].Streams[0].Labels) + topVal := ingester.Peek() + require.Equal(t, `{a="b", buzz="f"}`, topVal.Streams[0].Labels) } func Test_TruncateLogLines(t *testing.T) { @@ -519,7 +524,8 @@ func Test_TruncateLogLines(t *testing.T) { _, err := distributors[0].Push(ctx, makeWriteRequest(1, 10)) require.NoError(t, err) - require.Len(t, ingester.pushed[0].Streams[0].Entries[0].Line, 5) + topVal := ingester.Peek() + require.Len(t, topVal.Streams[0].Entries[0].Line, 5) }) } @@ -606,7 +612,7 @@ func TestStreamShard(t *testing.T) { overrides, err := validation.NewOverrides(*distributorLimits, nil) require.NoError(t, err) - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, nil) require.NoError(t, err) d := Distributor{ @@ -650,7 +656,7 @@ func TestStreamShardAcrossCalls(t *testing.T) { overrides, err := validation.NewOverrides(*distributorLimits, nil) require.NoError(t, err) - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, nil) require.NoError(t, err) t.Run("it generates 4 shards across 2 calls when calculated shards = 2 * entries per call", func(t *testing.T) { @@ -715,7 +721,7 @@ func BenchmarkShardStream(b *testing.B) { overrides, err := validation.NewOverrides(*distributorLimits, nil) require.NoError(b, err) - validator, err := NewValidator(overrides) + validator, err := NewValidator(overrides, nil) require.NoError(b, err) distributorBuilder := func(shards int) *Distributor { @@ -782,7 +788,7 @@ func Benchmark_SortLabelsOnPush(b *testing.B) { for n := 0; n < b.N; n++ { stream := request.Streams[0] stream.Labels = `{buzz="f", a="b"}` - _, _, err := d.parseStreamLabels(vCtx, stream.Labels, &stream) + _, _, _, err := d.parseStreamLabels(vCtx, stream.Labels, &stream) if err != nil { panic("parseStreamLabels fail,err:" + err.Error()) } @@ -1153,7 +1159,7 @@ func prepare(t *testing.T, numDistributors, numIngesters int, limits *validation overrides, err := validation.NewOverrides(*limits, nil) require.NoError(t, err) - d, err := New(distributorConfig, clientConfig, runtime.DefaultTenantConfigs(), ingestersRing, overrides, prometheus.NewPedanticRegistry(), constants.Loki, nil, log.NewNopLogger()) + d, err := New(distributorConfig, clientConfig, runtime.DefaultTenantConfigs(), ingestersRing, overrides, prometheus.NewPedanticRegistry(), constants.Loki, nil, nil, log.NewNopLogger()) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), d)) distributors[i] = d @@ -1231,6 +1237,17 @@ func (i *mockIngester) Push(_ context.Context, in *logproto.PushRequest, _ ...gr return nil, nil } +func (i *mockIngester) Peek() *logproto.PushRequest { + i.mu.Lock() + defer i.mu.Unlock() + + if len(i.pushed) == 0 { + return nil + } + + return i.pushed[0] +} + func (i *mockIngester) GetStreamRates(_ context.Context, _ *logproto.StreamRatesRequest, _ ...grpc.CallOption) (*logproto.StreamRatesResponse, error) { return &logproto.StreamRatesResponse{}, nil } diff --git a/pkg/distributor/http.go b/pkg/distributor/http.go index ce242355e077..d2582f027f9b 100644 --- a/pkg/distributor/http.go +++ b/pkg/distributor/http.go @@ -34,7 +34,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe http.Error(w, err.Error(), http.StatusBadRequest) return } - req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser) + req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser, d.usageTracker) if err != nil { if d.tenantConfigs.LogPushRequest(tenantID) { level.Debug(logger).Log( diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index 7fe76fae7823..f1f2e4acb0ea 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -8,6 +8,7 @@ import ( "github.com/prometheus/prometheus/model/labels" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/validation" ) @@ -18,13 +19,14 @@ const ( type Validator struct { Limits + usageTracker push.UsageTracker } -func NewValidator(l Limits) (*Validator, error) { +func NewValidator(l Limits, t push.UsageTracker) (*Validator, error) { if l == nil { return nil, errors.New("nil Limits") } - return &Validator{l}, nil + return &Validator{l, t}, nil } type validationContext struct { @@ -67,7 +69,7 @@ func (v Validator) getValidationContextForTime(now time.Time, userID string) val } // ValidateEntry returns an error if the entry is invalid and report metrics for invalid entries accordingly. -func (v Validator) ValidateEntry(ctx validationContext, labels string, entry logproto.Entry) error { +func (v Validator) ValidateEntry(ctx validationContext, labels labels.Labels, entry logproto.Entry) error { ts := entry.Timestamp.UnixNano() validation.LineLengthHist.Observe(float64(len(entry.Line))) @@ -77,6 +79,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log formatedRejectMaxAgeTime := time.Unix(0, ctx.rejectOldSampleMaxAge).Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.GreaterThanMaxSampleAge, ctx.userID).Add(float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.GreaterThanMaxSampleAge, labels, float64(len(entry.Line))) + } return fmt.Errorf(validation.GreaterThanMaxSampleAgeErrorMsg, labels, formatedEntryTime, formatedRejectMaxAgeTime) } @@ -84,6 +89,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log formatedEntryTime := entry.Timestamp.Format(timeFormat) validation.DiscardedSamples.WithLabelValues(validation.TooFarInFuture, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.TooFarInFuture, ctx.userID).Add(float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.TooFarInFuture, labels, float64(len(entry.Line))) + } return fmt.Errorf(validation.TooFarInFutureErrorMsg, labels, formatedEntryTime) } @@ -94,6 +102,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log // for parity. validation.DiscardedSamples.WithLabelValues(validation.LineTooLong, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.LineTooLong, ctx.userID).Add(float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.LineTooLong, labels, float64(len(entry.Line))) + } return fmt.Errorf(validation.LineTooLongErrorMsg, maxSize, labels, len(entry.Line)) } @@ -101,6 +112,9 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log if !ctx.allowStructuredMetadata { validation.DiscardedSamples.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.DisallowedStructuredMetadata, ctx.userID).Add(float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.DisallowedStructuredMetadata, labels, float64(len(entry.Line))) + } return fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, labels) } @@ -113,12 +127,18 @@ func (v Validator) ValidateEntry(ctx validationContext, labels string, entry log if maxSize := ctx.maxStructuredMetadataSize; maxSize != 0 && structuredMetadataSizeBytes > maxSize { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooLarge, ctx.userID).Add(float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooLarge, labels, float64(len(entry.Line))) + } return fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, labels, structuredMetadataSizeBytes, ctx.maxStructuredMetadataSize) } if maxCount := ctx.maxStructuredMetadataCount; maxCount != 0 && structuredMetadataCount > maxCount { validation.DiscardedSamples.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Inc() validation.DiscardedBytes.WithLabelValues(validation.StructuredMetadataTooMany, ctx.userID).Add(float64(len(entry.Line))) + if v.usageTracker != nil { + v.usageTracker.DiscardedBytesAdd(ctx.userID, validation.StructuredMetadataTooMany, labels, float64(len(entry.Line))) + } return fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, labels, structuredMetadataCount, ctx.maxStructuredMetadataCount) } } diff --git a/pkg/distributor/validator_test.go b/pkg/distributor/validator_test.go index 038f1dc4c5b7..0c37065e3056 100644 --- a/pkg/distributor/validator_test.go +++ b/pkg/distributor/validator_test.go @@ -18,8 +18,9 @@ import ( ) var ( - testStreamLabels = "FIXME" - testTime = time.Now() + testStreamLabels = labels.Labels{{Name: "my", Value: "label"}} + testStreamLabelsString = testStreamLabels.String() + testTime = time.Now() ) type fakeLimits struct { @@ -61,7 +62,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, logproto.Entry{Timestamp: testTime.Add(-time.Hour * 5), Line: "test"}, fmt.Errorf(validation.GreaterThanMaxSampleAgeErrorMsg, - testStreamLabels, + testStreamLabelsString, testTime.Add(-time.Hour*5).Format(timeFormat), testTime.Add(-1*time.Hour).Format(timeFormat), // same as RejectOldSamplesMaxAge ), @@ -71,7 +72,7 @@ func TestValidator_ValidateEntry(t *testing.T) { "test", nil, logproto.Entry{Timestamp: testTime.Add(time.Hour * 5), Line: "test"}, - fmt.Errorf(validation.TooFarInFutureErrorMsg, testStreamLabels, testTime.Add(time.Hour*5).Format(timeFormat)), + fmt.Errorf(validation.TooFarInFutureErrorMsg, testStreamLabelsString, testTime.Add(time.Hour*5).Format(timeFormat)), }, { "line too long", @@ -82,7 +83,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901"}, - fmt.Errorf(validation.LineTooLongErrorMsg, 10, testStreamLabels, 11), + fmt.Errorf(validation.LineTooLongErrorMsg, 10, testStreamLabelsString, 11), }, { "disallowed structured metadata", @@ -93,7 +94,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901", StructuredMetadata: push.LabelsAdapter{{Name: "foo", Value: "bar"}}}, - fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, testStreamLabels), + fmt.Errorf(validation.DisallowedStructuredMetadataErrorMsg, testStreamLabelsString), }, { "structured metadata too big", @@ -105,7 +106,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901", StructuredMetadata: push.LabelsAdapter{{Name: "foo", Value: "bar"}}}, - fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, testStreamLabels, 6, 4), + fmt.Errorf(validation.StructuredMetadataTooLargeErrorMsg, testStreamLabelsString, 6, 4), }, { "structured metadata too many", @@ -117,7 +118,7 @@ func TestValidator_ValidateEntry(t *testing.T) { }, }, logproto.Entry{Timestamp: testTime, Line: "12345678901", StructuredMetadata: push.LabelsAdapter{{Name: "foo", Value: "bar"}, {Name: "too", Value: "many"}}}, - fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, testStreamLabels, 2, 1), + fmt.Errorf(validation.StructuredMetadataTooManyErrorMsg, testStreamLabelsString, 2, 1), }, } for _, tt := range tests { @@ -126,7 +127,7 @@ func TestValidator_ValidateEntry(t *testing.T) { flagext.DefaultValues(l) o, err := validation.NewOverrides(*l, tt.overrides) assert.NoError(t, err) - v, err := NewValidator(o) + v, err := NewValidator(o, nil) assert.NoError(t, err) err = v.ValidateEntry(v.getValidationContextForTime(testTime, tt.userID), testStreamLabels, tt.entry) @@ -224,7 +225,7 @@ func TestValidator_ValidateLabels(t *testing.T) { flagext.DefaultValues(l) o, err := validation.NewOverrides(*l, tt.overrides) assert.NoError(t, err) - v, err := NewValidator(o) + v, err := NewValidator(o, nil) assert.NoError(t, err) err = v.ValidateLabels(v.getValidationContextForTime(testTime, tt.userID), mustParseLabels(tt.labels), logproto.Stream{Labels: tt.labels}) diff --git a/pkg/ingester/index/bitprefix.go b/pkg/ingester/index/bitprefix.go index 025005618d8c..8235c2821d6c 100644 --- a/pkg/ingester/index/bitprefix.go +++ b/pkg/ingester/index/bitprefix.go @@ -69,7 +69,7 @@ func (ii *BitPrefixInvertedIndex) getShards(shard *astmapper.ShardAnnotation) ([ } requestedShard := shard.TSDB() - minFp, maxFp := requestedShard.Bounds() + minFp, maxFp := requestedShard.GetFromThrough() // Determine how many bits we need to take from // the requested shard's min/max fingerprint values @@ -143,7 +143,7 @@ func (ii *BitPrefixInvertedIndex) Lookup(matchers []*labels.Matcher, shard *astm // Because bit prefix order is also ascending order, // the merged fingerprints from ascending shards are also in order. if filter { - minFP, maxFP := shard.TSDB().Bounds() + minFP, maxFP := shard.TSDB().GetFromThrough() minIdx := sort.Search(len(result), func(i int) bool { return result[i] >= minFP }) diff --git a/pkg/ingester/index/bitprefix_test.go b/pkg/ingester/index/bitprefix_test.go index 00640f42a23a..d4afb9f63572 100644 --- a/pkg/ingester/index/bitprefix_test.go +++ b/pkg/ingester/index/bitprefix_test.go @@ -90,9 +90,9 @@ func Test_BitPrefixDeleteAddLoopkup(t *testing.T) { func Test_BitPrefix_hash_mapping(t *testing.T) { lbs := labels.Labels{ - labels.Label{Name: "compose_project", Value: "loki-boltdb-storage-s3"}, + labels.Label{Name: "compose_project", Value: "loki-tsdb-storage-s3"}, labels.Label{Name: "compose_service", Value: "ingester-2"}, - labels.Label{Name: "container_name", Value: "loki-boltdb-storage-s3_ingester-2_1"}, + labels.Label{Name: "container_name", Value: "loki-tsdb-storage-s3_ingester-2_1"}, labels.Label{Name: "filename", Value: "/var/log/docker/790fef4c6a587c3b386fe85c07e03f3a1613f4929ca3abaa4880e14caadb5ad1/json.log"}, labels.Label{Name: "host", Value: "docker-desktop"}, labels.Label{Name: "source", Value: "stderr"}, @@ -115,7 +115,7 @@ func Test_BitPrefix_hash_mapping(t *testing.T) { res, err := ii.Lookup( []*labels.Matcher{{Type: labels.MatchEqual, Name: "compose_project", - Value: "loki-boltdb-storage-s3"}}, + Value: "loki-tsdb-storage-s3"}}, &astmapper.ShardAnnotation{ Shard: int(expShard), Of: requestedFactor, diff --git a/pkg/ingester/index/index_test.go b/pkg/ingester/index/index_test.go index 3cd5d0873469..bc6aaeebf344 100644 --- a/pkg/ingester/index/index_test.go +++ b/pkg/ingester/index/index_test.go @@ -95,9 +95,9 @@ func TestDeleteAddLoopkup(t *testing.T) { func Test_hash_mapping(t *testing.T) { lbs := labels.Labels{ - labels.Label{Name: "compose_project", Value: "loki-boltdb-storage-s3"}, + labels.Label{Name: "compose_project", Value: "loki-tsdb-storage-s3"}, labels.Label{Name: "compose_service", Value: "ingester-2"}, - labels.Label{Name: "container_name", Value: "loki-boltdb-storage-s3_ingester-2_1"}, + labels.Label{Name: "container_name", Value: "loki-tsdb-storage-s3_ingester-2_1"}, labels.Label{Name: "filename", Value: "/var/log/docker/790fef4c6a587c3b386fe85c07e03f3a1613f4929ca3abaa4880e14caadb5ad1/json.log"}, labels.Label{Name: "host", Value: "docker-desktop"}, labels.Label{Name: "source", Value: "stderr"}, @@ -108,7 +108,7 @@ func Test_hash_mapping(t *testing.T) { ii := NewWithShards(shard) ii.Add(logproto.FromLabelsToLabelAdapters(lbs), 1) - res, err := ii.Lookup([]*labels.Matcher{{Type: labels.MatchEqual, Name: "compose_project", Value: "loki-boltdb-storage-s3"}}, &astmapper.ShardAnnotation{Shard: int(labelsSeriesIDHash(lbs) % 16), Of: 16}) + res, err := ii.Lookup([]*labels.Matcher{{Type: labels.MatchEqual, Name: "compose_project", Value: "loki-tsdb-storage-s3"}}, &astmapper.ShardAnnotation{Shard: int(labelsSeriesIDHash(lbs) % 16), Of: 16}) require.NoError(t, err) require.Len(t, res, 1) require.Equal(t, model.Fingerprint(1), res[0]) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index a6d252a733ec..adff06187c64 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -871,13 +871,10 @@ func (i *Ingester) GetOrCreateInstance(instanceID string) (*instance, error) { / } // Query the ingests for log streams matching a set of matchers. -func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querier_QueryServer) (err error) { +func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querier_QueryServer) error { // initialize stats collection for ingester queries. _, ctx := stats.NewContext(queryServer.Context()) - start := time.Now().UTC() - var lines int32 - if req.Plan == nil { parsed, err := syntax.ParseLogSelector(req.Selector, true) if err != nil { @@ -888,17 +885,6 @@ func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querie } } - defer func() { - status := "successful" - if err != nil { - status = "failed" - } - statsCtx := stats.FromContext(ctx) - execTime := time.Since(start) - logql.RecordIngesterStreamsQueryMetrics(ctx, i.logger, req.Start, req.End, req.Selector, status, req.Limit, lines, req.Shards, - statsCtx.Result(execTime, time.Duration(0), 0)) - }() - instanceID, err := tenant.TenantID(ctx) if err != nil { return err @@ -940,17 +926,14 @@ func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querie batchLimit = -1 } - lines, err = sendBatches(ctx, it, queryServer, batchLimit) - return err + return sendBatches(ctx, it, queryServer, batchLimit) } // QuerySample the ingesters for series from logs matching a set of matchers. -func (i *Ingester) QuerySample(req *logproto.SampleQueryRequest, queryServer logproto.Querier_QuerySampleServer) (err error) { +func (i *Ingester) QuerySample(req *logproto.SampleQueryRequest, queryServer logproto.Querier_QuerySampleServer) error { // initialize stats collection for ingester queries. _, ctx := stats.NewContext(queryServer.Context()) sp := opentracing.SpanFromContext(ctx) - start := time.Now().UTC() - var lines int32 // If the plan is empty we want all series to be returned. if req.Plan == nil { @@ -963,17 +946,6 @@ func (i *Ingester) QuerySample(req *logproto.SampleQueryRequest, queryServer log } } - defer func() { - status := "successful" - if err != nil { - status = "failed" - } - statsCtx := stats.FromContext(ctx) - execTime := time.Since(start) - logql.RecordIngesterSeriesQueryMetrics(ctx, i.logger, req.Start, req.End, req.Selector, status, lines, req.Shards, - statsCtx.Result(execTime, time.Duration(0), 0)) - }() - instanceID, err := tenant.TenantID(ctx) if err != nil { return err @@ -1012,8 +984,7 @@ func (i *Ingester) QuerySample(req *logproto.SampleQueryRequest, queryServer log defer util.LogErrorWithContext(ctx, "closing iterator", it.Close) - lines, err = sendSampleBatches(ctx, it, queryServer) - return err + return sendSampleBatches(ctx, it, queryServer) } // asyncStoreMaxLookBack returns a max look back period only if active index type is one of async index stores like `boltdb-shipper` and `tsdb`. diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 4521daaf2012..c7953ea1aba1 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -30,6 +30,7 @@ import ( "github.com/grafana/loki/pkg/ingester/index" "github.com/grafana/loki/pkg/ingester/wal" "github.com/grafana/loki/pkg/iter" + "github.com/grafana/loki/pkg/loghttp/push" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" "github.com/grafana/loki/pkg/logql/log" @@ -119,6 +120,8 @@ type instance struct { writeFailures *writefailures.Manager schemaconfig *config.SchemaConfig + + customStreamsTracker push.UsageTracker } func newInstance( @@ -262,6 +265,20 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *wal.Recor // record is only nil when replaying WAL. We don't want to drop data when replaying a WAL after // reducing the stream limits, for instance. var err error + + labels, err := syntax.ParseLabels(pushReqStream.Labels) + if err != nil { + if i.configs.LogStreamCreation(i.instanceID) { + level.Debug(util_log.Logger).Log( + "msg", "failed to create stream, failed to parse labels", + "org_id", i.instanceID, + "err", err, + "stream", pushReqStream.Labels, + ) + } + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + if record != nil { err = i.limiter.AssertMaxStreamsPerUser(i.instanceID, i.streams.Len()) } @@ -282,21 +299,12 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *wal.Recor bytes += len(e.Line) } validation.DiscardedBytes.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(bytes)) + if i.customStreamsTracker != nil { + i.customStreamsTracker.DiscardedBytesAdd(i.instanceID, validation.StreamLimit, labels, float64(bytes)) + } return nil, httpgrpc.Errorf(http.StatusTooManyRequests, validation.StreamLimitErrorMsg, i.instanceID) } - labels, err := syntax.ParseLabels(pushReqStream.Labels) - if err != nil { - if i.configs.LogStreamCreation(i.instanceID) { - level.Debug(util_log.Logger).Log( - "msg", "failed to create stream, failed to parse labels", - "org_id", i.instanceID, - "err", err, - "stream", pushReqStream.Labels, - ) - } - return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) - } fp := i.getHashForLabels(labels) sortedLabels := i.index.Add(logproto.FromLabelsToLabelAdapters(labels), fp) @@ -949,9 +957,8 @@ type QuerierQueryServer interface { Send(res *logproto.QueryResponse) error } -func sendBatches(ctx context.Context, i iter.EntryIterator, queryServer QuerierQueryServer, limit int32) (int32, error) { +func sendBatches(ctx context.Context, i iter.EntryIterator, queryServer QuerierQueryServer, limit int32) error { stats := stats.FromContext(ctx) - var lines int32 // send until the limit is reached. for limit != 0 && !isDone(ctx) { @@ -961,7 +968,7 @@ func sendBatches(ctx context.Context, i iter.EntryIterator, queryServer QuerierQ } batch, batchSize, err := iter.ReadBatch(i, fetchSize) if err != nil { - return lines, err + return err } if limit > 0 { @@ -970,49 +977,46 @@ func sendBatches(ctx context.Context, i iter.EntryIterator, queryServer QuerierQ stats.AddIngesterBatch(int64(batchSize)) batch.Stats = stats.Ingester() - lines += int32(batchSize) if isDone(ctx) { break } if err := queryServer.Send(batch); err != nil && err != context.Canceled { - return lines, err + return err } // We check this after sending an empty batch to make sure stats are sent if len(batch.Streams) == 0 { - return lines, err + return nil } stats.Reset() } - return lines, nil + return nil } -func sendSampleBatches(ctx context.Context, it iter.SampleIterator, queryServer logproto.Querier_QuerySampleServer) (int32, error) { - var lines int32 +func sendSampleBatches(ctx context.Context, it iter.SampleIterator, queryServer logproto.Querier_QuerySampleServer) error { sp := opentracing.SpanFromContext(ctx) stats := stats.FromContext(ctx) for !isDone(ctx) { batch, size, err := iter.ReadSampleBatch(it, queryBatchSampleSize) if err != nil { - return lines, err + return err } stats.AddIngesterBatch(int64(size)) batch.Stats = stats.Ingester() - lines += int32(size) if isDone(ctx) { break } if err := queryServer.Send(batch); err != nil && err != context.Canceled { - return lines, err + return err } // We check this after sending an empty batch to make sure stats are sent if len(batch.Series) == 0 { - return lines, nil + return nil } stats.Reset() @@ -1021,7 +1025,7 @@ func sendSampleBatches(ctx context.Context, it iter.SampleIterator, queryServer } } - return lines, nil + return nil } func shouldConsiderStream(stream *stream, reqFrom, reqThrough time.Time) bool { diff --git a/pkg/ingester/instance_test.go b/pkg/ingester/instance_test.go index ea36cee5ddc9..48c3a8b0bccd 100644 --- a/pkg/ingester/instance_test.go +++ b/pkg/ingester/instance_test.go @@ -614,16 +614,16 @@ func Test_Iterator(t *testing.T) { // assert the order is preserved. var res *logproto.QueryResponse - lines, err := sendBatches(context.TODO(), it, - fakeQueryServer( - func(qr *logproto.QueryResponse) error { - res = qr - return nil - }, - ), - int32(2)) - require.NoError(t, err) - require.Equal(t, int32(2), lines) + require.NoError(t, + sendBatches(context.TODO(), it, + fakeQueryServer( + func(qr *logproto.QueryResponse) error { + res = qr + return nil + }, + ), + int32(2)), + ) require.Equal(t, 2, len(res.Streams)) // each entry translated into a unique stream require.Equal(t, 1, len(res.Streams[0].Entries)) diff --git a/pkg/ingester/stream.go b/pkg/ingester/stream.go index 4c6aa4f9a122..81ce43692925 100644 --- a/pkg/ingester/stream.go +++ b/pkg/ingester/stream.go @@ -288,30 +288,28 @@ func (s *stream) recordAndSendToTailers(record *wal.Record, entries []logproto.E hasTailers := len(s.tailers) != 0 s.tailerMtx.RUnlock() if hasTailers { - go func() { - stream := logproto.Stream{Labels: s.labelsString, Entries: entries} - - closedTailers := []uint32{} - - s.tailerMtx.RLock() - for _, tailer := range s.tailers { - if tailer.isClosed() { - closedTailers = append(closedTailers, tailer.getID()) - continue - } - tailer.send(stream, s.labels) + stream := logproto.Stream{Labels: s.labelsString, Entries: entries} + + closedTailers := []uint32{} + + s.tailerMtx.RLock() + for _, tailer := range s.tailers { + if tailer.isClosed() { + closedTailers = append(closedTailers, tailer.getID()) + continue } - s.tailerMtx.RUnlock() + tailer.send(stream, s.labels) + } + s.tailerMtx.RUnlock() - if len(closedTailers) != 0 { - s.tailerMtx.Lock() - defer s.tailerMtx.Unlock() + if len(closedTailers) != 0 { + s.tailerMtx.Lock() + defer s.tailerMtx.Unlock() - for _, closedTailerID := range closedTailers { - delete(s.tailers, closedTailerID) - } + for _, closedTailerID := range closedTailers { + delete(s.tailers, closedTailerID) } - }() + } } } diff --git a/pkg/ingester/tailer.go b/pkg/ingester/tailer.go index 3e9a8a64cfd8..25fdfdb740d7 100644 --- a/pkg/ingester/tailer.go +++ b/pkg/ingester/tailer.go @@ -17,13 +17,21 @@ import ( util_log "github.com/grafana/loki/pkg/util/log" ) -const bufferSizeForTailResponse = 5 +const ( + bufferSizeForTailResponse = 5 + bufferSizeForTailStream = 100 +) type TailServer interface { Send(*logproto.TailResponse) error Context() context.Context } +type tailRequest struct { + stream logproto.Stream + lbs labels.Labels +} + type tailer struct { id uint32 orgID string @@ -31,6 +39,7 @@ type tailer struct { pipeline syntax.Pipeline pipelineMtx sync.Mutex + queue chan tailRequest sendChan chan *logproto.Stream // Signaling channel used to notify once the tailer gets closed @@ -59,6 +68,7 @@ func newTailer(orgID string, expr syntax.LogSelectorExpr, conn TailServer, maxDr orgID: orgID, matchers: matchers, sendChan: make(chan *logproto.Stream, bufferSizeForTailResponse), + queue: make(chan tailRequest, bufferSizeForTailStream), conn: conn, droppedStreams: make([]*logproto.DroppedStream, 0, maxDroppedStreams), maxDroppedStreams: maxDroppedStreams, @@ -73,6 +83,9 @@ func (t *tailer) loop() { var err error var ok bool + // Launch a go routine to receive streams sent with t.send + go t.receiveStreamsLoop() + for { select { case <-t.conn.Context().Done(): @@ -102,6 +115,37 @@ func (t *tailer) loop() { } } +func (t *tailer) receiveStreamsLoop() { + defer t.close() + for { + select { + case <-t.conn.Context().Done(): + return + case <-t.closeChan: + return + case req, ok := <-t.queue: + if !ok { + return + } + + streams := t.processStream(req.stream, req.lbs) + if len(streams) == 0 { + continue + } + + for _, s := range streams { + select { + case t.sendChan <- s: + default: + t.dropStream(*s) + } + } + } + } +} + +// send sends a stream to the tailer for processing and sending to the client. +// It will drop the stream if the tailer is blocked or the queue is full. func (t *tailer) send(stream logproto.Stream, lbs labels.Labels) { if t.isClosed() { return @@ -117,16 +161,16 @@ func (t *tailer) send(stream logproto.Stream, lbs labels.Labels) { return } - streams := t.processStream(stream, lbs) - if len(streams) == 0 { - return + // Send stream to queue for processing asynchronously + // If the queue is full, drop the stream + req := tailRequest{ + stream: stream, + lbs: lbs, } - for _, s := range streams { - select { - case t.sendChan <- s: - default: - t.dropStream(*s) - } + select { + case t.queue <- req: + default: + t.dropStream(stream) } } diff --git a/pkg/ingester/tailer_test.go b/pkg/ingester/tailer_test.go index 674dde3df8af..11de0d4daf82 100644 --- a/pkg/ingester/tailer_test.go +++ b/pkg/ingester/tailer_test.go @@ -2,6 +2,7 @@ package ingester import ( "context" + "fmt" "math/rand" "sync" "testing" @@ -15,6 +16,55 @@ import ( "github.com/grafana/loki/pkg/logql/syntax" ) +func TestTailer_RoundTrip(t *testing.T) { + server := &fakeTailServer{} + + lbs := makeRandomLabels() + expr, err := syntax.ParseLogSelector(lbs.String(), true) + require.NoError(t, err) + tail, err := newTailer("org-id", expr, server, 10) + require.NoError(t, err) + var wg sync.WaitGroup + wg.Add(1) + go func() { + tail.loop() + wg.Done() + }() + + const numStreams = 1000 + var entries []logproto.Entry + for i := 0; i < numStreams; i += 3 { + var iterEntries []logproto.Entry + for j := 0; j < 3; j++ { + iterEntries = append(iterEntries, logproto.Entry{Timestamp: time.Unix(0, int64(i+j)), Line: fmt.Sprintf("line %d", i+j)}) + } + entries = append(entries, iterEntries...) + + tail.send(logproto.Stream{ + Labels: lbs.String(), + Entries: iterEntries, + }, lbs) + + // sleep a bit to allow the tailer to process the stream without dropping + // This should take about 5 seconds to process all the streams + time.Sleep(5 * time.Millisecond) + } + + // Wait for the stream to be received by the server. + require.Eventually(t, func() bool { + return len(server.GetResponses()) > 0 + }, 30*time.Second, 1*time.Second, "stream was not received") + + var processedEntries []logproto.Entry + for _, response := range server.GetResponses() { + processedEntries = append(processedEntries, response.Stream.Entries...) + } + require.ElementsMatch(t, entries, processedEntries) + + tail.close() + wg.Wait() +} + func TestTailer_sendRaceConditionOnSendWhileClosing(t *testing.T) { runs := 100 diff --git a/pkg/loghttp/entry.go b/pkg/loghttp/entry.go index 2a55ac9ecd28..0529bf536a2d 100644 --- a/pkg/loghttp/entry.go +++ b/pkg/loghttp/entry.go @@ -6,7 +6,7 @@ import ( "time" "unsafe" - "github.com/buger/jsonparser" + "github.com/grafana/jsonparser" jsoniter "github.com/json-iterator/go" "github.com/modern-go/reflect2" "github.com/prometheus/prometheus/model/labels" diff --git a/pkg/loghttp/labels.go b/pkg/loghttp/labels.go index b15a94ab2341..98bad4e95786 100644 --- a/pkg/loghttp/labels.go +++ b/pkg/loghttp/labels.go @@ -6,8 +6,8 @@ import ( "strconv" "strings" - "github.com/buger/jsonparser" "github.com/gorilla/mux" + "github.com/grafana/jsonparser" "github.com/grafana/loki/pkg/logproto" ) diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go index c25477a984e2..cb73f6db59ee 100644 --- a/pkg/loghttp/push/otlp.go +++ b/pkg/loghttp/push/otlp.go @@ -43,14 +43,14 @@ func newPushStats() *Stats { } } -func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) { +func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { stats := newPushStats() otlpLogs, err := extractLogs(r, stats) if err != nil { return nil, nil, err } - req := otlpToLokiPushRequest(otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), stats) + req := otlpToLokiPushRequest(otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), tracker, stats) return req, stats, nil } @@ -101,7 +101,7 @@ func extractLogs(r *http.Request, pushStats *Stats) (plog.Logs, error) { return req.Logs(), nil } -func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, stats *Stats) *logproto.PushRequest { +func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, tracker UsageTracker, stats *Stats) *logproto.PushRequest { if ld.LogRecordCount() == 0 { return &logproto.PushRequest{} } @@ -145,6 +145,7 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants labelsStr := streamLabels.String() lbs := modelLabelsSetToLabelsList(streamLabels) + if _, ok := pushRequestsByStream[labelsStr]; !ok { pushRequestsByStream[labelsStr] = logproto.Stream{ Labels: labelsStr, @@ -223,8 +224,15 @@ func otlpToLokiPushRequest(ld plog.Logs, userID string, tenantsRetention Tenants stream.Entries = append(stream.Entries, entry) pushRequestsByStream[labelsStr] = stream - stats.structuredMetadataBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) + metadataSize := int64(labelsSize(entry.StructuredMetadata) - resourceAttributesAsStructuredMetadataSize - scopeAttributesAsStructuredMetadataSize) + stats.structuredMetadataBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += metadataSize stats.logLinesBytes[tenantsRetention.RetentionPeriodFor(userID, lbs)] += int64(len(entry.Line)) + + if tracker != nil { + tracker.ReceivedBytesAdd(userID, tenantsRetention.RetentionPeriodFor(userID, lbs), lbs, float64(len(entry.Line))) + tracker.ReceivedBytesAdd(userID, tenantsRetention.RetentionPeriodFor(userID, lbs), lbs, float64(metadataSize)) + } + stats.numLines++ if entry.Timestamp.After(stats.mostRecentEntryTimestamp) { stats.mostRecentEntryTimestamp = entry.Timestamp diff --git a/pkg/loghttp/push/otlp_config.go b/pkg/loghttp/push/otlp_config.go index 64120d4a6252..44c0e932f9c1 100644 --- a/pkg/loghttp/push/otlp_config.go +++ b/pkg/loghttp/push/otlp_config.go @@ -56,9 +56,9 @@ var DefaultOTLPConfig = OTLPConfig{ } type OTLPConfig struct { - ResourceAttributes ResourceAttributesConfig `yaml:"resource_attributes,omitempty"` - ScopeAttributes []AttributesConfig `yaml:"scope_attributes,omitempty"` - LogAttributes []AttributesConfig `yaml:"log_attributes,omitempty"` + ResourceAttributes ResourceAttributesConfig `yaml:"resource_attributes,omitempty" doc:"description=Configuration for resource attributes to store them as index labels or Structured Metadata or drop them altogether"` + ScopeAttributes []AttributesConfig `yaml:"scope_attributes,omitempty" doc:"description=Configuration for scope attributes to store them as Structured Metadata or drop them altogether"` + LogAttributes []AttributesConfig `yaml:"log_attributes,omitempty" doc:"description=Configuration for log attributes to store them as Structured Metadata or drop them altogether"` } func (c *OTLPConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -115,9 +115,9 @@ func (c *OTLPConfig) Validate() error { } type AttributesConfig struct { - Action Action `yaml:"action,omitempty"` - Attributes []string `yaml:"attributes,omitempty"` - Regex relabel.Regexp `yaml:"regex,omitempty"` + Action Action `yaml:"action,omitempty" doc:"description=Configures action to take on matching attributes. It allows one of [structured_metadata, drop] for all attribute types. It additionally allows index_label action for resource attributes"` + Attributes []string `yaml:"attributes,omitempty" doc:"description=List of attributes to configure how to store them or drop them altogether"` + Regex relabel.Regexp `yaml:"regex,omitempty" doc:"description=Regex to choose attributes to configure how to store them or drop them altogether"` } func (c *AttributesConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -146,8 +146,8 @@ func (c *AttributesConfig) UnmarshalYAML(unmarshal func(interface{}) error) erro } type ResourceAttributesConfig struct { - IgnoreDefaults bool `yaml:"ignore_defaults,omitempty"` - AttributesConfig []AttributesConfig `yaml:"attributes,omitempty"` + IgnoreDefaults bool `yaml:"ignore_defaults,omitempty" doc:"default=false|description=Configure whether to ignore the default list of resource attributes to be stored as index labels and only use the given resource attributes config"` + AttributesConfig []AttributesConfig `yaml:"attributes_config,omitempty"` } func (c *ResourceAttributesConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { diff --git a/pkg/loghttp/push/otlp_config_test.go b/pkg/loghttp/push/otlp_config_test.go index a1cfc15ff52c..5fa625162850 100644 --- a/pkg/loghttp/push/otlp_config_test.go +++ b/pkg/loghttp/push/otlp_config_test.go @@ -19,7 +19,7 @@ func TestUnmarshalOTLPConfig(t *testing.T) { name: "only resource_attributes set", yamlConfig: []byte(` resource_attributes: - attributes: + attributes_config: - action: index_label regex: foo`), expectedCfg: OTLPConfig{ @@ -39,7 +39,7 @@ resource_attributes: yamlConfig: []byte(` resource_attributes: ignore_defaults: true - attributes: + attributes_config: - action: index_label regex: foo`), expectedCfg: OTLPConfig{ @@ -82,7 +82,7 @@ scope_attributes: name: "all 3 set", yamlConfig: []byte(` resource_attributes: - attributes: + attributes_config: - action: index_label regex: foo scope_attributes: diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go index badb6cd000e4..593ac380e669 100644 --- a/pkg/loghttp/push/otlp_test.go +++ b/pkg/loghttp/push/otlp_test.go @@ -25,6 +25,7 @@ func TestOTLPToLokiPushRequest(t *testing.T) { expectedPushRequest logproto.PushRequest expectedStats Stats otlpConfig OTLPConfig + tracker UsageTracker }{ { name: "no logs", @@ -121,6 +122,7 @@ func TestOTLPToLokiPushRequest(t *testing.T) { { name: "service.name not defined in resource attributes", otlpConfig: DefaultOTLPConfig, + tracker: NewMockTracker(), generateLogs: func() plog.Logs { ld := plog.NewLogs() ld.ResourceLogs().AppendEmpty().Resource().Attributes().PutStr("service.namespace", "foo") @@ -152,7 +154,32 @@ func TestOTLPToLokiPushRequest(t *testing.T) { }, streamLabelsSize: 47, mostRecentEntryTimestamp: now, + /* + logLinesBytesCustomTrackers: []customTrackerPair{ + { + Labels: []labels.Label{ + {Name: "service_namespace", Value: "foo"}, + {Name: "tracker", Value: "foo"}, + }, + Bytes: map[time.Duration]int64{ + time.Hour: 9, + }, + }, + }, + structuredMetadataBytesCustomTrackers: []customTrackerPair{ + { + Labels: []labels.Label{ + {Name: "service_namespace", Value: "foo"}, + {Name: "tracker", Value: "foo"}, + }, + Bytes: map[time.Duration]int64{ + time.Hour: 0, + }, + }, + }, + */ }, + //expectedTrackedUsaged: }, { name: "resource attributes and scope attributes stored as structured metadata", @@ -459,7 +486,7 @@ func TestOTLPToLokiPushRequest(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { stats := newPushStats() - pushReq := otlpToLokiPushRequest(tc.generateLogs(), "foo", fakeRetention{}, tc.otlpConfig, stats) + pushReq := otlpToLokiPushRequest(tc.generateLogs(), "foo", fakeRetention{}, tc.otlpConfig, tc.tracker, stats) require.Equal(t, tc.expectedPushRequest, *pushReq) require.Equal(t, tc.expectedStats, *stats) }) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 15b7bba0a78c..012a70386bd7 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -36,6 +36,7 @@ var ( Name: "distributor_bytes_received_total", Help: "The total number of uncompressed bytes received per tenant. Includes structured metadata bytes.", }, []string{"tenant", "retention_hours"}) + structuredMetadataBytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_structured_metadata_bytes_received_total", @@ -62,7 +63,13 @@ type Limits interface { OTLPConfig(userID string) OTLPConfig } -type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits) (*logproto.PushRequest, *Stats, error) +type EmptyLimits struct{} + +func (EmptyLimits) OTLPConfig(string) OTLPConfig { + return DefaultOTLPConfig +} + +type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) type Stats struct { errs []error @@ -76,8 +83,8 @@ type Stats struct { bodySize int64 } -func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser) (*logproto.PushRequest, error) { - req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits) +func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker UsageTracker) (*logproto.PushRequest, error) { + req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits, tracker) if err != nil { return nil, err } @@ -87,10 +94,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete structuredMetadataSize int64 ) for retentionPeriod, size := range pushStats.logLinesBytes { - var retentionHours string - if retentionPeriod > 0 { - retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) - } + retentionHours := retentionPeriodToString(retentionPeriod) bytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) bytesReceivedStats.Inc(size) @@ -98,10 +102,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete } for retentionPeriod, size := range pushStats.structuredMetadataBytes { - var retentionHours string - if retentionPeriod > 0 { - retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) - } + retentionHours := retentionPeriodToString(retentionPeriod) structuredMetadataBytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) bytesIngested.WithLabelValues(userID, retentionHours).Add(float64(size)) @@ -135,7 +136,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete return req, nil } -func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, _ Limits) (*logproto.PushRequest, *Stats, error) { +func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, _ Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { // Body var body io.Reader // bodySize should always reflect the compressed size of the request body @@ -206,12 +207,17 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe for _, s := range req.Streams { pushStats.streamLabelsSize += int64(len(s.Labels)) - var retentionPeriod time.Duration - if tenantsRetention != nil { - lbs, err := syntax.ParseLabels(s.Labels) + + var lbs labels.Labels + if tenantsRetention != nil || tracker != nil { + lbs, err = syntax.ParseLabels(s.Labels) if err != nil { return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) } + } + + var retentionPeriod time.Duration + if tenantsRetention != nil { retentionPeriod = tenantsRetention.RetentionPeriodFor(userID, lbs) } for _, e := range s.Entries { @@ -222,6 +228,12 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe } pushStats.logLinesBytes[retentionPeriod] += int64(len(e.Line)) pushStats.structuredMetadataBytes[retentionPeriod] += entryLabelsSize + + if tracker != nil { + tracker.ReceivedBytesAdd(userID, retentionPeriod, lbs, float64(len(e.Line))) + tracker.ReceivedBytesAdd(userID, retentionPeriod, lbs, float64(entryLabelsSize)) + } + if e.Timestamp.After(pushStats.mostRecentEntryTimestamp) { pushStats.mostRecentEntryTimestamp = e.Timestamp } @@ -230,3 +242,11 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe return &req, pushStats, nil } + +func retentionPeriodToString(retentionPeriod time.Duration) string { + var retentionHours string + if retentionPeriod > 0 { + retentionHours = fmt.Sprintf("%d", int64(math.Floor(retentionPeriod.Hours()))) + } + return retentionHours +} diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index fa1e2fb28d11..ec4fd8c8f818 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -9,8 +9,10 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -54,6 +56,7 @@ func TestParseRequest(t *testing.T) { expectedStructuredMetadataBytes int expectedBytes int expectedLines int + expectedBytesUsageTracker map[string]float64 }{ { path: `/loki/api/v1/push`, @@ -68,21 +71,23 @@ func TestParseRequest(t *testing.T) { valid: false, }, { - path: `/loki/api/v1/push`, - body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, - contentType: `application/json`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, + contentType: `application/json`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { - path: `/loki/api/v1/push`, - body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, - contentType: `application/json`, - contentEncoding: ``, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, + contentType: `application/json`, + contentEncoding: ``, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { path: `/loki/api/v1/push`, @@ -92,22 +97,24 @@ func TestParseRequest(t *testing.T) { valid: false, }, { - path: `/loki/api/v1/push`, - body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json`, - contentEncoding: `gzip`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json`, + contentEncoding: `gzip`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { - path: `/loki/api/v1/push`, - body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json`, - contentEncoding: `deflate`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json`, + contentEncoding: `deflate`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { path: `/loki/api/v1/push`, @@ -117,22 +124,24 @@ func TestParseRequest(t *testing.T) { valid: false, }, { - path: `/loki/api/v1/push`, - body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json; charset=utf-8`, - contentEncoding: `gzip`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: gzipString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json; charset=utf-8`, + contentEncoding: `gzip`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { - path: `/loki/api/v1/push`, - body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), - contentType: `application/json; charset=utf-8`, - contentEncoding: `deflate`, - valid: true, - expectedBytes: len("fizzbuzz"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`), + contentType: `application/json; charset=utf-8`, + contentEncoding: `deflate`, + valid: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, }, { path: `/loki/api/v1/push`, @@ -185,6 +194,7 @@ func TestParseRequest(t *testing.T) { expectedStructuredMetadataBytes: 2*len("a") + 2*len("b"), expectedBytes: len("fizzbuzz") + 2*len("a") + 2*len("b"), expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuzz") + 2*len("a") + 2*len("b"))}, }, } { t.Run(fmt.Sprintf("test %d", index), func(t *testing.T) { @@ -200,7 +210,8 @@ func TestParseRequest(t *testing.T) { request.Header.Add("Content-Encoding", test.contentEncoding) } - data, err := ParseRequest(util_log.Logger, "fake", request, nil, nil, ParseLokiRequest) + tracker := NewMockTracker() + data, err := ParseRequest(util_log.Logger, "fake", request, nil, nil, ParseLokiRequest, tracker) structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived previousStructuredMetadataBytesReceived += structuredMetadataBytesReceived @@ -210,7 +221,7 @@ func TestParseRequest(t *testing.T) { previousLinesReceived += linesReceived if test.valid { - assert.Nil(t, err, "Should not give error for %d", index) + assert.NoErrorf(t, err, "Should not give error for %d", index) assert.NotNil(t, data, "Should give data for %d", index) require.Equal(t, test.expectedStructuredMetadataBytes, structuredMetadataBytesReceived) require.Equal(t, test.expectedBytes, bytesReceived) @@ -218,8 +229,9 @@ func TestParseRequest(t *testing.T) { require.Equal(t, float64(test.expectedStructuredMetadataBytes), testutil.ToFloat64(structuredMetadataBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedBytes), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedLines), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) + require.InDeltaMapValuesf(t, test.expectedBytesUsageTracker, tracker.receivedBytes, 0.0, "%s != %s", test.expectedBytesUsageTracker, tracker.receivedBytes) } else { - assert.NotNil(t, err, "Should give error for %d", index) + assert.Errorf(t, err, "Should give error for %d", index) assert.Nil(t, data, "Should not give data for %d", index) require.Equal(t, 0, structuredMetadataBytesReceived) require.Equal(t, 0, bytesReceived) @@ -231,3 +243,25 @@ func TestParseRequest(t *testing.T) { }) } } + +type MockCustomTracker struct { + receivedBytes map[string]float64 + discardedBytes map[string]float64 +} + +func NewMockTracker() *MockCustomTracker { + return &MockCustomTracker{ + receivedBytes: map[string]float64{}, + discardedBytes: map[string]float64{}, + } +} + +// DiscardedBytesAdd implements CustomTracker. +func (t *MockCustomTracker) DiscardedBytesAdd(_, _ string, labels labels.Labels, value float64) { + t.discardedBytes[labels.String()] += value +} + +// ReceivedBytesAdd implements CustomTracker. +func (t *MockCustomTracker) ReceivedBytesAdd(_ string, _ time.Duration, labels labels.Labels, value float64) { + t.receivedBytes[labels.String()] += value +} diff --git a/pkg/loghttp/push/usage_tracker.go b/pkg/loghttp/push/usage_tracker.go new file mode 100644 index 000000000000..ab84da5c6acc --- /dev/null +++ b/pkg/loghttp/push/usage_tracker.go @@ -0,0 +1,16 @@ +package push + +import ( + "time" + + "github.com/prometheus/prometheus/model/labels" +) + +type UsageTracker interface { + + // ReceivedBytesAdd records ingested bytes by tenant, retention period and labels. + ReceivedBytesAdd(tenant string, retentionPeriod time.Duration, labels labels.Labels, value float64) + + // DiscardedBytesAdd records discarded bytes by tenant and labels. + DiscardedBytesAdd(tenant, reason string, labels labels.Labels, value float64) +} diff --git a/pkg/loghttp/query.go b/pkg/loghttp/query.go index 617754393538..854ccd5ae711 100644 --- a/pkg/loghttp/query.go +++ b/pkg/loghttp/query.go @@ -8,7 +8,7 @@ import ( "time" "unsafe" - "github.com/buger/jsonparser" + "github.com/grafana/jsonparser" json "github.com/json-iterator/go" "github.com/prometheus/common/model" diff --git a/pkg/logproto/bloomgateway.pb.go b/pkg/logproto/bloomgateway.pb.go index e5c57e058bd2..98a22fd13168 100644 --- a/pkg/logproto/bloomgateway.pb.go +++ b/pkg/logproto/bloomgateway.pb.go @@ -9,6 +9,7 @@ import ( _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" github_com_grafana_loki_pkg_logql_syntax "github.com/grafana/loki/pkg/logql/syntax" + github_com_grafana_loki_pkg_querier_plan "github.com/grafana/loki/pkg/querier/plan" github_com_prometheus_common_model "github.com/prometheus/common/model" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -32,10 +33,12 @@ var _ = math.Inf const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type FilterChunkRefRequest struct { - From github_com_prometheus_common_model.Time `protobuf:"varint,1,opt,name=from,proto3,customtype=github.com/prometheus/common/model.Time" json:"from"` - Through github_com_prometheus_common_model.Time `protobuf:"varint,2,opt,name=through,proto3,customtype=github.com/prometheus/common/model.Time" json:"through"` - Refs []*GroupedChunkRefs `protobuf:"bytes,3,rep,name=refs,proto3" json:"refs,omitempty"` + From github_com_prometheus_common_model.Time `protobuf:"varint,1,opt,name=from,proto3,customtype=github.com/prometheus/common/model.Time" json:"from"` + Through github_com_prometheus_common_model.Time `protobuf:"varint,2,opt,name=through,proto3,customtype=github.com/prometheus/common/model.Time" json:"through"` + Refs []*GroupedChunkRefs `protobuf:"bytes,3,rep,name=refs,proto3" json:"refs,omitempty"` + // TODO(salvacorts): Delete this field once the weekly release is done. Filters []github_com_grafana_loki_pkg_logql_syntax.LineFilter `protobuf:"bytes,4,rep,name=filters,proto3,customtype=github.com/grafana/loki/pkg/logql/syntax.LineFilter" json:"filters"` + Plan github_com_grafana_loki_pkg_querier_plan.QueryPlan `protobuf:"bytes,5,opt,name=plan,proto3,customtype=github.com/grafana/loki/pkg/querier/plan.QueryPlan" json:"plan"` } func (m *FilterChunkRefRequest) Reset() { *m = FilterChunkRefRequest{} } @@ -234,37 +237,40 @@ func init() { func init() { proto.RegisterFile("pkg/logproto/bloomgateway.proto", fileDescriptor_a50b5dd1dbcd1415) } var fileDescriptor_a50b5dd1dbcd1415 = []byte{ - // 480 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x53, 0xbd, 0x6e, 0xd4, 0x30, - 0x1c, 0x8f, 0x7b, 0xa7, 0xf6, 0xea, 0x82, 0x40, 0x56, 0xa9, 0xa2, 0x20, 0xf9, 0xa2, 0x08, 0xc1, - 0x4d, 0x89, 0xd4, 0x2e, 0x48, 0x6c, 0x57, 0x89, 0x0a, 0x89, 0xc9, 0x20, 0x86, 0x6e, 0xb9, 0xd4, - 0xf9, 0x50, 0x12, 0xff, 0x53, 0xdb, 0x11, 0x74, 0xe3, 0x11, 0x78, 0x0c, 0x9e, 0x80, 0x27, 0x60, - 0xe8, 0x78, 0x63, 0xc5, 0x50, 0x71, 0xb9, 0x85, 0xb1, 0x8f, 0x80, 0xea, 0x5c, 0x7a, 0x77, 0x15, - 0xe8, 0x24, 0x26, 0x26, 0x7f, 0xfc, 0xff, 0x3f, 0xfb, 0xf7, 0x61, 0xe3, 0x61, 0x95, 0x27, 0x41, - 0x01, 0x49, 0x25, 0x41, 0x43, 0x30, 0x29, 0x00, 0xca, 0x24, 0xd4, 0xfc, 0x63, 0x78, 0xe1, 0x9b, - 0x2d, 0x32, 0xe8, 0x8a, 0xce, 0x7e, 0x02, 0x09, 0xb4, 0x7d, 0xb7, 0xb3, 0xb6, 0xee, 0x3c, 0x5d, - 0x3b, 0xa0, 0x9b, 0xb4, 0x45, 0xef, 0xfb, 0x16, 0x7e, 0xf2, 0x3a, 0x2b, 0x34, 0x97, 0xc7, 0x69, - 0x2d, 0x72, 0xc6, 0x63, 0xc6, 0xcf, 0x6b, 0xae, 0x34, 0x39, 0xc6, 0xfd, 0x58, 0x42, 0x69, 0x23, - 0x17, 0x8d, 0x7a, 0xe3, 0xe0, 0xf2, 0x7a, 0x68, 0xfd, 0xb8, 0x1e, 0xbe, 0x48, 0x32, 0x9d, 0xd6, - 0x13, 0x3f, 0x82, 0x32, 0xa8, 0x24, 0x94, 0x5c, 0xa7, 0xbc, 0x56, 0x41, 0x04, 0x65, 0x09, 0x22, - 0x28, 0xe1, 0x8c, 0x17, 0xfe, 0xfb, 0xac, 0xe4, 0xcc, 0x80, 0xc9, 0x1b, 0xbc, 0xa3, 0x53, 0x09, - 0x75, 0x92, 0xda, 0x5b, 0xff, 0x76, 0x4e, 0x87, 0x27, 0x3e, 0xee, 0x4b, 0x1e, 0x2b, 0xbb, 0xe7, - 0xf6, 0x46, 0x7b, 0x87, 0x8e, 0x7f, 0x27, 0xe4, 0x44, 0x42, 0x5d, 0xf1, 0xb3, 0x8e, 0xbf, 0x62, - 0xa6, 0x8f, 0xe4, 0x78, 0x27, 0x36, 0xc2, 0x94, 0xdd, 0x37, 0x90, 0xfd, 0x25, 0xe4, 0x6d, 0x26, - 0x78, 0xab, 0x7a, 0xfc, 0x6a, 0x41, 0xe8, 0x68, 0x85, 0x50, 0x22, 0xc3, 0x38, 0x14, 0x61, 0x50, - 0x40, 0x9e, 0x05, 0x0b, 0xf7, 0xce, 0x8b, 0x40, 0x5d, 0x08, 0x1d, 0x7e, 0x5a, 0x01, 0xb3, 0xee, - 0x06, 0x8f, 0xe1, 0x83, 0xfb, 0x2e, 0xaa, 0x0a, 0x84, 0xe2, 0xe4, 0x25, 0xde, 0x8d, 0x3a, 0x66, - 0x36, 0xda, 0xc8, 0x7d, 0xd9, 0xec, 0x7d, 0x43, 0x78, 0xf0, 0x2e, 0x05, 0xa9, 0x19, 0x8f, 0xff, - 0xbb, 0x34, 0x1c, 0x3c, 0x88, 0x52, 0x1e, 0xe5, 0xaa, 0x2e, 0xed, 0x9e, 0x8b, 0x46, 0x0f, 0xd9, - 0xdd, 0xda, 0xd3, 0xf8, 0xf1, 0x7d, 0x5d, 0xc4, 0xc5, 0x7b, 0x71, 0x26, 0x12, 0x2e, 0x2b, 0x99, - 0x09, 0x6d, 0x64, 0xf4, 0xd9, 0xea, 0x16, 0x39, 0xc0, 0xdb, 0x9a, 0x8b, 0x50, 0x68, 0xc3, 0x6d, - 0x97, 0x2d, 0x56, 0xe4, 0xf9, 0x5a, 0xee, 0x64, 0xe9, 0x5d, 0xe7, 0x4d, 0x9b, 0xf7, 0x61, 0x8c, - 0x1f, 0x8c, 0x6f, 0x3f, 0xc7, 0x49, 0xfb, 0x39, 0xc8, 0x07, 0xfc, 0x68, 0x3d, 0x12, 0x45, 0x86, - 0x4b, 0xf0, 0x1f, 0xdf, 0xbc, 0xe3, 0xfe, 0xbd, 0xa1, 0x8d, 0xd3, 0xb3, 0xc6, 0xa7, 0xd3, 0x19, - 0xb5, 0xae, 0x66, 0xd4, 0xba, 0x99, 0x51, 0xf4, 0xb9, 0xa1, 0xe8, 0x6b, 0x43, 0xd1, 0x65, 0x43, - 0xd1, 0xb4, 0xa1, 0xe8, 0x67, 0x43, 0xd1, 0xaf, 0x86, 0x5a, 0x37, 0x0d, 0x45, 0x5f, 0xe6, 0xd4, - 0x9a, 0xce, 0xa9, 0x75, 0x35, 0xa7, 0xd6, 0xe9, 0xb3, 0x0d, 0xcf, 0xcb, 0x5c, 0x3a, 0xd9, 0x36, - 0xc3, 0xd1, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x6d, 0x30, 0x9d, 0x8e, 0xf4, 0x03, 0x00, 0x00, + // 525 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x53, 0xbb, 0x6e, 0x13, 0x41, + 0x14, 0xdd, 0xc1, 0x26, 0x8f, 0x31, 0x2f, 0x8d, 0x42, 0xb4, 0x32, 0xd2, 0x78, 0x65, 0x21, 0x70, + 0xb5, 0x2b, 0x39, 0x0d, 0x82, 0xce, 0x91, 0x88, 0x90, 0x28, 0x60, 0x40, 0x14, 0x29, 0x90, 0xd6, + 0xce, 0xdd, 0x87, 0xbc, 0x3b, 0xb3, 0x9e, 0x99, 0x15, 0xb8, 0xe3, 0x13, 0xf8, 0x08, 0x0a, 0xbe, + 0x80, 0x6f, 0x48, 0xe9, 0x32, 0xa2, 0x88, 0xf0, 0xba, 0xa1, 0xcc, 0x27, 0x20, 0xcf, 0x7a, 0xb3, + 0x76, 0x04, 0x44, 0xa2, 0xa2, 0x9a, 0xc7, 0xbd, 0xe7, 0x9e, 0x7b, 0xee, 0x03, 0x77, 0xb2, 0x71, + 0xe8, 0x25, 0x22, 0xcc, 0xa4, 0xd0, 0xc2, 0x1b, 0x26, 0x42, 0xa4, 0xa1, 0xaf, 0xe1, 0x83, 0x3f, + 0x75, 0xcd, 0x17, 0xd9, 0xa9, 0x8c, 0xed, 0xbd, 0x50, 0x84, 0xa2, 0xf4, 0x5b, 0xde, 0x4a, 0x7b, + 0xfb, 0xc1, 0x46, 0x80, 0xea, 0x52, 0x1a, 0xbb, 0x5f, 0x1a, 0xf8, 0xfe, 0xf3, 0x38, 0xd1, 0x20, + 0x0f, 0xa3, 0x9c, 0x8f, 0x19, 0x04, 0x0c, 0x26, 0x39, 0x28, 0x4d, 0x0e, 0x71, 0x33, 0x90, 0x22, + 0xb5, 0x91, 0x83, 0x7a, 0x8d, 0x81, 0x77, 0x7a, 0xde, 0xb1, 0xbe, 0x9f, 0x77, 0x1e, 0x87, 0xb1, + 0x8e, 0xf2, 0xa1, 0x3b, 0x12, 0xa9, 0x97, 0x49, 0x91, 0x82, 0x8e, 0x20, 0x57, 0xde, 0x48, 0xa4, + 0xa9, 0xe0, 0x5e, 0x2a, 0x4e, 0x20, 0x71, 0xdf, 0xc6, 0x29, 0x30, 0x03, 0x26, 0x2f, 0xf0, 0xb6, + 0x8e, 0xa4, 0xc8, 0xc3, 0xc8, 0xbe, 0xf1, 0x6f, 0x71, 0x2a, 0x3c, 0x71, 0x71, 0x53, 0x42, 0xa0, + 0xec, 0x86, 0xd3, 0xe8, 0xb5, 0xfa, 0x6d, 0xf7, 0x52, 0xc8, 0x91, 0x14, 0x79, 0x06, 0x27, 0x55, + 0xfe, 0x8a, 0x19, 0x3f, 0x32, 0xc6, 0xdb, 0x81, 0x11, 0xa6, 0xec, 0xa6, 0x81, 0xec, 0xd5, 0x90, + 0x97, 0x31, 0x87, 0x52, 0xf5, 0xe0, 0xd9, 0x2a, 0xa1, 0x83, 0xb5, 0x84, 0x42, 0xe9, 0x07, 0x3e, + 0xf7, 0xbd, 0x44, 0x8c, 0x63, 0x6f, 0x55, 0xbd, 0x49, 0xe2, 0xa9, 0x29, 0xd7, 0xfe, 0xc7, 0x35, + 0x30, 0xab, 0x18, 0xc8, 0x7b, 0xdc, 0xcc, 0x12, 0x9f, 0xdb, 0x37, 0x1d, 0xd4, 0x6b, 0xf5, 0xef, + 0xd4, 0x4c, 0xaf, 0x12, 0x9f, 0x0f, 0x9e, 0xae, 0x38, 0xfa, 0x7f, 0xe3, 0x98, 0xe4, 0x20, 0x63, + 0x90, 0xde, 0x32, 0x8e, 0xfb, 0x3a, 0x07, 0x39, 0x5d, 0x62, 0x99, 0x89, 0xdb, 0x65, 0x78, 0xff, + 0x6a, 0x97, 0x54, 0x26, 0xb8, 0x02, 0xf2, 0x04, 0xef, 0x8e, 0x2a, 0xe5, 0x36, 0xba, 0xb6, 0x36, + 0xb5, 0x73, 0xf7, 0x1b, 0xc2, 0x3b, 0x6f, 0x22, 0x21, 0x35, 0x83, 0xe0, 0xbf, 0xeb, 0x76, 0x1b, + 0xef, 0x8c, 0x22, 0x18, 0x8d, 0x55, 0x9e, 0xda, 0x0d, 0x07, 0xf5, 0x6e, 0xb3, 0xcb, 0x77, 0x57, + 0xe3, 0x7b, 0x57, 0x75, 0x11, 0x07, 0xb7, 0x82, 0x98, 0x87, 0x20, 0x33, 0x19, 0x73, 0x6d, 0x64, + 0x34, 0xd9, 0xfa, 0x17, 0xd9, 0xc7, 0x5b, 0x1a, 0xb8, 0xcf, 0xb5, 0xc9, 0x6d, 0x97, 0xad, 0x5e, + 0xe4, 0xd1, 0xc6, 0x5c, 0x91, 0xba, 0x76, 0x55, 0x6d, 0xca, 0x79, 0xea, 0x07, 0xf8, 0xd6, 0x60, + 0xb9, 0x7c, 0x47, 0xe5, 0xf2, 0x91, 0x77, 0xf8, 0xee, 0x66, 0x4b, 0x14, 0xe9, 0xd4, 0xe0, 0xdf, + 0xee, 0x54, 0xdb, 0xf9, 0xb3, 0x43, 0xd9, 0xce, 0xae, 0x35, 0x38, 0x9e, 0xcd, 0xa9, 0x75, 0x36, + 0xa7, 0xd6, 0xc5, 0x9c, 0xa2, 0x4f, 0x05, 0x45, 0x5f, 0x0b, 0x8a, 0x4e, 0x0b, 0x8a, 0x66, 0x05, + 0x45, 0x3f, 0x0a, 0x8a, 0x7e, 0x16, 0xd4, 0xba, 0x28, 0x28, 0xfa, 0xbc, 0xa0, 0xd6, 0x6c, 0x41, + 0xad, 0xb3, 0x05, 0xb5, 0x8e, 0x1f, 0x5e, 0x33, 0xbe, 0x86, 0x74, 0xb8, 0x65, 0x8e, 0x83, 0x5f, + 0x01, 0x00, 0x00, 0xff, 0xff, 0xbe, 0xe2, 0x64, 0x8a, 0x54, 0x04, 0x00, 0x00, } func (this *FilterChunkRefRequest) Equal(that interface{}) bool { @@ -308,6 +314,9 @@ func (this *FilterChunkRefRequest) Equal(that interface{}) bool { return false } } + if !this.Plan.Equal(that1.Plan) { + return false + } return true } func (this *FilterChunkRefResponse) Equal(that interface{}) bool { @@ -408,7 +417,7 @@ func (this *FilterChunkRefRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 8) + s := make([]string, 0, 9) s = append(s, "&logproto.FilterChunkRefRequest{") s = append(s, "From: "+fmt.Sprintf("%#v", this.From)+",\n") s = append(s, "Through: "+fmt.Sprintf("%#v", this.Through)+",\n") @@ -416,6 +425,7 @@ func (this *FilterChunkRefRequest) GoString() string { s = append(s, "Refs: "+fmt.Sprintf("%#v", this.Refs)+",\n") } s = append(s, "Filters: "+fmt.Sprintf("%#v", this.Filters)+",\n") + s = append(s, "Plan: "+fmt.Sprintf("%#v", this.Plan)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -566,6 +576,16 @@ func (m *FilterChunkRefRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + { + size := m.Plan.Size() + i -= size + if _, err := m.Plan.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintBloomgateway(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a if len(m.Filters) > 0 { for iNdEx := len(m.Filters) - 1; iNdEx >= 0; iNdEx-- { { @@ -766,6 +786,8 @@ func (m *FilterChunkRefRequest) Size() (n int) { n += 1 + l + sovBloomgateway(uint64(l)) } } + l = m.Plan.Size() + n += 1 + l + sovBloomgateway(uint64(l)) return n } @@ -844,6 +866,7 @@ func (this *FilterChunkRefRequest) String() string { `Through:` + fmt.Sprintf("%v", this.Through) + `,`, `Refs:` + repeatedStringForRefs + `,`, `Filters:` + fmt.Sprintf("%v", this.Filters) + `,`, + `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `}`, }, "") return s @@ -1035,6 +1058,39 @@ func (m *FilterChunkRefRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Plan", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBloomgateway + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBloomgateway + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBloomgateway + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Plan.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipBloomgateway(dAtA[iNdEx:]) diff --git a/pkg/logproto/bloomgateway.proto b/pkg/logproto/bloomgateway.proto index 473ecf3f8153..13d5c25e763f 100644 --- a/pkg/logproto/bloomgateway.proto +++ b/pkg/logproto/bloomgateway.proto @@ -17,10 +17,15 @@ message FilterChunkRefRequest { (gogoproto.nullable) = false ]; repeated GroupedChunkRefs refs = 3; - repeated logproto.LineFilter filters = 4 [ + // TODO(salvacorts): Delete this field once the weekly release is done. + repeated LineFilter filters = 4 [ (gogoproto.customtype) = "github.com/grafana/loki/pkg/logql/syntax.LineFilter", (gogoproto.nullable) = false ]; + Plan plan = 5 [ + (gogoproto.customtype) = "github.com/grafana/loki/pkg/querier/plan.QueryPlan", + (gogoproto.nullable) = false + ]; } message FilterChunkRefResponse { diff --git a/pkg/logproto/compat.go b/pkg/logproto/compat.go index ee3d9ce1e003..0e65a90da02f 100644 --- a/pkg/logproto/compat.go +++ b/pkg/logproto/compat.go @@ -367,24 +367,8 @@ func (m *FilterChunkRefRequest) GetQuery() string { chunksHash = h.Sum64() } - // Short circuit if there are no filters. - if len(m.Filters) == 0 { - return fmt.Sprintf("%d", chunksHash) - } - - var sb strings.Builder - for i, filter := range m.Filters { - if i > 0 { - sb.WriteString(",") - } - sb.Write(fmt.Appendf(encodeBuf[:0], "%d", filter.Ty)) - sb.WriteString("-") - sb.WriteString(filter.Match) - sb.WriteString("-") - sb.WriteString(filter.Op) - } - - return fmt.Sprintf("%d/%s", chunksHash, sb.String()) + // TODO(salvacorts): plan.String() will return the whole query. This is not optimal since we are only interested in the filter expressions. + return fmt.Sprintf("%d/%d", chunksHash, m.Plan.Hash()) } // GetCachingOptions returns the caching options. diff --git a/pkg/logproto/compat_test.go b/pkg/logproto/compat_test.go index 4cfad825e183..d4de93638f82 100644 --- a/pkg/logproto/compat_test.go +++ b/pkg/logproto/compat_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/plan" ) // This test verifies that jsoninter uses our custom method for marshalling. @@ -287,7 +288,7 @@ func TestFilterChunkRefRequestGetQuery(t *testing.T) { }{ { desc: "empty request", - expected: `0`, + expected: `0/0`, }, { desc: "request no filters", @@ -299,19 +300,16 @@ func TestFilterChunkRefRequestGetQuery(t *testing.T) { }, }, }, - expected: `9962287286179718960`, + expected: `9962287286179718960/0`, }, { desc: "request with filters but no chunks", request: FilterChunkRefRequest{ - Filters: []syntax.LineFilter{ - { - Ty: 0, - Match: "uuid", - }, + Plan: plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"} |= "uuid"`), }, }, - expected: `0/0-uuid-`, + expected: `0/938557591`, }, { desc: "request with filters and chunks", @@ -326,18 +324,11 @@ func TestFilterChunkRefRequestGetQuery(t *testing.T) { Tenant: "test", }, }, - Filters: []syntax.LineFilter{ - { - Ty: 0, - Match: "uuid", - }, - { - Ty: 1, - Match: "trace", - }, + Plan: plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"} |= "uuid" != "trace"`), }, }, - expected: `8827404902424034886/0-uuid-,1-trace-`, + expected: `8827404902424034886/2710035654`, }, } { t.Run(tc.desc, func(t *testing.T) { diff --git a/pkg/logproto/logproto.pb.go b/pkg/logproto/logproto.pb.go index f0d826c5df6d..d50ae7d1e5db 100644 --- a/pkg/logproto/logproto.pb.go +++ b/pkg/logproto/logproto.pb.go @@ -1784,10 +1784,12 @@ func (m *LineFilter) GetRaw() []byte { } type GetChunkRefRequest struct { - From github_com_prometheus_common_model.Time `protobuf:"varint,1,opt,name=from,proto3,customtype=github.com/prometheus/common/model.Time" json:"from"` - Through github_com_prometheus_common_model.Time `protobuf:"varint,2,opt,name=through,proto3,customtype=github.com/prometheus/common/model.Time" json:"through"` - Matchers string `protobuf:"bytes,3,opt,name=matchers,proto3" json:"matchers,omitempty"` - Filters []github_com_grafana_loki_pkg_logql_syntax.LineFilter `protobuf:"bytes,4,rep,name=filters,proto3,customtype=github.com/grafana/loki/pkg/logql/syntax.LineFilter" json:"filters"` + From github_com_prometheus_common_model.Time `protobuf:"varint,1,opt,name=from,proto3,customtype=github.com/prometheus/common/model.Time" json:"from"` + Through github_com_prometheus_common_model.Time `protobuf:"varint,2,opt,name=through,proto3,customtype=github.com/prometheus/common/model.Time" json:"through"` + Matchers string `protobuf:"bytes,3,opt,name=matchers,proto3" json:"matchers,omitempty"` + // TODO(salvacorts): Delete this field once the weekly release is done. + Filters []github_com_grafana_loki_pkg_logql_syntax.LineFilter `protobuf:"bytes,4,rep,name=filters,proto3,customtype=github.com/grafana/loki/pkg/logql/syntax.LineFilter" json:"filters"` + Plan github_com_grafana_loki_pkg_querier_plan.QueryPlan `protobuf:"bytes,5,opt,name=plan,proto3,customtype=github.com/grafana/loki/pkg/querier/plan.QueryPlan" json:"plan"` } func (m *GetChunkRefRequest) Reset() { *m = GetChunkRefRequest{} } @@ -2561,149 +2563,150 @@ func init() { func init() { proto.RegisterFile("pkg/logproto/logproto.proto", fileDescriptor_c28a5f14f1f4c79a) } var fileDescriptor_c28a5f14f1f4c79a = []byte{ - // 2265 bytes of a gzipped FileDescriptorProto + // 2278 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x19, 0x4d, 0x6f, 0x1b, 0xc7, 0x95, 0x4b, 0x2e, 0xbf, 0x1e, 0x29, 0x59, 0x1e, 0x31, 0x36, 0x41, 0xdb, 0xa4, 0x3c, 0x48, 0x1d, 0xc1, 0x71, 0xc8, 0x58, 0x6e, 0xdc, 0xd4, 0x6e, 0xd0, 0x9a, 0x52, 0xec, 0xc8, 0x96, 0x3f, 0x32, - 0x72, 0xdd, 0xc2, 0x68, 0x61, 0xac, 0xc4, 0x11, 0x45, 0x88, 0xbb, 0x4b, 0xef, 0x0e, 0x63, 0x0b, - 0xe8, 0xa1, 0x7f, 0xa0, 0x68, 0x6e, 0x45, 0x2f, 0x45, 0x0f, 0x05, 0x52, 0xa0, 0xc8, 0xa5, 0x3f, + 0x72, 0xdd, 0xc2, 0x68, 0x6b, 0xac, 0xc4, 0x11, 0x45, 0x88, 0xbb, 0x4b, 0xef, 0x0e, 0x63, 0x0b, + 0xe8, 0xa1, 0x7f, 0x20, 0x68, 0x6e, 0x45, 0x2f, 0x45, 0x0f, 0x05, 0x52, 0xa0, 0xe8, 0xa5, 0x3f, 0xa0, 0xbd, 0xf4, 0xe0, 0xde, 0xdc, 0x5b, 0x90, 0x03, 0x5b, 0xcb, 0x97, 0x42, 0xa7, 0xdc, 0x72, - 0x2d, 0xe6, 0x6b, 0x77, 0x96, 0xa2, 0xdc, 0xd0, 0x75, 0x11, 0xf8, 0xc2, 0x9d, 0xf7, 0xe6, 0xcd, - 0x9b, 0xf7, 0x35, 0xef, 0xcd, 0x1b, 0xc2, 0x89, 0xc1, 0x4e, 0xb7, 0xd5, 0xf7, 0xbb, 0x83, 0xc0, - 0x67, 0x7e, 0x34, 0x68, 0x8a, 0x5f, 0x54, 0xd0, 0x70, 0xad, 0xd2, 0xf5, 0xbb, 0xbe, 0xa4, 0xe1, - 0x23, 0x39, 0x5f, 0x6b, 0x74, 0x7d, 0xbf, 0xdb, 0xa7, 0x2d, 0x01, 0x6d, 0x0c, 0xb7, 0x5a, 0xac, - 0xe7, 0xd2, 0x90, 0x39, 0xee, 0x40, 0x11, 0x2c, 0x28, 0xee, 0x0f, 0xfb, 0xae, 0xdf, 0xa1, 0xfd, - 0x56, 0xc8, 0x1c, 0x16, 0xca, 0x5f, 0x45, 0x31, 0xcf, 0x29, 0x06, 0xc3, 0x70, 0x5b, 0xfc, 0x48, - 0x24, 0xae, 0x00, 0x5a, 0x67, 0x01, 0x75, 0x5c, 0xe2, 0x30, 0x1a, 0x12, 0xfa, 0x70, 0x48, 0x43, - 0x86, 0x6f, 0xc2, 0x7c, 0x02, 0x1b, 0x0e, 0x7c, 0x2f, 0xa4, 0xe8, 0x22, 0x94, 0xc2, 0x18, 0x5d, - 0xb5, 0x16, 0x32, 0x8b, 0xa5, 0xa5, 0x4a, 0x33, 0x52, 0x25, 0x5e, 0x43, 0x4c, 0x42, 0xfc, 0x3b, - 0x0b, 0x20, 0x9e, 0x43, 0x75, 0x00, 0x39, 0xfb, 0x91, 0x13, 0x6e, 0x57, 0xad, 0x05, 0x6b, 0xd1, - 0x26, 0x06, 0x06, 0x9d, 0x83, 0xa3, 0x31, 0x74, 0xcb, 0x5f, 0xdf, 0x76, 0x82, 0x4e, 0x35, 0x2d, - 0xc8, 0x0e, 0x4e, 0x20, 0x04, 0x76, 0xe0, 0x30, 0x5a, 0xcd, 0x2c, 0x58, 0x8b, 0x19, 0x22, 0xc6, - 0xe8, 0x18, 0xe4, 0x18, 0xf5, 0x1c, 0x8f, 0x55, 0xed, 0x05, 0x6b, 0xb1, 0x48, 0x14, 0xc4, 0xf1, - 0x5c, 0x77, 0x1a, 0x56, 0xb3, 0x0b, 0xd6, 0xe2, 0x0c, 0x51, 0x10, 0xfe, 0x2c, 0x03, 0xe5, 0x8f, - 0x87, 0x34, 0xd8, 0x55, 0x06, 0x40, 0x75, 0x28, 0x84, 0xb4, 0x4f, 0x37, 0x99, 0x1f, 0x08, 0x01, - 0x8b, 0xed, 0x74, 0xd5, 0x22, 0x11, 0x0e, 0x55, 0x20, 0xdb, 0xef, 0xb9, 0x3d, 0x26, 0xc4, 0x9a, - 0x21, 0x12, 0x40, 0x97, 0x20, 0x1b, 0x32, 0x27, 0x60, 0x42, 0x96, 0xd2, 0x52, 0xad, 0x29, 0x9d, - 0xd6, 0xd4, 0x4e, 0x6b, 0xde, 0xd5, 0x4e, 0x6b, 0x17, 0x9e, 0x8c, 0x1a, 0xa9, 0x4f, 0xff, 0xd9, - 0xb0, 0x88, 0x5c, 0x82, 0x2e, 0x42, 0x86, 0x7a, 0x1d, 0x21, 0xef, 0x37, 0x5d, 0xc9, 0x17, 0xa0, - 0xf3, 0x50, 0xec, 0xf4, 0x02, 0xba, 0xc9, 0x7a, 0xbe, 0x27, 0xb4, 0x9a, 0x5d, 0x9a, 0x8f, 0x3d, - 0xb2, 0xa2, 0xa7, 0x48, 0x4c, 0x85, 0xce, 0x41, 0x2e, 0xe4, 0xa6, 0x0b, 0xab, 0xf9, 0x85, 0xcc, - 0x62, 0xb1, 0x5d, 0xd9, 0x1f, 0x35, 0xe6, 0x24, 0xe6, 0x9c, 0xef, 0xf6, 0x18, 0x75, 0x07, 0x6c, - 0x97, 0x28, 0x1a, 0x74, 0x16, 0xf2, 0x1d, 0xda, 0xa7, 0xdc, 0xe1, 0x05, 0xe1, 0xf0, 0x39, 0x83, - 0xbd, 0x98, 0x20, 0x9a, 0x00, 0xdd, 0x07, 0x7b, 0xd0, 0x77, 0xbc, 0x6a, 0x51, 0x68, 0x31, 0x1b, - 0x13, 0xde, 0xe9, 0x3b, 0x5e, 0xfb, 0xe2, 0x97, 0xa3, 0xc6, 0x52, 0xb7, 0xc7, 0xb6, 0x87, 0x1b, - 0xcd, 0x4d, 0xdf, 0x6d, 0x75, 0x03, 0x67, 0xcb, 0xf1, 0x9c, 0x56, 0xdf, 0xdf, 0xe9, 0xb5, 0x78, - 0x70, 0x3e, 0x1c, 0xd2, 0xa0, 0x47, 0x83, 0x16, 0xe7, 0xd1, 0x14, 0xfe, 0xe0, 0xeb, 0x88, 0xe0, - 0x79, 0xdd, 0x2e, 0xe4, 0xe6, 0xf2, 0x78, 0x94, 0x06, 0xb4, 0xee, 0xb8, 0x83, 0x3e, 0x9d, 0xca, - 0x5f, 0x91, 0x67, 0xd2, 0x2f, 0xed, 0x99, 0xcc, 0xb4, 0x9e, 0x89, 0xcd, 0x6c, 0x4f, 0x67, 0xe6, - 0xec, 0x37, 0x35, 0x73, 0xee, 0xd5, 0x9b, 0x19, 0x57, 0xc1, 0xe6, 0x10, 0x9a, 0x83, 0x4c, 0xe0, - 0x3c, 0x12, 0xc6, 0x2c, 0x13, 0x3e, 0xc4, 0x6b, 0x90, 0x93, 0x82, 0xa0, 0xda, 0xb8, 0xb5, 0x93, - 0x27, 0x23, 0xb6, 0x74, 0x46, 0xdb, 0x70, 0x2e, 0xb6, 0x61, 0x46, 0x58, 0x07, 0xff, 0xde, 0x82, - 0x19, 0xe5, 0x42, 0x95, 0x5d, 0x36, 0x20, 0x2f, 0x4f, 0xb7, 0xce, 0x2c, 0xc7, 0xc7, 0x33, 0xcb, - 0x95, 0x8e, 0x33, 0x60, 0x34, 0x68, 0xb7, 0x9e, 0x8c, 0x1a, 0xd6, 0x97, 0xa3, 0xc6, 0x5b, 0x2f, - 0xd2, 0x52, 0x24, 0x39, 0x95, 0x75, 0x34, 0x63, 0xf4, 0xb6, 0x90, 0x8e, 0x85, 0x2a, 0x0e, 0x8e, - 0x34, 0x65, 0x82, 0x5c, 0xf5, 0xba, 0x34, 0xe4, 0x9c, 0x6d, 0xee, 0x42, 0x22, 0x69, 0xf0, 0x2f, - 0x60, 0x3e, 0x11, 0x6a, 0x4a, 0xce, 0xf7, 0x21, 0x17, 0x72, 0x03, 0x6a, 0x31, 0x0d, 0x47, 0xad, - 0x0b, 0x7c, 0x7b, 0x56, 0xc9, 0x97, 0x93, 0x30, 0x51, 0xf4, 0xd3, 0xed, 0xfe, 0x37, 0x0b, 0xca, - 0x6b, 0xce, 0x06, 0xed, 0xeb, 0x18, 0x47, 0x60, 0x7b, 0x8e, 0x4b, 0x95, 0xc5, 0xc5, 0x98, 0x27, - 0xb4, 0x4f, 0x9c, 0xfe, 0x90, 0x4a, 0x96, 0x05, 0xa2, 0xa0, 0x69, 0x33, 0x91, 0xf5, 0xd2, 0x99, - 0xc8, 0x8a, 0xe3, 0xbd, 0x02, 0x59, 0x1e, 0x59, 0xbb, 0x22, 0x0b, 0x15, 0x89, 0x04, 0xf0, 0x5b, - 0x30, 0xa3, 0xb4, 0x50, 0xe6, 0x8b, 0x45, 0xe6, 0xe6, 0x2b, 0x6a, 0x91, 0xb1, 0x0b, 0x39, 0x69, - 0x6d, 0xf4, 0x26, 0x14, 0xa3, 0xea, 0x26, 0xb4, 0xcd, 0xb4, 0x73, 0xfb, 0xa3, 0x46, 0x9a, 0x85, - 0x24, 0x9e, 0x40, 0x0d, 0xc8, 0x8a, 0x95, 0x42, 0x73, 0xab, 0x5d, 0xdc, 0x1f, 0x35, 0x24, 0x82, - 0xc8, 0x0f, 0x3a, 0x09, 0xf6, 0x36, 0x2f, 0x30, 0xdc, 0x04, 0x76, 0xbb, 0xb0, 0x3f, 0x6a, 0x08, - 0x98, 0x88, 0x5f, 0x7c, 0x0d, 0xca, 0x6b, 0xb4, 0xeb, 0x6c, 0xee, 0xaa, 0x4d, 0x2b, 0x9a, 0x1d, - 0xdf, 0xd0, 0xd2, 0x3c, 0x4e, 0x43, 0x39, 0xda, 0xf1, 0x81, 0x1b, 0xaa, 0xa0, 0x2e, 0x45, 0xb8, - 0x9b, 0x21, 0xfe, 0xad, 0x05, 0xca, 0xcf, 0x08, 0x43, 0xae, 0xcf, 0x75, 0x0d, 0x55, 0x0e, 0x82, - 0xfd, 0x51, 0x43, 0x61, 0x88, 0xfa, 0xa2, 0xcb, 0x90, 0x0f, 0xc5, 0x8e, 0x9c, 0xd9, 0x78, 0xf8, - 0x88, 0x89, 0xf6, 0x11, 0x1e, 0x06, 0xfb, 0xa3, 0x86, 0x26, 0x24, 0x7a, 0x80, 0x9a, 0x89, 0xca, - 0x29, 0x15, 0x9b, 0xdd, 0x1f, 0x35, 0x0c, 0xac, 0x59, 0x49, 0xf1, 0xd7, 0x16, 0x94, 0xee, 0x3a, - 0xbd, 0x28, 0x84, 0xaa, 0xda, 0x45, 0x71, 0x8e, 0x94, 0x08, 0x7e, 0xa4, 0x3b, 0xb4, 0xef, 0xec, - 0x5e, 0xf5, 0x03, 0xc1, 0x77, 0x86, 0x44, 0x70, 0x5c, 0xec, 0xec, 0x89, 0xc5, 0x2e, 0x3b, 0x7d, - 0x4a, 0xfd, 0x3f, 0x26, 0xb0, 0xeb, 0x76, 0x21, 0x3d, 0x97, 0xc1, 0x9f, 0x5b, 0x50, 0x96, 0x9a, - 0xab, 0xb0, 0xfb, 0x19, 0xe4, 0xa4, 0x61, 0x84, 0xee, 0x2f, 0x48, 0x2e, 0x6f, 0x4f, 0x93, 0x58, - 0x14, 0x4f, 0xf4, 0x43, 0x98, 0xed, 0x04, 0xfe, 0x60, 0x40, 0x3b, 0xeb, 0x2a, 0x85, 0xa5, 0xc7, - 0x53, 0xd8, 0x8a, 0x39, 0x4f, 0xc6, 0xc8, 0xf1, 0xdf, 0x2d, 0x98, 0x51, 0xd9, 0x42, 0xf9, 0x2a, - 0xb2, 0xaf, 0xf5, 0xd2, 0x25, 0x2b, 0x3d, 0x6d, 0xc9, 0x3a, 0x06, 0xb9, 0x6e, 0xe0, 0x0f, 0x07, - 0x61, 0x35, 0x23, 0xcf, 0xa6, 0x84, 0xa6, 0x2b, 0x65, 0xf8, 0x3a, 0xcc, 0x6a, 0x55, 0x0e, 0x49, - 0x99, 0xb5, 0xf1, 0x94, 0xb9, 0xda, 0xa1, 0x1e, 0xeb, 0x6d, 0xf5, 0xa2, 0x24, 0xa8, 0xe8, 0xf1, - 0xaf, 0x2d, 0x98, 0x1b, 0x27, 0x41, 0x2b, 0xc6, 0x39, 0xe3, 0xec, 0xce, 0x1c, 0xce, 0xae, 0x29, - 0x92, 0x4f, 0xf8, 0xa1, 0xc7, 0x82, 0x5d, 0xcd, 0x5a, 0xae, 0xad, 0xbd, 0x07, 0x25, 0x63, 0x92, - 0x97, 0xa8, 0x1d, 0xaa, 0x4e, 0x06, 0xe1, 0xc3, 0x38, 0x25, 0xa4, 0x65, 0x42, 0x13, 0x00, 0xfe, - 0x8d, 0x05, 0x33, 0x09, 0x5f, 0xa2, 0xf7, 0xc1, 0xde, 0x0a, 0x7c, 0x77, 0x2a, 0x47, 0x89, 0x15, - 0xe8, 0xbb, 0x90, 0x66, 0xfe, 0x54, 0x6e, 0x4a, 0x33, 0x9f, 0x7b, 0x49, 0xa9, 0x9f, 0x91, 0xb7, - 0x5b, 0x09, 0xe1, 0xf7, 0xa0, 0x28, 0x14, 0xba, 0xe3, 0xf4, 0x82, 0x89, 0xd5, 0x62, 0xb2, 0x42, - 0x97, 0xe1, 0x88, 0xcc, 0x84, 0x93, 0x17, 0x97, 0x27, 0x2d, 0x2e, 0xeb, 0xc5, 0x27, 0x20, 0xbb, - 0xbc, 0x3d, 0xf4, 0x76, 0xf8, 0x92, 0x8e, 0xc3, 0x1c, 0xbd, 0x84, 0x8f, 0xf1, 0x1b, 0x30, 0xcf, - 0xcf, 0x20, 0x0d, 0xc2, 0x65, 0x7f, 0xe8, 0x31, 0xdd, 0x5d, 0x9c, 0x83, 0x4a, 0x12, 0xad, 0xa2, - 0xa4, 0x02, 0xd9, 0x4d, 0x8e, 0x10, 0x3c, 0x66, 0x88, 0x04, 0xf0, 0x1f, 0x2c, 0x40, 0xd7, 0x28, - 0x13, 0xbb, 0xac, 0xae, 0x44, 0xc7, 0xa3, 0x06, 0x05, 0xd7, 0x61, 0x9b, 0xdb, 0x34, 0x08, 0xf5, - 0x1d, 0x44, 0xc3, 0xdf, 0xc6, 0x6d, 0x0f, 0x9f, 0x87, 0xf9, 0x84, 0x94, 0x4a, 0xa7, 0x1a, 0x14, - 0x36, 0x15, 0x4e, 0xd5, 0xbb, 0x08, 0xc6, 0x7f, 0x4e, 0x43, 0x41, 0x2c, 0x20, 0x74, 0x0b, 0x9d, - 0x87, 0xd2, 0x56, 0xcf, 0xeb, 0xd2, 0x60, 0x10, 0xf4, 0x94, 0x09, 0xec, 0xf6, 0x91, 0xfd, 0x51, - 0xc3, 0x44, 0x13, 0x13, 0x40, 0xef, 0x40, 0x7e, 0x18, 0xd2, 0xe0, 0x41, 0x4f, 0x9e, 0xf4, 0x62, - 0xbb, 0xb2, 0x37, 0x6a, 0xe4, 0x7e, 0x1c, 0xd2, 0x60, 0x75, 0x85, 0x57, 0x9e, 0xa1, 0x18, 0x11, - 0xf9, 0xed, 0xa0, 0x1b, 0x2a, 0x4c, 0xc5, 0x25, 0xac, 0xfd, 0x3d, 0x2e, 0xfe, 0x58, 0xaa, 0x1b, - 0x04, 0xbe, 0x4b, 0xd9, 0x36, 0x1d, 0x86, 0xad, 0x4d, 0xdf, 0x75, 0x7d, 0xaf, 0x25, 0x7a, 0x49, - 0xa1, 0x34, 0x2f, 0x9f, 0x7c, 0xb9, 0x8a, 0xdc, 0xbb, 0x90, 0x67, 0xdb, 0x81, 0x3f, 0xec, 0x6e, - 0x8b, 0xaa, 0x90, 0x69, 0x5f, 0x9a, 0x9e, 0x9f, 0xe6, 0x40, 0xf4, 0x00, 0x9d, 0xe6, 0xd6, 0xa2, - 0x9b, 0x3b, 0xe1, 0xd0, 0x95, 0x1d, 0x5a, 0x3b, 0xbb, 0x3f, 0x6a, 0x58, 0xef, 0x90, 0x08, 0x8d, - 0x7f, 0x95, 0x86, 0x86, 0x08, 0xd4, 0x7b, 0xe2, 0xda, 0x70, 0xd5, 0x0f, 0x6e, 0x52, 0x16, 0xf4, - 0x36, 0x6f, 0x39, 0x2e, 0xd5, 0xb1, 0xd1, 0x80, 0x92, 0x2b, 0x90, 0x0f, 0x8c, 0x23, 0x00, 0x6e, - 0x44, 0x87, 0x4e, 0x01, 0x88, 0x33, 0x23, 0xe7, 0xe5, 0x69, 0x28, 0x0a, 0x8c, 0x98, 0x5e, 0x4e, - 0x58, 0xaa, 0x35, 0xa5, 0x66, 0xca, 0x42, 0xab, 0xe3, 0x16, 0x9a, 0x9a, 0x4f, 0x64, 0x16, 0x33, - 0xd6, 0xb3, 0xc9, 0x58, 0xc7, 0xff, 0xb0, 0xa0, 0xbe, 0xa6, 0x25, 0x7f, 0x49, 0x73, 0x68, 0x7d, - 0xd3, 0xaf, 0x48, 0xdf, 0xcc, 0xff, 0xa6, 0x2f, 0xae, 0x03, 0xac, 0xf5, 0x3c, 0x7a, 0xb5, 0xd7, - 0x67, 0x34, 0x98, 0xd0, 0x89, 0x7c, 0x9e, 0x8e, 0x53, 0x02, 0xa1, 0x5b, 0x5a, 0xcf, 0x65, 0x23, - 0x0f, 0xbf, 0x0a, 0x35, 0xd2, 0xaf, 0xd0, 0x6d, 0x99, 0xb1, 0x14, 0xb5, 0x03, 0xf9, 0x2d, 0xa1, - 0x9e, 0x2c, 0xa9, 0x89, 0x67, 0x94, 0x58, 0xf7, 0xf6, 0x65, 0xb5, 0xf9, 0x85, 0x17, 0x5d, 0x48, - 0xc4, 0xab, 0x4f, 0x2b, 0xdc, 0xf5, 0x98, 0xf3, 0xd8, 0x58, 0x4c, 0xf4, 0x0e, 0xf8, 0x83, 0x38, - 0x37, 0x09, 0x73, 0xa9, 0xdc, 0x74, 0x06, 0xec, 0x80, 0x6e, 0xe9, 0x22, 0x8a, 0x62, 0x01, 0x22, - 0x4a, 0x31, 0x8f, 0xff, 0x62, 0xc1, 0xdc, 0x35, 0xca, 0x92, 0xd7, 0x93, 0xd7, 0xc8, 0xd8, 0xf8, - 0x23, 0x38, 0x6a, 0xc8, 0xaf, 0xb4, 0xbf, 0x30, 0x76, 0x27, 0x79, 0x23, 0xd6, 0x7f, 0xd5, 0xeb, - 0xd0, 0xc7, 0xaa, 0x97, 0x4b, 0x5e, 0x47, 0xee, 0x40, 0xc9, 0x98, 0x44, 0x57, 0xc6, 0x2e, 0x22, - 0xc6, 0xcb, 0x4b, 0x54, 0x4c, 0xdb, 0x15, 0xa5, 0x93, 0xec, 0xe6, 0xd4, 0x35, 0x33, 0x2a, 0xda, - 0xeb, 0x80, 0xc4, 0x0d, 0x56, 0xb0, 0x35, 0xcb, 0x86, 0xc0, 0xde, 0x88, 0x6e, 0x24, 0x11, 0x8c, - 0x4e, 0x83, 0x1d, 0xf8, 0x8f, 0xf4, 0x0d, 0x73, 0x26, 0xde, 0x92, 0xf8, 0x8f, 0x88, 0x98, 0xc2, - 0x97, 0x21, 0x43, 0xfc, 0x47, 0xa8, 0x0e, 0x10, 0x38, 0x5e, 0x97, 0xde, 0x8b, 0x1a, 0x9b, 0x32, - 0x31, 0x30, 0x87, 0x94, 0xf4, 0x65, 0x38, 0x6a, 0x4a, 0x24, 0xdd, 0xdd, 0x84, 0xfc, 0xc7, 0x43, - 0xd3, 0x5c, 0x95, 0x31, 0x73, 0xc9, 0x1e, 0x59, 0x13, 0xf1, 0x98, 0x81, 0x18, 0x8f, 0x4e, 0x42, - 0x91, 0x39, 0x1b, 0x7d, 0x7a, 0x2b, 0x4e, 0x40, 0x31, 0x82, 0xcf, 0xf2, 0x9e, 0xec, 0x9e, 0x71, - 0x37, 0x89, 0x11, 0xe8, 0x2c, 0xcc, 0xc5, 0x32, 0xdf, 0x09, 0xe8, 0x56, 0xef, 0xb1, 0xf0, 0x70, - 0x99, 0x1c, 0xc0, 0xa3, 0x45, 0x38, 0x12, 0xe3, 0xd6, 0xc5, 0x1d, 0xc0, 0x16, 0xa4, 0xe3, 0x68, - 0x6e, 0x1b, 0xa1, 0xee, 0x87, 0x0f, 0x87, 0x4e, 0x5f, 0x64, 0xd5, 0x32, 0x31, 0x30, 0xf8, 0xaf, - 0x16, 0x1c, 0x95, 0xae, 0xe6, 0xdd, 0xf8, 0xeb, 0x18, 0xf5, 0x9f, 0x59, 0x80, 0x4c, 0x0d, 0x54, - 0x68, 0x7d, 0xc7, 0x7c, 0x66, 0xe1, 0x97, 0x8c, 0x92, 0x68, 0x35, 0x25, 0x2a, 0x7e, 0x29, 0xc1, - 0x90, 0x13, 0x17, 0x15, 0xd9, 0xf3, 0xda, 0xb2, 0x97, 0x95, 0x18, 0xa2, 0xbe, 0xbc, 0x05, 0xdf, - 0xd8, 0x65, 0x34, 0x54, 0x9d, 0xa8, 0x68, 0xc1, 0x05, 0x82, 0xc8, 0x0f, 0xdf, 0x8b, 0x7a, 0x4c, - 0x44, 0x8d, 0x1d, 0xef, 0xa5, 0x50, 0x44, 0x0f, 0xf0, 0x9f, 0xd2, 0x30, 0x73, 0xcf, 0xef, 0x0f, - 0xe3, 0x92, 0xf5, 0x3a, 0xa5, 0xf2, 0x44, 0x7b, 0x9c, 0xd5, 0xed, 0x31, 0x02, 0x3b, 0x64, 0x74, - 0x20, 0x22, 0x2b, 0x43, 0xc4, 0x18, 0x61, 0x28, 0x33, 0x27, 0xe8, 0x52, 0x26, 0xfb, 0x8e, 0x6a, - 0x4e, 0x5c, 0x08, 0x13, 0x38, 0xb4, 0x00, 0x25, 0xa7, 0xdb, 0x0d, 0x68, 0xd7, 0x61, 0xb4, 0xbd, - 0x5b, 0xcd, 0x8b, 0xcd, 0x4c, 0x14, 0xfe, 0x29, 0xcc, 0x6a, 0x63, 0x29, 0x97, 0xbe, 0x0b, 0xf9, - 0x4f, 0x04, 0x66, 0xc2, 0x93, 0x94, 0x24, 0x55, 0x69, 0x4c, 0x93, 0x25, 0xdf, 0xaf, 0xb5, 0xcc, - 0xf8, 0x3a, 0xe4, 0x24, 0x39, 0x3a, 0x69, 0x76, 0x0f, 0xf2, 0xed, 0x84, 0xc3, 0xaa, 0x15, 0xc0, - 0x90, 0x93, 0x8c, 0x94, 0xe3, 0x45, 0x6c, 0x48, 0x0c, 0x51, 0xdf, 0xb3, 0x67, 0xa0, 0x18, 0x3d, - 0x3e, 0xa3, 0x12, 0xe4, 0xaf, 0xde, 0x26, 0x3f, 0xb9, 0x42, 0x56, 0xe6, 0x52, 0xa8, 0x0c, 0x85, - 0xf6, 0x95, 0xe5, 0x1b, 0x02, 0xb2, 0x96, 0xbe, 0xb6, 0x75, 0x66, 0x09, 0xd0, 0x0f, 0x20, 0x2b, - 0xd3, 0xc5, 0xb1, 0x58, 0x7e, 0xf3, 0x99, 0xb7, 0x76, 0xfc, 0x00, 0x5e, 0x5a, 0x00, 0xa7, 0xde, - 0xb5, 0xd0, 0x2d, 0x28, 0x09, 0xa4, 0x7a, 0xd0, 0x39, 0x39, 0xfe, 0xae, 0x92, 0xe0, 0x74, 0xea, - 0x90, 0x59, 0x83, 0xdf, 0x25, 0xc8, 0x0a, 0x9f, 0x98, 0xd2, 0x98, 0x0f, 0x72, 0xa6, 0x34, 0x89, - 0x27, 0x2e, 0x9c, 0x42, 0xdf, 0x07, 0x9b, 0xb7, 0x38, 0xc8, 0x28, 0x2a, 0xc6, 0x3b, 0x4c, 0xed, - 0xd8, 0x38, 0xda, 0xd8, 0xf6, 0x83, 0xe8, 0x39, 0xe9, 0xf8, 0x78, 0x5b, 0xab, 0x97, 0x57, 0x0f, - 0x4e, 0x44, 0x3b, 0xdf, 0x96, 0xef, 0x1e, 0xba, 0xb9, 0x42, 0xa7, 0x92, 0x5b, 0x8d, 0xf5, 0x62, - 0xb5, 0xfa, 0x61, 0xd3, 0x11, 0xc3, 0x35, 0x28, 0x19, 0x8d, 0x8d, 0x69, 0xd6, 0x83, 0x5d, 0x99, - 0x69, 0xd6, 0x09, 0xdd, 0x10, 0x4e, 0xa1, 0x6b, 0x50, 0xe0, 0xa5, 0x98, 0x67, 0x24, 0x74, 0x62, - 0xbc, 0xe2, 0x1a, 0x99, 0xb6, 0x76, 0x72, 0xf2, 0x64, 0xc4, 0xe8, 0x47, 0x50, 0xbc, 0x46, 0x99, - 0x0a, 0xd7, 0xe3, 0xe3, 0xf1, 0x3e, 0xc1, 0x52, 0xc9, 0x33, 0x83, 0x53, 0x4b, 0x3f, 0xd7, 0x7f, - 0x4a, 0xad, 0x38, 0xcc, 0x41, 0xb7, 0x61, 0x56, 0x08, 0x16, 0xfd, 0x6b, 0x95, 0x08, 0xa0, 0x03, - 0x7f, 0x91, 0x25, 0x02, 0xe8, 0xe0, 0x5f, 0x65, 0x38, 0xd5, 0xbe, 0xff, 0xf4, 0x59, 0x3d, 0xf5, - 0xc5, 0xb3, 0x7a, 0xea, 0xab, 0x67, 0x75, 0xeb, 0x97, 0x7b, 0x75, 0xeb, 0x8f, 0x7b, 0x75, 0xeb, - 0xc9, 0x5e, 0xdd, 0x7a, 0xba, 0x57, 0xb7, 0xfe, 0xb5, 0x57, 0xb7, 0xfe, 0xbd, 0x57, 0x4f, 0x7d, - 0xb5, 0x57, 0xb7, 0x3e, 0x7d, 0x5e, 0x4f, 0x3d, 0x7d, 0x5e, 0x4f, 0x7d, 0xf1, 0xbc, 0x9e, 0xba, - 0xff, 0xe6, 0x7f, 0xb9, 0xe8, 0xc9, 0x46, 0x34, 0x27, 0x3e, 0x17, 0xfe, 0x13, 0x00, 0x00, 0xff, - 0xff, 0xb0, 0x19, 0x00, 0xf7, 0x53, 0x1c, 0x00, 0x00, + 0x2d, 0xe6, 0x6b, 0x77, 0x96, 0xa2, 0xdd, 0x50, 0x75, 0x51, 0xf8, 0xc2, 0x9d, 0x79, 0xf3, 0xe6, + 0xcd, 0xfb, 0x9a, 0xf7, 0x31, 0x84, 0x13, 0x83, 0x9d, 0x6e, 0xab, 0xef, 0x77, 0x07, 0x81, 0xcf, + 0xfc, 0x68, 0xd0, 0x14, 0xbf, 0xa8, 0xa0, 0xe7, 0xb5, 0x4a, 0xd7, 0xef, 0xfa, 0x12, 0x87, 0x8f, + 0xe4, 0x7a, 0xad, 0xd1, 0xf5, 0xfd, 0x6e, 0x9f, 0xb6, 0xc4, 0x6c, 0x63, 0xb8, 0xd5, 0x62, 0x3d, + 0x97, 0x86, 0xcc, 0x71, 0x07, 0x0a, 0x61, 0x41, 0x51, 0x7f, 0xd8, 0x77, 0xfd, 0x0e, 0xed, 0xb7, + 0x42, 0xe6, 0xb0, 0x50, 0xfe, 0x2a, 0x8c, 0x79, 0x8e, 0x31, 0x18, 0x86, 0xdb, 0xe2, 0x47, 0x02, + 0x71, 0x05, 0xd0, 0x3a, 0x0b, 0xa8, 0xe3, 0x12, 0x87, 0xd1, 0x90, 0xd0, 0x87, 0x43, 0x1a, 0x32, + 0x7c, 0x13, 0xe6, 0x13, 0xd0, 0x70, 0xe0, 0x7b, 0x21, 0x45, 0x17, 0xa1, 0x14, 0xc6, 0xe0, 0xaa, + 0xb5, 0x90, 0x59, 0x2c, 0x2d, 0x55, 0x9a, 0x91, 0x28, 0xf1, 0x1e, 0x62, 0x22, 0xe2, 0xdf, 0x58, + 0x00, 0xf1, 0x1a, 0xaa, 0x03, 0xc8, 0xd5, 0x8f, 0x9c, 0x70, 0xbb, 0x6a, 0x2d, 0x58, 0x8b, 0x36, + 0x31, 0x20, 0xe8, 0x1c, 0x1c, 0x8d, 0x67, 0xb7, 0xfc, 0xf5, 0x6d, 0x27, 0xe8, 0x54, 0xd3, 0x02, + 0xed, 0xe0, 0x02, 0x42, 0x60, 0x07, 0x0e, 0xa3, 0xd5, 0xcc, 0x82, 0xb5, 0x98, 0x21, 0x62, 0x8c, + 0x8e, 0x41, 0x8e, 0x51, 0xcf, 0xf1, 0x58, 0xd5, 0x5e, 0xb0, 0x16, 0x8b, 0x44, 0xcd, 0x38, 0x9c, + 0xcb, 0x4e, 0xc3, 0x6a, 0x76, 0xc1, 0x5a, 0x9c, 0x21, 0x6a, 0x86, 0x3f, 0xcf, 0x40, 0xf9, 0xe3, + 0x21, 0x0d, 0x76, 0x95, 0x02, 0x50, 0x1d, 0x0a, 0x21, 0xed, 0xd3, 0x4d, 0xe6, 0x07, 0x82, 0xc1, + 0x62, 0x3b, 0x5d, 0xb5, 0x48, 0x04, 0x43, 0x15, 0xc8, 0xf6, 0x7b, 0x6e, 0x8f, 0x09, 0xb6, 0x66, + 0x88, 0x9c, 0xa0, 0x4b, 0x90, 0x0d, 0x99, 0x13, 0x30, 0xc1, 0x4b, 0x69, 0xa9, 0xd6, 0x94, 0x46, + 0x6b, 0x6a, 0xa3, 0x35, 0xef, 0x6a, 0xa3, 0xb5, 0x0b, 0x4f, 0x46, 0x8d, 0xd4, 0x67, 0xff, 0x68, + 0x58, 0x44, 0x6e, 0x41, 0x17, 0x21, 0x43, 0xbd, 0x8e, 0xe0, 0xf7, 0x9b, 0xee, 0xe4, 0x1b, 0xd0, + 0x79, 0x28, 0x76, 0x7a, 0x01, 0xdd, 0x64, 0x3d, 0xdf, 0x13, 0x52, 0xcd, 0x2e, 0xcd, 0xc7, 0x16, + 0x59, 0xd1, 0x4b, 0x24, 0xc6, 0x42, 0xe7, 0x20, 0x17, 0x72, 0xd5, 0x85, 0xd5, 0xfc, 0x42, 0x66, + 0xb1, 0xd8, 0xae, 0xec, 0x8f, 0x1a, 0x73, 0x12, 0x72, 0xce, 0x77, 0x7b, 0x8c, 0xba, 0x03, 0xb6, + 0x4b, 0x14, 0x0e, 0x3a, 0x0b, 0xf9, 0x0e, 0xed, 0x53, 0x6e, 0xf0, 0x82, 0x30, 0xf8, 0x9c, 0x41, + 0x5e, 0x2c, 0x10, 0x8d, 0x80, 0xee, 0x83, 0x3d, 0xe8, 0x3b, 0x5e, 0xb5, 0x28, 0xa4, 0x98, 0x8d, + 0x11, 0xef, 0xf4, 0x1d, 0xaf, 0x7d, 0xf1, 0xcb, 0x51, 0x63, 0xa9, 0xdb, 0x63, 0xdb, 0xc3, 0x8d, + 0xe6, 0xa6, 0xef, 0xb6, 0xba, 0x81, 0xb3, 0xe5, 0x78, 0x4e, 0xab, 0xef, 0xef, 0xf4, 0x5a, 0xdc, + 0x39, 0x1f, 0x0e, 0x69, 0xd0, 0xa3, 0x41, 0x8b, 0xd3, 0x68, 0x0a, 0x7b, 0xf0, 0x7d, 0x44, 0xd0, + 0xbc, 0x6e, 0x17, 0x72, 0x73, 0x79, 0x3c, 0x4a, 0x03, 0x5a, 0x77, 0xdc, 0x41, 0x9f, 0x4e, 0x65, + 0xaf, 0xc8, 0x32, 0xe9, 0x43, 0x5b, 0x26, 0x33, 0xad, 0x65, 0x62, 0x35, 0xdb, 0xd3, 0xa9, 0x39, + 0xfb, 0x4d, 0xd5, 0x9c, 0x7b, 0xf5, 0x6a, 0xc6, 0x55, 0xb0, 0xf9, 0x0c, 0xcd, 0x41, 0x26, 0x70, + 0x1e, 0x09, 0x65, 0x96, 0x09, 0x1f, 0xe2, 0x35, 0xc8, 0x49, 0x46, 0x50, 0x6d, 0x5c, 0xdb, 0xc9, + 0x9b, 0x11, 0x6b, 0x3a, 0xa3, 0x75, 0x38, 0x17, 0xeb, 0x30, 0x23, 0xb4, 0x83, 0x7f, 0x6b, 0xc1, + 0x8c, 0x32, 0xa1, 0x8a, 0x2e, 0x1b, 0x90, 0x97, 0xb7, 0x5b, 0x47, 0x96, 0xe3, 0xe3, 0x91, 0xe5, + 0x4a, 0xc7, 0x19, 0x30, 0x1a, 0xb4, 0x5b, 0x4f, 0x46, 0x0d, 0xeb, 0xcb, 0x51, 0xe3, 0xad, 0x97, + 0x49, 0x29, 0x82, 0x9c, 0x8a, 0x3a, 0x9a, 0x30, 0x7a, 0x5b, 0x70, 0xc7, 0x42, 0xe5, 0x07, 0x47, + 0x9a, 0x32, 0x40, 0xae, 0x7a, 0x5d, 0x1a, 0x72, 0xca, 0x36, 0x37, 0x21, 0x91, 0x38, 0xf8, 0xe7, + 0x30, 0x9f, 0x70, 0x35, 0xc5, 0xe7, 0xfb, 0x90, 0x0b, 0xb9, 0x02, 0x35, 0x9b, 0x86, 0xa1, 0xd6, + 0x05, 0xbc, 0x3d, 0xab, 0xf8, 0xcb, 0xc9, 0x39, 0x51, 0xf8, 0xd3, 0x9d, 0xfe, 0x57, 0x0b, 0xca, + 0x6b, 0xce, 0x06, 0xed, 0x6b, 0x1f, 0x47, 0x60, 0x7b, 0x8e, 0x4b, 0x95, 0xc6, 0xc5, 0x98, 0x07, + 0xb4, 0x4f, 0x9c, 0xfe, 0x90, 0x4a, 0x92, 0x05, 0xa2, 0x66, 0xd3, 0x46, 0x22, 0xeb, 0xd0, 0x91, + 0xc8, 0x8a, 0xfd, 0xbd, 0x02, 0x59, 0xee, 0x59, 0xbb, 0x22, 0x0a, 0x15, 0x89, 0x9c, 0xe0, 0xb7, + 0x60, 0x46, 0x49, 0xa1, 0xd4, 0x17, 0xb3, 0xcc, 0xd5, 0x57, 0xd4, 0x2c, 0x63, 0x17, 0x72, 0x52, + 0xdb, 0xe8, 0x4d, 0x28, 0x46, 0xd9, 0x4d, 0x48, 0x9b, 0x69, 0xe7, 0xf6, 0x47, 0x8d, 0x34, 0x0b, + 0x49, 0xbc, 0x80, 0x1a, 0x90, 0x15, 0x3b, 0x85, 0xe4, 0x56, 0xbb, 0xb8, 0x3f, 0x6a, 0x48, 0x00, + 0x91, 0x1f, 0x74, 0x12, 0xec, 0x6d, 0x9e, 0x60, 0xb8, 0x0a, 0xec, 0x76, 0x61, 0x7f, 0xd4, 0x10, + 0x73, 0x22, 0x7e, 0xf1, 0x35, 0x28, 0xaf, 0xd1, 0xae, 0xb3, 0xb9, 0xab, 0x0e, 0xad, 0x68, 0x72, + 0xfc, 0x40, 0x4b, 0xd3, 0x38, 0x0d, 0xe5, 0xe8, 0xc4, 0x07, 0x6e, 0xa8, 0x9c, 0xba, 0x14, 0xc1, + 0x6e, 0x86, 0xf8, 0xd7, 0x16, 0x28, 0x3b, 0x23, 0x0c, 0xb9, 0x3e, 0x97, 0x35, 0x54, 0x31, 0x08, + 0xf6, 0x47, 0x0d, 0x05, 0x21, 0xea, 0x8b, 0x2e, 0x43, 0x3e, 0x14, 0x27, 0x72, 0x62, 0xe3, 0xee, + 0x23, 0x16, 0xda, 0x47, 0xb8, 0x1b, 0xec, 0x8f, 0x1a, 0x1a, 0x91, 0xe8, 0x01, 0x6a, 0x26, 0x32, + 0xa7, 0x14, 0x6c, 0x76, 0x7f, 0xd4, 0x30, 0xa0, 0x66, 0x26, 0xc5, 0x5f, 0x5b, 0x50, 0xba, 0xeb, + 0xf4, 0x22, 0x17, 0xaa, 0x6a, 0x13, 0xc5, 0x31, 0x52, 0x02, 0xf8, 0x95, 0xee, 0xd0, 0xbe, 0xb3, + 0x7b, 0xd5, 0x0f, 0x04, 0xdd, 0x19, 0x12, 0xcd, 0xe3, 0x64, 0x67, 0x4f, 0x4c, 0x76, 0xd9, 0xe9, + 0x43, 0xea, 0xff, 0x30, 0x80, 0x5d, 0xb7, 0x0b, 0xe9, 0xb9, 0x0c, 0xfe, 0xa3, 0x05, 0x65, 0x29, + 0xb9, 0x72, 0xbb, 0x9f, 0x40, 0x4e, 0x2a, 0x46, 0xc8, 0xfe, 0x92, 0xe0, 0xf2, 0xf6, 0x34, 0x81, + 0x45, 0xd1, 0x44, 0xdf, 0x87, 0xd9, 0x4e, 0xe0, 0x0f, 0x06, 0xb4, 0xb3, 0xae, 0x42, 0x58, 0x7a, + 0x3c, 0x84, 0xad, 0x98, 0xeb, 0x64, 0x0c, 0x1d, 0xff, 0xcd, 0x82, 0x19, 0x15, 0x2d, 0x94, 0xad, + 0x22, 0xfd, 0x5a, 0x87, 0x4e, 0x59, 0xe9, 0x69, 0x53, 0xd6, 0x31, 0xc8, 0x75, 0x03, 0x7f, 0x38, + 0x08, 0xab, 0x19, 0x79, 0x37, 0xe5, 0x6c, 0xba, 0x54, 0x86, 0xaf, 0xc3, 0xac, 0x16, 0xe5, 0x05, + 0x21, 0xb3, 0x36, 0x1e, 0x32, 0x57, 0x3b, 0xd4, 0x63, 0xbd, 0xad, 0x5e, 0x14, 0x04, 0x15, 0x3e, + 0xfe, 0xa5, 0x05, 0x73, 0xe3, 0x28, 0x68, 0xc5, 0xb8, 0x67, 0x9c, 0xdc, 0x99, 0x17, 0x93, 0x6b, + 0x8a, 0xe0, 0x13, 0x7e, 0xe8, 0xb1, 0x60, 0x57, 0x93, 0x96, 0x7b, 0x6b, 0xef, 0x41, 0xc9, 0x58, + 0xe4, 0x29, 0x6a, 0x87, 0xaa, 0x9b, 0x41, 0xf8, 0x30, 0x0e, 0x09, 0x69, 0x19, 0xd0, 0xc4, 0x04, + 0xff, 0xca, 0x82, 0x99, 0x84, 0x2d, 0xd1, 0xfb, 0x60, 0x6f, 0x05, 0xbe, 0x3b, 0x95, 0xa1, 0xc4, + 0x0e, 0xf4, 0x6d, 0x48, 0x33, 0x7f, 0x2a, 0x33, 0xa5, 0x99, 0xcf, 0xad, 0xa4, 0xc4, 0xcf, 0xc8, + 0xea, 0x56, 0xce, 0xf0, 0x7b, 0x50, 0x14, 0x02, 0xdd, 0x71, 0x7a, 0xc1, 0xc4, 0x6c, 0x31, 0x59, + 0xa0, 0xcb, 0x70, 0x44, 0x46, 0xc2, 0xc9, 0x9b, 0xcb, 0x93, 0x36, 0x97, 0xf5, 0xe6, 0x13, 0x90, + 0x5d, 0xde, 0x1e, 0x7a, 0x3b, 0x7c, 0x4b, 0xc7, 0x61, 0x8e, 0xde, 0xc2, 0xc7, 0xf8, 0x0d, 0x98, + 0xe7, 0x77, 0x90, 0x06, 0xe1, 0xb2, 0x3f, 0xf4, 0x98, 0xee, 0x2e, 0xce, 0x41, 0x25, 0x09, 0x56, + 0x5e, 0x52, 0x81, 0xec, 0x26, 0x07, 0x08, 0x1a, 0x33, 0x44, 0x4e, 0xf0, 0xef, 0x2c, 0x40, 0xd7, + 0x28, 0x13, 0xa7, 0xac, 0xae, 0x44, 0xd7, 0xa3, 0x06, 0x05, 0xd7, 0x61, 0x9b, 0xdb, 0x34, 0x08, + 0x75, 0x0d, 0xa2, 0xe7, 0xff, 0x8f, 0x6a, 0x0f, 0x9f, 0x87, 0xf9, 0x04, 0x97, 0x4a, 0xa6, 0x1a, + 0x14, 0x36, 0x15, 0x4c, 0xe5, 0xbb, 0x68, 0x8e, 0xff, 0x94, 0x86, 0x82, 0xd8, 0x40, 0xe8, 0x16, + 0x3a, 0x0f, 0xa5, 0xad, 0x9e, 0xd7, 0xa5, 0xc1, 0x20, 0xe8, 0x29, 0x15, 0xd8, 0xed, 0x23, 0xfb, + 0xa3, 0x86, 0x09, 0x26, 0xe6, 0x04, 0xbd, 0x03, 0xf9, 0x61, 0x48, 0x83, 0x07, 0x3d, 0x79, 0xd3, + 0x8b, 0xed, 0xca, 0xde, 0xa8, 0x91, 0xfb, 0x61, 0x48, 0x83, 0xd5, 0x15, 0x9e, 0x79, 0x86, 0x62, + 0x44, 0xe4, 0xb7, 0x83, 0x6e, 0x28, 0x37, 0x15, 0x45, 0x58, 0xfb, 0x3b, 0x9c, 0xfd, 0xb1, 0x50, + 0x37, 0x08, 0x7c, 0x97, 0xb2, 0x6d, 0x3a, 0x0c, 0x5b, 0x9b, 0xbe, 0xeb, 0xfa, 0x5e, 0x4b, 0xf4, + 0x92, 0x42, 0x68, 0x9e, 0x3e, 0xf9, 0x76, 0xe5, 0xb9, 0x77, 0x21, 0xcf, 0xb6, 0x03, 0x7f, 0xd8, + 0xdd, 0x16, 0x59, 0x21, 0xd3, 0xbe, 0x34, 0x3d, 0x3d, 0x4d, 0x81, 0xe8, 0x01, 0x3a, 0xcd, 0xb5, + 0x45, 0x37, 0x77, 0xc2, 0xa1, 0x2b, 0x3b, 0xb4, 0x76, 0x76, 0x7f, 0xd4, 0xb0, 0xde, 0x21, 0x11, + 0x18, 0x7f, 0x9a, 0x86, 0x86, 0x70, 0xd4, 0x7b, 0xa2, 0x6c, 0xb8, 0xea, 0x07, 0x37, 0x29, 0x0b, + 0x7a, 0x9b, 0xb7, 0x1c, 0x97, 0x6a, 0xdf, 0x68, 0x40, 0xc9, 0x15, 0xc0, 0x07, 0xc6, 0x15, 0x00, + 0x37, 0xc2, 0x43, 0xa7, 0x00, 0xc4, 0x9d, 0x91, 0xeb, 0xf2, 0x36, 0x14, 0x05, 0x44, 0x2c, 0x2f, + 0x27, 0x34, 0xd5, 0x9a, 0x52, 0x32, 0xa5, 0xa1, 0xd5, 0x71, 0x0d, 0x4d, 0x4d, 0x27, 0x52, 0x8b, + 0xe9, 0xeb, 0xd9, 0xa4, 0xaf, 0xe3, 0xbf, 0x5b, 0x50, 0x5f, 0xd3, 0x9c, 0x1f, 0x52, 0x1d, 0x5a, + 0xde, 0xf4, 0x2b, 0x92, 0x37, 0xf3, 0xdf, 0xc9, 0x8b, 0xeb, 0x00, 0x6b, 0x3d, 0x8f, 0x5e, 0xed, + 0xf5, 0x19, 0x0d, 0x26, 0x74, 0x22, 0x9f, 0x66, 0xe2, 0x90, 0x40, 0xe8, 0x96, 0x96, 0x73, 0xd9, + 0x88, 0xc3, 0xaf, 0x42, 0x8c, 0xf4, 0x2b, 0x34, 0x5b, 0x66, 0x2c, 0x44, 0xed, 0x40, 0x7e, 0x4b, + 0x88, 0x27, 0x53, 0x6a, 0xe2, 0x19, 0x25, 0x96, 0xbd, 0x7d, 0x59, 0x1d, 0x7e, 0xe1, 0x65, 0x05, + 0x89, 0x78, 0xf5, 0x69, 0x85, 0xbb, 0x1e, 0x73, 0x1e, 0x1b, 0x9b, 0x89, 0x3e, 0x01, 0xfd, 0x4c, + 0x95, 0x5b, 0xd9, 0x89, 0xe5, 0x96, 0xbe, 0xb9, 0x87, 0xef, 0x19, 0x3f, 0x88, 0x63, 0x9f, 0x30, + 0x87, 0x8a, 0x7d, 0x67, 0xc0, 0x0e, 0xe8, 0x96, 0x4e, 0xd2, 0x28, 0x3e, 0x36, 0xc2, 0x14, 0xeb, + 0xf8, 0xcf, 0x16, 0xcc, 0x5d, 0xa3, 0x2c, 0x59, 0xfe, 0xbc, 0x46, 0xc6, 0xc4, 0x1f, 0xc1, 0x51, + 0x83, 0x7f, 0x25, 0xfd, 0x85, 0xb1, 0x9a, 0xe7, 0x8d, 0x58, 0xfe, 0x55, 0xaf, 0x43, 0x1f, 0xab, + 0x5e, 0x31, 0x59, 0xee, 0xdc, 0x81, 0x92, 0xb1, 0x88, 0xae, 0x8c, 0x15, 0x3a, 0xc6, 0xcb, 0x4e, + 0x94, 0xac, 0xdb, 0x15, 0x25, 0x93, 0xec, 0x16, 0x55, 0x19, 0x1b, 0x15, 0x05, 0xeb, 0x80, 0x84, + 0xb9, 0x04, 0x59, 0x33, 0x2d, 0x09, 0xe8, 0x8d, 0xa8, 0xe2, 0x89, 0xe6, 0xe8, 0x34, 0xd8, 0x81, + 0xff, 0x48, 0x57, 0xb0, 0x33, 0xf1, 0x91, 0xc4, 0x7f, 0x44, 0xc4, 0x12, 0xbe, 0x0c, 0x19, 0xe2, + 0x3f, 0x42, 0x75, 0x80, 0xc0, 0xf1, 0xba, 0xf4, 0x5e, 0xd4, 0x38, 0x95, 0x89, 0x01, 0x79, 0x41, + 0xc9, 0xb0, 0x0c, 0x47, 0x4d, 0x8e, 0xa4, 0xb9, 0x9b, 0x90, 0xff, 0x78, 0x68, 0xaa, 0xab, 0x32, + 0xa6, 0x2e, 0xd9, 0x83, 0x6b, 0x24, 0xee, 0x33, 0x10, 0xc3, 0xd1, 0x49, 0x28, 0x32, 0x67, 0xa3, + 0x4f, 0x6f, 0xc5, 0x01, 0x2e, 0x06, 0xf0, 0x55, 0xde, 0xf3, 0xdd, 0x33, 0x6a, 0x9f, 0x18, 0x80, + 0xce, 0xc2, 0x5c, 0xcc, 0xf3, 0x9d, 0x80, 0x6e, 0xf5, 0x1e, 0x0b, 0x0b, 0x97, 0xc9, 0x01, 0x38, + 0x5a, 0x84, 0x23, 0x31, 0x6c, 0x5d, 0xd4, 0x18, 0xb6, 0x40, 0x1d, 0x07, 0x73, 0xdd, 0x08, 0x71, + 0x3f, 0x7c, 0x38, 0x74, 0xfa, 0xe2, 0xe6, 0x95, 0x89, 0x01, 0xc1, 0x7f, 0xb1, 0xe0, 0xa8, 0x34, + 0x35, 0xef, 0xf6, 0x5f, 0x47, 0xaf, 0xff, 0xdc, 0x02, 0x64, 0x4a, 0xa0, 0x5c, 0xeb, 0x5b, 0xe6, + 0x33, 0x0e, 0x2f, 0x62, 0x4a, 0xa2, 0x95, 0x95, 0xa0, 0xf8, 0x25, 0x06, 0x43, 0x4e, 0x14, 0x42, + 0xb2, 0xa7, 0xb6, 0x65, 0xaf, 0x2c, 0x21, 0x44, 0x7d, 0x79, 0x8b, 0xbf, 0xb1, 0xcb, 0x68, 0xa8, + 0x3a, 0x5d, 0xd1, 0xe2, 0x0b, 0x00, 0x91, 0x1f, 0x7e, 0x16, 0xf5, 0x98, 0xf0, 0x1a, 0x3b, 0x3e, + 0x4b, 0x81, 0x88, 0x1e, 0xe0, 0x3f, 0xa4, 0x61, 0xe6, 0x9e, 0xdf, 0x1f, 0xc6, 0x29, 0xf1, 0x75, + 0x4a, 0x15, 0x89, 0xf6, 0x3b, 0xab, 0xdb, 0x6f, 0x04, 0x76, 0xc8, 0xe8, 0x40, 0x78, 0x56, 0x86, + 0x88, 0x31, 0xc2, 0x50, 0x66, 0x4e, 0xd0, 0xa5, 0x4c, 0xf6, 0x35, 0xd5, 0x9c, 0x28, 0x38, 0x13, + 0x30, 0xb4, 0x00, 0x25, 0xa7, 0xdb, 0x0d, 0x68, 0xd7, 0x61, 0xb4, 0xbd, 0x5b, 0xcd, 0x8b, 0xc3, + 0x4c, 0x10, 0xfe, 0x31, 0xcc, 0x6a, 0x65, 0x29, 0x93, 0xbe, 0x0b, 0xf9, 0x4f, 0x04, 0x64, 0xc2, + 0x93, 0x97, 0x44, 0x55, 0x61, 0x4c, 0xa3, 0x25, 0xdf, 0xc7, 0x35, 0xcf, 0xf8, 0x3a, 0xe4, 0x24, + 0x3a, 0x3a, 0x69, 0x76, 0x27, 0xf2, 0x6d, 0x86, 0xcf, 0x55, 0xab, 0x81, 0x21, 0x27, 0x09, 0x29, + 0xc3, 0x0b, 0xdf, 0x90, 0x10, 0xa2, 0xbe, 0x67, 0xcf, 0x40, 0x31, 0x7a, 0xdc, 0x46, 0x25, 0xc8, + 0x5f, 0xbd, 0x4d, 0x7e, 0x74, 0x85, 0xac, 0xcc, 0xa5, 0x50, 0x19, 0x0a, 0xed, 0x2b, 0xcb, 0x37, + 0xc4, 0xcc, 0x5a, 0xfa, 0xda, 0xd6, 0x91, 0x25, 0x40, 0xdf, 0x83, 0xac, 0x0c, 0x17, 0xc7, 0x62, + 0xfe, 0xcd, 0x67, 0xe4, 0xda, 0xf1, 0x03, 0x70, 0xa9, 0x01, 0x9c, 0x7a, 0xd7, 0x42, 0xb7, 0xa0, + 0x24, 0x80, 0xea, 0xc1, 0xe8, 0xe4, 0xf8, 0xbb, 0x4d, 0x82, 0xd2, 0xa9, 0x17, 0xac, 0x1a, 0xf4, + 0x2e, 0x41, 0x56, 0xd8, 0xc4, 0xe4, 0xc6, 0x7c, 0xf0, 0x33, 0xb9, 0x49, 0x3c, 0xa1, 0xe1, 0x14, + 0xfa, 0x2e, 0xd8, 0xbc, 0x85, 0x42, 0x46, 0x52, 0x31, 0xde, 0x79, 0x6a, 0xc7, 0xc6, 0xc1, 0xc6, + 0xb1, 0x1f, 0x44, 0xcf, 0x55, 0xc7, 0xc7, 0xdb, 0x66, 0xbd, 0xbd, 0x7a, 0x70, 0x21, 0x3a, 0xf9, + 0xb6, 0x7c, 0x57, 0xd1, 0xcd, 0x1b, 0x3a, 0x95, 0x3c, 0x6a, 0xac, 0xd7, 0xab, 0xd5, 0x5f, 0xb4, + 0x1c, 0x11, 0x5c, 0x83, 0x92, 0xd1, 0x38, 0x99, 0x6a, 0x3d, 0xd8, 0xf5, 0x99, 0x6a, 0x9d, 0xd0, + 0x6d, 0xe1, 0x14, 0xba, 0x06, 0x05, 0x9e, 0x8a, 0x79, 0x44, 0x42, 0x27, 0xc6, 0x33, 0xae, 0x11, + 0x69, 0x6b, 0x27, 0x27, 0x2f, 0x46, 0x84, 0x7e, 0x00, 0xc5, 0x6b, 0x94, 0x29, 0x77, 0x3d, 0x3e, + 0xee, 0xef, 0x13, 0x34, 0x95, 0xbc, 0x33, 0x38, 0xb5, 0xf4, 0x53, 0xfd, 0xa7, 0xd7, 0x8a, 0xc3, + 0x1c, 0x74, 0x1b, 0x66, 0x05, 0x63, 0xd1, 0xbf, 0x62, 0x09, 0x07, 0x3a, 0xf0, 0x17, 0x5c, 0xc2, + 0x81, 0x0e, 0xfe, 0x15, 0x87, 0x53, 0xed, 0xfb, 0x4f, 0x9f, 0xd5, 0x53, 0x5f, 0x3c, 0xab, 0xa7, + 0xbe, 0x7a, 0x56, 0xb7, 0x7e, 0xb1, 0x57, 0xb7, 0x7e, 0xbf, 0x57, 0xb7, 0x9e, 0xec, 0xd5, 0xad, + 0xa7, 0x7b, 0x75, 0xeb, 0x9f, 0x7b, 0x75, 0xeb, 0x5f, 0x7b, 0xf5, 0xd4, 0x57, 0x7b, 0x75, 0xeb, + 0xb3, 0xe7, 0xf5, 0xd4, 0xd3, 0xe7, 0xf5, 0xd4, 0x17, 0xcf, 0xeb, 0xa9, 0xfb, 0x6f, 0xfe, 0x87, + 0x42, 0x52, 0x36, 0xba, 0x39, 0xf1, 0xb9, 0xf0, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3e, 0xbe, + 0x5b, 0x4c, 0xb3, 0x1c, 0x00, 0x00, } func (x Direction) String() string { @@ -3772,6 +3775,9 @@ func (this *GetChunkRefRequest) Equal(that interface{}) bool { return false } } + if !this.Plan.Equal(that1.Plan) { + return false + } return true } func (this *GetChunkRefResponse) Equal(that interface{}) bool { @@ -4586,12 +4592,13 @@ func (this *GetChunkRefRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 8) + s := make([]string, 0, 9) s = append(s, "&logproto.GetChunkRefRequest{") s = append(s, "From: "+fmt.Sprintf("%#v", this.From)+",\n") s = append(s, "Through: "+fmt.Sprintf("%#v", this.Through)+",\n") s = append(s, "Matchers: "+fmt.Sprintf("%#v", this.Matchers)+",\n") s = append(s, "Filters: "+fmt.Sprintf("%#v", this.Filters)+",\n") + s = append(s, "Plan: "+fmt.Sprintf("%#v", this.Plan)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -6720,6 +6727,16 @@ func (m *GetChunkRefRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + { + size := m.Plan.Size() + i -= size + if _, err := m.Plan.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintLogproto(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a if len(m.Filters) > 0 { for iNdEx := len(m.Filters) - 1; iNdEx >= 0; iNdEx-- { { @@ -7942,6 +7959,8 @@ func (m *GetChunkRefRequest) Size() (n int) { n += 1 + l + sovLogproto(uint64(l)) } } + l = m.Plan.Size() + n += 1 + l + sovLogproto(uint64(l)) return n } @@ -8620,6 +8639,7 @@ func (this *GetChunkRefRequest) String() string { `Through:` + fmt.Sprintf("%v", this.Through) + `,`, `Matchers:` + fmt.Sprintf("%v", this.Matchers) + `,`, `Filters:` + fmt.Sprintf("%v", this.Filters) + `,`, + `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `}`, }, "") return s @@ -13037,6 +13057,39 @@ func (m *GetChunkRefRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Plan", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogproto + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogproto + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogproto + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Plan.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogproto(dAtA[iNdEx:]) diff --git a/pkg/logproto/logproto.proto b/pkg/logproto/logproto.proto index 0fde83d2715d..bf175168cfd9 100644 --- a/pkg/logproto/logproto.proto +++ b/pkg/logproto/logproto.proto @@ -311,10 +311,15 @@ message GetChunkRefRequest { (gogoproto.nullable) = false ]; string matchers = 3; + // TODO(salvacorts): Delete this field once the weekly release is done. repeated LineFilter filters = 4 [ (gogoproto.customtype) = "github.com/grafana/loki/pkg/logql/syntax.LineFilter", (gogoproto.nullable) = false ]; + Plan plan = 5 [ + (gogoproto.customtype) = "github.com/grafana/loki/pkg/querier/plan.QueryPlan", + (gogoproto.nullable) = false + ]; } message GetChunkRefResponse { diff --git a/pkg/logql/accumulator.go b/pkg/logql/accumulator.go new file mode 100644 index 000000000000..9e9784cb037e --- /dev/null +++ b/pkg/logql/accumulator.go @@ -0,0 +1,379 @@ +package logql + +import ( + "container/heap" + "context" + "fmt" + "sort" + "time" + + "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/logqlmodel/metadata" + "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" + "github.com/grafana/loki/pkg/util/math" +) + +// NewBufferedAccumulator returns an accumulator which aggregates all query +// results in a slice. This is useful for metric queries, which are generally +// small payloads and the memory overhead for buffering is negligible. +func NewBufferedAccumulator(n int) *BufferedAccumulator { + return &BufferedAccumulator{ + results: make([]logqlmodel.Result, n), + } +} + +type BufferedAccumulator struct { + results []logqlmodel.Result +} + +func (a *BufferedAccumulator) Accumulate(_ context.Context, acc logqlmodel.Result, i int) error { + a.results[i] = acc + return nil +} + +func (a *BufferedAccumulator) Result() []logqlmodel.Result { + return a.results +} + +type QuantileSketchAccumulator struct { + matrix ProbabilisticQuantileMatrix +} + +// newQuantileSketchAccumulator returns an accumulator for sharded +// probabilistic quantile queries that merges results as they come in. +func newQuantileSketchAccumulator() *QuantileSketchAccumulator { + return &QuantileSketchAccumulator{} +} + +func (a *QuantileSketchAccumulator) Accumulate(_ context.Context, res logqlmodel.Result, _ int) error { + if res.Data.Type() != QuantileSketchMatrixType { + return fmt.Errorf("unexpected matrix data type: got (%s), want (%s)", res.Data.Type(), QuantileSketchMatrixType) + } + data, ok := res.Data.(ProbabilisticQuantileMatrix) + if !ok { + return fmt.Errorf("unexpected matrix type: got (%T), want (ProbabilisticQuantileMatrix)", res.Data) + } + if a.matrix == nil { + a.matrix = data + return nil + } + + var err error + a.matrix, err = a.matrix.Merge(data) + return err +} + +func (a *QuantileSketchAccumulator) Result() []logqlmodel.Result { + return []logqlmodel.Result{{Data: a.matrix}} +} + +// heap impl for keeping only the top n results across m streams +// importantly, AccumulatedStreams is _bounded_, so it will only +// store the top `limit` results across all streams. +// To implement this, we use a min-heap when looking +// for the max values (logproto.FORWARD) +// and vice versa for logproto.BACKWARD. +// This allows us to easily find the 'worst' value +// and replace it with a better one. +// Once we've fully processed all log lines, +// we return the heap in opposite order and then reverse it +// to get the correct order. +// Heap implements container/heap.Interface +// solely to use heap.Interface as a library. +// It is not intended for the heap pkg functions +// to otherwise call this type. +type AccumulatedStreams struct { + count, limit int + labelmap map[string]int + streams []*logproto.Stream + order logproto.Direction + + stats stats.Result // for accumulating statistics from downstream requests + headers map[string][]string // for accumulating headers from downstream requests +} + +// NewStreamAccumulator returns an accumulator for limited log queries. +// Log queries, sharded thousands of times and each returning +// results, can be _considerably_ larger. In this case, we eagerly +// accumulate the results into a logsAccumulator, discarding values +// over the limit to keep memory pressure down while other subqueries +// are executing. +func NewStreamAccumulator(params Params) *AccumulatedStreams { + // the stream accumulator stores a heap with reversed order + // from the results we expect, so we need to reverse the direction + order := logproto.FORWARD + if params.Direction() == logproto.FORWARD { + order = logproto.BACKWARD + } + + return &AccumulatedStreams{ + labelmap: make(map[string]int), + order: order, + limit: int(params.Limit()), + + headers: make(map[string][]string), + } +} + +// returns the top priority +func (acc *AccumulatedStreams) top() (time.Time, bool) { + if len(acc.streams) > 0 && len(acc.streams[0].Entries) > 0 { + return acc.streams[0].Entries[len(acc.streams[0].Entries)-1].Timestamp, true + } + return time.Time{}, false +} + +func (acc *AccumulatedStreams) Find(labels string) (int, bool) { + i, ok := acc.labelmap[labels] + return i, ok +} + +// number of streams +func (acc *AccumulatedStreams) Len() int { return len(acc.streams) } + +func (acc *AccumulatedStreams) Swap(i, j int) { + // for i=0, j=1 + + // {'a': 0, 'b': 1} + // [a, b] + acc.streams[i], acc.streams[j] = acc.streams[j], acc.streams[i] + // {'a': 0, 'b': 1} + // [b, a] + acc.labelmap[acc.streams[i].Labels] = i + acc.labelmap[acc.streams[j].Labels] = j + // {'a': 1, 'b': 0} + // [b, a] +} + +// first order by timestamp, then by labels +func (acc *AccumulatedStreams) Less(i, j int) bool { + // order by the 'oldest' entry in the stream + if a, b := acc.streams[i].Entries[len(acc.streams[i].Entries)-1].Timestamp, acc.streams[j].Entries[len(acc.streams[j].Entries)-1].Timestamp; !a.Equal(b) { + return acc.less(a, b) + } + return acc.streams[i].Labels <= acc.streams[j].Labels +} + +func (acc *AccumulatedStreams) less(a, b time.Time) bool { + // use after for stable sort + if acc.order == logproto.FORWARD { + return !a.After(b) + } + return !b.After(a) +} + +func (acc *AccumulatedStreams) Push(x any) { + s := x.(*logproto.Stream) + if len(s.Entries) == 0 { + return + } + + if room := acc.limit - acc.count; room >= len(s.Entries) { + if i, ok := acc.Find(s.Labels); ok { + // stream already exists, append entries + + // these are already guaranteed to be sorted + // Reasoning: we shard subrequests so each stream exists on only one + // shard. Therefore, the only time a stream should already exist + // is in successive splits, which are already guaranteed to be ordered + // and we can just append. + acc.appendTo(acc.streams[i], s) + + return + } + + // new stream + acc.addStream(s) + return + } + + // there's not enough room for all the entries, + // so we need to + acc.push(s) +} + +// there's not enough room for all the entries. +// since we store them in a reverse heap relative to what we _want_ +// (i.e. the max value for FORWARD, the min value for BACKWARD), +// we test if the new entry is better than the worst entry, +// swapping them if so. +func (acc *AccumulatedStreams) push(s *logproto.Stream) { + worst, ok := acc.top() + room := math.Min(acc.limit-acc.count, len(s.Entries)) + + if !ok { + if room == 0 { + // special case: limit must be zero since there's no room and no worst entry + return + } + s.Entries = s.Entries[:room] + // special case: there are no entries in the heap. Push entries up to the limit + acc.addStream(s) + return + } + + // since entries are sorted by timestamp from best -> worst, + // we can discard the entire stream if the incoming best entry + // is worse than the worst entry in the heap. + cutoff := sort.Search(len(s.Entries), func(i int) bool { + // TODO(refactor label comparison -- should be in another fn) + if worst.Equal(s.Entries[i].Timestamp) { + return acc.streams[0].Labels < s.Labels + } + return acc.less(s.Entries[i].Timestamp, worst) + }) + s.Entries = s.Entries[:cutoff] + + for i := 0; i < len(s.Entries) && acc.less(worst, s.Entries[i].Timestamp); i++ { + + // push one entry at a time + room = acc.limit - acc.count + // pop if there's no room to make the heap small enough for an append; + // in the short path of Push() we know that there's room for at least one entry + if room == 0 { + acc.Pop() + } + + cpy := *s + cpy.Entries = []logproto.Entry{s.Entries[i]} + acc.Push(&cpy) + + // update worst + worst, _ = acc.top() + } +} + +func (acc *AccumulatedStreams) addStream(s *logproto.Stream) { + // ensure entries conform to order we expect + // TODO(owen-d): remove? should be unnecessary since we insert in appropriate order + // but it's nice to have the safeguard + sort.Slice(s.Entries, func(i, j int) bool { + return acc.less(s.Entries[j].Timestamp, s.Entries[i].Timestamp) + }) + + acc.streams = append(acc.streams, s) + i := len(acc.streams) - 1 + acc.labelmap[s.Labels] = i + acc.count += len(s.Entries) + heap.Fix(acc, i) +} + +// dst must already exist in acc +func (acc *AccumulatedStreams) appendTo(dst, src *logproto.Stream) { + // these are already guaranteed to be sorted + // Reasoning: we shard subrequests so each stream exists on only one + // shard. Therefore, the only time a stream should already exist + // is in successive splits, which are already guaranteed to be ordered + // and we can just append. + + var needsSort bool + for _, e := range src.Entries { + // sort if order has broken + if len(dst.Entries) > 0 && acc.less(dst.Entries[len(dst.Entries)-1].Timestamp, e.Timestamp) { + needsSort = true + } + dst.Entries = append(dst.Entries, e) + } + + if needsSort { + sort.Slice(dst.Entries, func(i, j int) bool { + // store in reverse order so we can more reliably insert without sorting and pop from end + return acc.less(dst.Entries[j].Timestamp, dst.Entries[i].Timestamp) + }) + } + + acc.count += len(src.Entries) + heap.Fix(acc, acc.labelmap[dst.Labels]) + +} + +// Pop returns a stream with one entry. It pops the first entry of the first stream +func (acc *AccumulatedStreams) Pop() any { + n := acc.Len() + if n == 0 { + return nil + } + + stream := acc.streams[0] + cpy := *stream + cpy.Entries = []logproto.Entry{cpy.Entries[len(stream.Entries)-1]} + stream.Entries = stream.Entries[:len(stream.Entries)-1] + + acc.count-- + + if len(stream.Entries) == 0 { + // remove stream + acc.Swap(0, n-1) + acc.streams[n-1] = nil // avoid leaking reference + delete(acc.labelmap, stream.Labels) + acc.streams = acc.streams[:n-1] + + } + + if acc.Len() > 0 { + heap.Fix(acc, 0) + } + + return &cpy +} + +// Note: can only be called once as it will alter stream ordreing. +func (acc *AccumulatedStreams) Result() []logqlmodel.Result { + // sort streams by label + sort.Slice(acc.streams, func(i, j int) bool { + return acc.streams[i].Labels < acc.streams[j].Labels + }) + + streams := make(logqlmodel.Streams, 0, len(acc.streams)) + + for _, s := range acc.streams { + // sort entries by timestamp, inversely based on direction + sort.Slice(s.Entries, func(i, j int) bool { + return acc.less(s.Entries[j].Timestamp, s.Entries[i].Timestamp) + }) + streams = append(streams, *s) + } + + res := logqlmodel.Result{ + // stats & headers are already aggregated in the context + Data: streams, + Statistics: acc.stats, + Headers: make([]*definitions.PrometheusResponseHeader, 0, len(acc.headers)), + } + + for name, vals := range acc.headers { + res.Headers = append( + res.Headers, + &definitions.PrometheusResponseHeader{ + Name: name, + Values: vals, + }, + ) + } + + return []logqlmodel.Result{res} +} + +func (acc *AccumulatedStreams) Accumulate(_ context.Context, x logqlmodel.Result, _ int) error { + // TODO(owen-d/ewelch): Shard counts should be set by the querier + // so we don't have to do it in tricky ways in multiple places. + // See pkg/logql/downstream.go:DownstreamEvaluator.Downstream + // for another example. + if x.Statistics.Summary.Shards == 0 { + x.Statistics.Summary.Shards = 1 + } + acc.stats.Merge(x.Statistics) + metadata.ExtendHeaders(acc.headers, x.Headers) + + switch got := x.Data.(type) { + case logqlmodel.Streams: + for i := range got { + acc.Push(&got[i]) + } + default: + return fmt.Errorf("unexpected response type during response result accumulation. Got (%T), wanted %s", got, logqlmodel.ValueTypeStreams) + } + return nil +} diff --git a/pkg/logql/accumulator_test.go b/pkg/logql/accumulator_test.go new file mode 100644 index 000000000000..d827e3ea02e7 --- /dev/null +++ b/pkg/logql/accumulator_test.go @@ -0,0 +1,273 @@ +package logql + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/sketch" + "github.com/grafana/loki/pkg/logqlmodel" +) + +func TestAccumulatedStreams(t *testing.T) { + lim := 30 + nStreams := 10 + start, end := 0, 10 + // for a logproto.BACKWARD query, we use a min heap based on FORWARD + // to store the _earliest_ timestamp of the _latest_ entries, up to `limit` + xs := newStreams(time.Unix(int64(start), 0), time.Unix(int64(end), 0), time.Second, nStreams, logproto.BACKWARD) + acc := NewStreamAccumulator(LiteralParams{ + direction: logproto.BACKWARD, + limit: uint32(lim), + }) + for _, x := range xs { + acc.Push(x) + } + + for i := 0; i < lim; i++ { + got := acc.Pop().(*logproto.Stream) + require.Equal(t, fmt.Sprintf(`{n="%d"}`, i%nStreams), got.Labels) + exp := (nStreams*(end-start) - lim + i) / nStreams + require.Equal(t, time.Unix(int64(exp), 0), got.Entries[0].Timestamp) + } + +} + +func TestDownstreamAccumulatorSimple(t *testing.T) { + lim := 30 + start, end := 0, 10 + direction := logproto.BACKWARD + + streams := newStreams(time.Unix(int64(start), 0), time.Unix(int64(end), 0), time.Second, 10, direction) + x := make(logqlmodel.Streams, 0, len(streams)) + for _, s := range streams { + x = append(x, *s) + } + // dummy params. Only need to populate direction & limit + params, err := NewLiteralParams( + `{app="foo"}`, time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, + ) + require.NoError(t, err) + + acc := NewStreamAccumulator(params) + result := logqlmodel.Result{ + Data: x, + } + + require.Nil(t, acc.Accumulate(context.Background(), result, 0)) + + res := acc.Result()[0] + got, ok := res.Data.(logqlmodel.Streams) + require.Equal(t, true, ok) + require.Equal(t, 10, len(got), "correct number of streams") + + // each stream should have the top 3 entries + for i := 0; i < 10; i++ { + require.Equal(t, 3, len(got[i].Entries), "correct number of entries in stream") + for j := 0; j < 3; j++ { + require.Equal(t, time.Unix(int64(9-j), 0), got[i].Entries[j].Timestamp, "correct timestamp") + } + } +} + +// TestDownstreamAccumulatorMultiMerge simulates merging multiple +// sub-results from different queries. +func TestDownstreamAccumulatorMultiMerge(t *testing.T) { + for _, direction := range []logproto.Direction{logproto.BACKWARD, logproto.FORWARD} { + t.Run(direction.String(), func(t *testing.T) { + nQueries := 10 + delta := 10 // 10 entries per stream, 1s apart + streamsPerQuery := 10 + lim := 30 + + payloads := make([]logqlmodel.Streams, 0, nQueries) + for i := 0; i < nQueries; i++ { + start := i * delta + end := start + delta + streams := newStreams(time.Unix(int64(start), 0), time.Unix(int64(end), 0), time.Second, streamsPerQuery, direction) + var res logqlmodel.Streams + for i := range streams { + res = append(res, *streams[i]) + } + payloads = append(payloads, res) + + } + + // queries are always dispatched in the correct order. + // oldest time ranges first in the case of logproto.FORWARD + // and newest time ranges first in the case of logproto.BACKWARD + if direction == logproto.BACKWARD { + for i, j := 0, len(payloads)-1; i < j; i, j = i+1, j-1 { + payloads[i], payloads[j] = payloads[j], payloads[i] + } + } + + // dummy params. Only need to populate direction & limit + params, err := NewLiteralParams( + `{app="foo"}`, time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, + ) + require.NoError(t, err) + + acc := NewStreamAccumulator(params) + for i := 0; i < nQueries; i++ { + err := acc.Accumulate(context.Background(), logqlmodel.Result{ + Data: payloads[i], + }, i) + require.Nil(t, err) + } + + got, ok := acc.Result()[0].Data.(logqlmodel.Streams) + require.Equal(t, true, ok) + require.Equal(t, int64(nQueries), acc.Result()[0].Statistics.Summary.Shards) + + // each stream should have the top 3 entries + for i := 0; i < streamsPerQuery; i++ { + stream := got[i] + require.Equal(t, fmt.Sprintf(`{n="%d"}`, i), stream.Labels, "correct labels") + ln := lim / streamsPerQuery + require.Equal(t, ln, len(stream.Entries), "correct number of entries in stream") + switch direction { + case logproto.BACKWARD: + for i := 0; i < ln; i++ { + offset := delta*nQueries - 1 - i + require.Equal(t, time.Unix(int64(offset), 0), stream.Entries[i].Timestamp, "correct timestamp") + } + default: + for i := 0; i < ln; i++ { + offset := i + require.Equal(t, time.Unix(int64(offset), 0), stream.Entries[i].Timestamp, "correct timestamp") + } + } + } + }) + } +} + +func BenchmarkAccumulator(b *testing.B) { + + // dummy params. Only need to populate direction & limit + lim := 30 + params, err := NewLiteralParams( + `{app="foo"}`, time.Time{}, time.Time{}, 0, 0, logproto.BACKWARD, uint32(lim), nil, + ) + require.NoError(b, err) + + for acc, tc := range map[string]struct { + results []logqlmodel.Result + newAcc func(Params, []logqlmodel.Result) Accumulator + params Params + }{ + "streams": { + newStreamResults(), + func(p Params, _ []logqlmodel.Result) Accumulator { + return NewStreamAccumulator(p) + }, + params, + }, + "quantile sketches": { + newQuantileSketchResults(), + func(p Params, _ []logqlmodel.Result) Accumulator { + return newQuantileSketchAccumulator() + }, + params, + }, + } { + b.Run(acc, func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for n := 0; n < b.N; n++ { + + acc := tc.newAcc(params, tc.results) + for i, r := range tc.results { + err := acc.Accumulate(context.Background(), r, i) + require.Nil(b, err) + } + + acc.Result() + } + }) + } +} + +func newStreamResults() []logqlmodel.Result { + nQueries := 50 + delta := 100 // 10 entries per stream, 1s apart + streamsPerQuery := 50 + + results := make([]logqlmodel.Result, nQueries) + for i := 0; i < nQueries; i++ { + start := i * delta + end := start + delta + streams := newStreams(time.Unix(int64(start), 0), time.Unix(int64(end), 0), time.Second, streamsPerQuery, logproto.BACKWARD) + var res logqlmodel.Streams + for i := range streams { + res = append(res, *streams[i]) + } + results[i] = logqlmodel.Result{Data: res} + + } + + return results +} + +func newQuantileSketchResults() []logqlmodel.Result { + results := make([]logqlmodel.Result, 100) + + for r := range results { + vectors := make([]ProbabilisticQuantileVector, 10) + for i := range vectors { + vectors[i] = make(ProbabilisticQuantileVector, 10) + for j := range vectors[i] { + vectors[i][j] = ProbabilisticQuantileSample{ + T: int64(i), + F: newRandomSketch(), + Metric: []labels.Label{{Name: "foo", Value: fmt.Sprintf("bar-%d", j)}}, + } + } + } + results[r] = logqlmodel.Result{Data: ProbabilisticQuantileMatrix(vectors)} + } + + return results +} + +func newStreamWithDirection(start, end time.Time, delta time.Duration, ls string, direction logproto.Direction) *logproto.Stream { + s := &logproto.Stream{ + Labels: ls, + } + for t := start; t.Before(end); t = t.Add(delta) { + s.Entries = append(s.Entries, logproto.Entry{ + Timestamp: t, + Line: fmt.Sprintf("%d", t.Unix()), + }) + } + if direction == logproto.BACKWARD { + // simulate data coming in reverse order (logproto.BACKWARD) + for i, j := 0, len(s.Entries)-1; i < j; i, j = i+1, j-1 { + s.Entries[i], s.Entries[j] = s.Entries[j], s.Entries[i] + } + } + return s +} + +func newStreams(start, end time.Time, delta time.Duration, n int, direction logproto.Direction) (res []*logproto.Stream) { + for i := 0; i < n; i++ { + res = append(res, newStreamWithDirection(start, end, delta, fmt.Sprintf(`{n="%d"}`, i), direction)) + } + return res +} + +func newRandomSketch() sketch.QuantileSketch { + r := rand.New(rand.NewSource(42)) + s := sketch.NewDDSketch() + for i := 0; i < 1000; i++ { + _ = s.Add(r.Float64()) + } + return s +} diff --git a/pkg/logql/downstream.go b/pkg/logql/downstream.go index 76594dc040c2..6946c06e54a0 100644 --- a/pkg/logql/downstream.go +++ b/pkg/logql/downstream.go @@ -83,6 +83,29 @@ func (d DownstreamSampleExpr) String() string { return fmt.Sprintf("downstream<%s, shard=%s>", d.SampleExpr.String(), d.shard) } +// The DownstreamSampleExpr is not part of LogQL. In the prettified version it's +// represented as e.g. `downstream` +func (d DownstreamSampleExpr) Pretty(level int) string { + s := syntax.Indent(level) + if !syntax.NeedSplit(d) { + return s + d.String() + } + + s += "downstream<\n" + + s += d.SampleExpr.Pretty(level + 1) + s += ",\n" + s += syntax.Indent(level+1) + "shard=" + if d.shard != nil { + s += d.shard.String() + "\n" + } else { + s += "nil\n" + } + + s += syntax.Indent(level) + ">" + return s +} + // DownstreamLogSelectorExpr is a LogSelectorExpr which signals downstream computation type DownstreamLogSelectorExpr struct { shard *astmapper.ShardAnnotation @@ -93,6 +116,29 @@ func (d DownstreamLogSelectorExpr) String() string { return fmt.Sprintf("downstream<%s, shard=%s>", d.LogSelectorExpr.String(), d.shard) } +// The DownstreamLogSelectorExpr is not part of LogQL. In the prettified version it's +// represented as e.g. `downstream<{foo="bar"} |= "error", shard=1_of_3>` +func (d DownstreamLogSelectorExpr) Pretty(level int) string { + s := syntax.Indent(level) + if !syntax.NeedSplit(d) { + return s + d.String() + } + + s += "downstream<\n" + + s += d.LogSelectorExpr.Pretty(level + 1) + s += ",\n" + s += syntax.Indent(level+1) + "shard=" + if d.shard != nil { + s += d.shard.String() + "\n" + } else { + s += "nil\n" + } + + s += syntax.Indent(level) + ">" + return s +} + func (d DownstreamSampleExpr) Walk(f syntax.WalkFn) { f(d) } var defaultMaxDepth = 4 @@ -105,7 +151,7 @@ type ConcatSampleExpr struct { next *ConcatSampleExpr } -func (c ConcatSampleExpr) String() string { +func (c *ConcatSampleExpr) String() string { if c.next == nil { return c.DownstreamSampleExpr.String() } @@ -115,7 +161,7 @@ func (c ConcatSampleExpr) String() string { // in order to not display huge queries with thousands of shards, // we can limit the number of stringified subqueries. -func (c ConcatSampleExpr) string(maxDepth int) string { +func (c *ConcatSampleExpr) string(maxDepth int) string { if c.next == nil { return c.DownstreamSampleExpr.String() } @@ -125,18 +171,46 @@ func (c ConcatSampleExpr) string(maxDepth int) string { return fmt.Sprintf("%s ++ %s", c.DownstreamSampleExpr.String(), c.next.string(maxDepth-1)) } -func (c ConcatSampleExpr) Walk(f syntax.WalkFn) { +func (c *ConcatSampleExpr) Walk(f syntax.WalkFn) { f(c) f(c.next) } +// ConcatSampleExpr has no LogQL repretenstation. It is expressed in in the +// prettified version as e.g. `concat(downstream ++ )` +func (c *ConcatSampleExpr) Pretty(level int) string { + s := syntax.Indent(level) + if !syntax.NeedSplit(c) { + return s + c.String() + } + + s += "concat(\n" + + head := c + for i := 0; i < defaultMaxDepth && head != nil; i++ { + if i > 0 { + s += syntax.Indent(level+1) + "++\n" + } + s += head.DownstreamSampleExpr.Pretty(level + 1) + s += "\n" + head = head.next + } + // There are more downstream samples... + if head != nil { + s += syntax.Indent(level+1) + "++ ...\n" + } + s += syntax.Indent(level) + ")" + + return s +} + // ConcatLogSelectorExpr is an expr for concatenating multiple LogSelectorExpr type ConcatLogSelectorExpr struct { DownstreamLogSelectorExpr next *ConcatLogSelectorExpr } -func (c ConcatLogSelectorExpr) String() string { +func (c *ConcatLogSelectorExpr) String() string { if c.next == nil { return c.DownstreamLogSelectorExpr.String() } @@ -146,7 +220,7 @@ func (c ConcatLogSelectorExpr) String() string { // in order to not display huge queries with thousands of shards, // we can limit the number of stringified subqueries. -func (c ConcatLogSelectorExpr) string(maxDepth int) string { +func (c *ConcatLogSelectorExpr) string(maxDepth int) string { if c.next == nil { return c.DownstreamLogSelectorExpr.String() } @@ -156,6 +230,34 @@ func (c ConcatLogSelectorExpr) string(maxDepth int) string { return fmt.Sprintf("%s ++ %s", c.DownstreamLogSelectorExpr.String(), c.next.string(maxDepth-1)) } +// ConcatLogSelectorExpr has no representation in LogQL. Its prettified version +// is e.g. `concat(downstream<{foo="bar"} |= "error", shard=1_of_3>)` +func (c *ConcatLogSelectorExpr) Pretty(level int) string { + s := syntax.Indent(level) + if !syntax.NeedSplit(c) { + return s + c.String() + } + + s += "concat(\n" + + head := c + for i := 0; i < defaultMaxDepth && head != nil; i++ { + if i > 0 { + s += syntax.Indent(level+1) + "++\n" + } + s += head.DownstreamLogSelectorExpr.Pretty(level + 1) + s += "\n" + head = head.next + } + // There are more downstream samples... + if head != nil { + s += syntax.Indent(level+1) + "++ ...\n" + } + s += ")" + + return s +} + // QuantileSketchEvalExpr evaluates a quantile sketch to the actual quantile. type QuantileSketchEvalExpr struct { syntax.SampleExpr @@ -244,7 +346,13 @@ type Resp struct { // Downstreamer is an interface for deferring responsibility for query execution. // It is decoupled from but consumed by a downStreamEvaluator to dispatch ASTs. type Downstreamer interface { - Downstream(context.Context, []DownstreamQuery) ([]logqlmodel.Result, error) + Downstream(context.Context, []DownstreamQuery, Accumulator) ([]logqlmodel.Result, error) +} + +// Accumulator is an interface for accumulating query results. +type Accumulator interface { + Accumulate(context.Context, logqlmodel.Result, int) error + Result() []logqlmodel.Result } // DownstreamEvaluator is an evaluator which handles shard aware AST nodes @@ -254,8 +362,8 @@ type DownstreamEvaluator struct { } // Downstream runs queries and collects stats from the embedded Downstreamer -func (ev DownstreamEvaluator) Downstream(ctx context.Context, queries []DownstreamQuery) ([]logqlmodel.Result, error) { - results, err := ev.Downstreamer.Downstream(ctx, queries) +func (ev DownstreamEvaluator) Downstream(ctx context.Context, queries []DownstreamQuery, acc Accumulator) ([]logqlmodel.Result, error) { + results, err := ev.Downstreamer.Downstream(ctx, queries, acc) if err != nil { return nil, err } @@ -314,12 +422,13 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( if e.shard != nil { shards = append(shards, *e.shard) } + acc := NewBufferedAccumulator(1) results, err := ev.Downstream(ctx, []DownstreamQuery{{ Params: ParamsWithShardsOverride{ Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: e.SampleExpr}, ShardsOverride: Shards(shards).Encode(), }, - }}) + }}, acc) if err != nil { return nil, err } @@ -339,7 +448,8 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( cur = cur.next } - results, err := ev.Downstream(ctx, queries) + acc := NewBufferedAccumulator(len(queries)) + results, err := ev.Downstream(ctx, queries, acc) if err != nil { return nil, err } @@ -379,7 +489,8 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( } } - results, err := ev.Downstream(ctx, queries) + acc := newQuantileSketchAccumulator() + results, err := ev.Downstream(ctx, queries, acc) if err != nil { return nil, err } @@ -413,12 +524,13 @@ func (ev *DownstreamEvaluator) NewIterator( if e.shard != nil { shards = append(shards, *e.shard) } + acc := NewStreamAccumulator(params) results, err := ev.Downstream(ctx, []DownstreamQuery{{ Params: ParamsWithShardsOverride{ Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: e.LogSelectorExpr}, ShardsOverride: shards.Encode(), }, - }}) + }}, acc) if err != nil { return nil, err } @@ -438,7 +550,8 @@ func (ev *DownstreamEvaluator) NewIterator( cur = cur.next } - results, err := ev.Downstream(ctx, queries) + acc := NewStreamAccumulator(params) + results, err := ev.Downstream(ctx, queries, acc) if err != nil { return nil, err } @@ -523,6 +636,10 @@ func NewResultStepEvaluator(res logqlmodel.Result, params Params) (StepEvaluator step = params.Step() ) + if res.Data == nil { + return nil, fmt.Errorf("data in the passed result is nil (res.Data), cannot be processed by stepevaluator") + } + switch data := res.Data.(type) { case promql.Vector: return NewVectorStepEvaluator(start, data), nil diff --git a/pkg/logql/downstream_test.go b/pkg/logql/downstream_test.go index 426722a55459..46575c44d8ed 100644 --- a/pkg/logql/downstream_test.go +++ b/pkg/logql/downstream_test.go @@ -8,12 +8,14 @@ import ( "github.com/go-kit/log" "github.com/grafana/dskit/user" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/astmapper" ) var nilShardMetrics = NewShardMapperMetrics(nil) @@ -144,7 +146,7 @@ func TestMappingEquivalenceSketches(t *testing.T) { regular := NewEngine(opts, q, NoLimits, log.NewNopLogger()) sharded := NewDownstreamEngine(opts, MockDownstreamer{regular}, NoLimits, log.NewNopLogger()) - t.Run(tc.query, func(t *testing.T) { + t.Run(tc.query+"_range", func(t *testing.T) { params, err := NewLiteralParams( tc.query, start, @@ -176,6 +178,40 @@ func TestMappingEquivalenceSketches(t *testing.T) { relativeError(t, res.Data.(promql.Matrix), shardedRes.Data.(promql.Matrix), tc.realtiveError) }) + t.Run(tc.query+"_instant", func(t *testing.T) { + // for an instant query we set the start and end to the same timestamp + // plus set step and interval to 0 + params, err := NewLiteralParams( + tc.query, + time.Unix(0, int64(rounds+1)), + time.Unix(0, int64(rounds+1)), + 0, + 0, + logproto.FORWARD, + uint32(limit), + nil, + ) + require.NoError(t, err) + qry := regular.Query(params) + ctx := user.InjectOrgID(context.Background(), "fake") + + mapper := NewShardMapper(ConstantShards(shards), nilShardMetrics, []string{ShardQuantileOverTime}) + _, _, mapped, err := mapper.Parse(params.GetExpression()) + require.NoError(t, err) + + shardedQry := sharded.Query(ctx, ParamsWithExpressionOverride{ + Params: params, + ExpressionOverride: mapped, + }) + + res, err := qry.Exec(ctx) + require.NoError(t, err) + + shardedRes, err := shardedQry.Exec(ctx) + require.NoError(t, err) + + relativeErrorVector(t, res.Data.(promql.Vector), shardedRes.Data.(promql.Vector), tc.realtiveError) + }) } } @@ -543,3 +579,157 @@ func relativeError(t *testing.T, expected, actual promql.Matrix, alpha float64) require.InEpsilonSlice(t, e, a, alpha) } } + +func relativeErrorVector(t *testing.T, expected, actual promql.Vector, alpha float64) { + require.Len(t, actual, len(expected)) + + e := make([]float64, len(expected)) + a := make([]float64, len(expected)) + for i := 0; i < len(expected); i++ { + require.Equal(t, expected[i].Metric, actual[i].Metric) + + e[i] = expected[i].F + a[i] = expected[i].F + } + require.InEpsilonSlice(t, e, a, alpha) + +} + +func TestFormat_ShardedExpr(t *testing.T) { + oldMax := syntax.MaxCharsPerLine + syntax.MaxCharsPerLine = 20 + + oldDefaultDepth := defaultMaxDepth + defaultMaxDepth = 2 + defer func() { + syntax.MaxCharsPerLine = oldMax + defaultMaxDepth = oldDefaultDepth + }() + + cases := []struct { + name string + in syntax.Expr + exp string + }{ + { + name: "ConcatSampleExpr", + in: &ConcatSampleExpr{ + DownstreamSampleExpr: DownstreamSampleExpr{ + shard: &astmapper.ShardAnnotation{ + Shard: 0, + Of: 3, + }, + SampleExpr: &syntax.RangeAggregationExpr{ + Operation: syntax.OpRangeTypeRate, + Left: &syntax.LogRange{ + Left: &syntax.MatchersExpr{ + Mts: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}, + }, + Interval: time.Minute, + }, + }, + }, + next: &ConcatSampleExpr{ + DownstreamSampleExpr: DownstreamSampleExpr{ + shard: &astmapper.ShardAnnotation{ + Shard: 1, + Of: 3, + }, + SampleExpr: &syntax.RangeAggregationExpr{ + Operation: syntax.OpRangeTypeRate, + Left: &syntax.LogRange{ + Left: &syntax.MatchersExpr{ + Mts: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}, + }, + Interval: time.Minute, + }, + }, + }, + next: &ConcatSampleExpr{ + DownstreamSampleExpr: DownstreamSampleExpr{ + shard: &astmapper.ShardAnnotation{ + Shard: 1, + Of: 3, + }, + SampleExpr: &syntax.RangeAggregationExpr{ + Operation: syntax.OpRangeTypeRate, + Left: &syntax.LogRange{ + Left: &syntax.MatchersExpr{ + Mts: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}, + }, + Interval: time.Minute, + }, + }, + }, + next: nil, + }, + }, + }, + exp: `concat( + downstream< + rate( + {foo="bar"} [1m] + ), + shard=0_of_3 + > + ++ + downstream< + rate( + {foo="bar"} [1m] + ), + shard=1_of_3 + > + ++ ... +)`, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := syntax.Prettify(c.in) + assert.Equal(t, c.exp, got) + }) + } +} + +func TestPrettierWithoutShards(t *testing.T) { + q := `((quantile_over_time(0.5,{foo="bar"} | json | unwrap bytes[1d]) by (cluster) > 42) and (count by (cluster)(max_over_time({foo="baz"} |= "error" | json | unwrap bytes[1d]) by (cluster,namespace)) > 10))` + e := syntax.MustParseExpr(q) + + mapper := NewShardMapper(ConstantShards(4), nilShardMetrics, []string{}) + _, _, mapped, err := mapper.Parse(e) + require.NoError(t, err) + got := syntax.Prettify(mapped) + expected := ` downstream> + > + 42 +and + count by (cluster)( + max by (cluster, namespace)( + concat( + downstream< + max_over_time({foo="baz"} |= "error" | json | unwrap bytes[1d]) by (cluster,namespace), + shard=0_of_4 + > + ++ + downstream< + max_over_time({foo="baz"} |= "error" | json | unwrap bytes[1d]) by (cluster,namespace), + shard=1_of_4 + > + ++ + downstream< + max_over_time({foo="baz"} |= "error" | json | unwrap bytes[1d]) by (cluster,namespace), + shard=2_of_4 + > + ++ + downstream< + max_over_time({foo="baz"} |= "error" | json | unwrap bytes[1d]) by (cluster,namespace), + shard=3_of_4 + > + ) + ) + ) + > + 10` + assert.Equal(t, expected, got) +} diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index 8d951ad64c94..a9f3dabe14ee 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -363,7 +363,7 @@ func (q *query) evalSample(ctx context.Context, expr syntax.SampleExpr) (promql_ maxSeries := validation.SmallestPositiveIntPerTenant(tenantIDs, maxSeriesCapture) return q.JoinSampleVector(next, ts, vec, stepEvaluator, maxSeries) case ProbabilisticQuantileVector: - return JoinQuantileSketchVector(next, vec, stepEvaluator, q.params) + return MergeQuantileSketchVector(next, vec, stepEvaluator, q.params) default: return nil, fmt.Errorf("unsupported result type: %T", r) } diff --git a/pkg/logql/log/parser.go b/pkg/logql/log/parser.go index c03e7c91cb96..90d4a4bebf8a 100644 --- a/pkg/logql/log/parser.go +++ b/pkg/logql/log/parser.go @@ -6,7 +6,7 @@ import ( "fmt" "unicode/utf8" - "github.com/buger/jsonparser" + "github.com/grafana/jsonparser" "github.com/grafana/loki/pkg/logql/log/jsonexpr" "github.com/grafana/loki/pkg/logql/log/logfmt" diff --git a/pkg/logql/log/parser_test.go b/pkg/logql/log/parser_test.go index bd57603ab808..f8cf6373a152 100644 --- a/pkg/logql/log/parser_test.go +++ b/pkg/logql/log/parser_test.go @@ -237,7 +237,7 @@ func (p *fakeParseHints) ShouldContinueParsingLine(_ string, _ *LabelsBuilder) b } func TestJSONExpressionParser(t *testing.T) { - testLine := []byte(`{"app":"foo","field with space":"value","field with ÜFT8👌":"value","null_field":null,"bool_field":false,"namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar", "params": [1,2,3]}}}`) + testLine := []byte(`{"app":"foo","field with space":"value","field with ÜFT8👌":"value","null_field":null,"bool_field":false,"namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar", "params": [1,2,3,"string_value"]}}}`) tests := []struct { name string @@ -340,6 +340,16 @@ func TestJSONExpressionParser(t *testing.T) { labels.FromStrings("param", "1"), NoParserHints(), }, + { + "array string element", + testLine, + []LabelExtractionExpr{ + NewLabelExtractionExpr("param", `pod.deployment.params[3]`), + }, + labels.EmptyLabels(), + labels.FromStrings("param", "string_value"), + NoParserHints(), + }, { "full array", testLine, @@ -347,7 +357,7 @@ func TestJSONExpressionParser(t *testing.T) { NewLabelExtractionExpr("params", `pod.deployment.params`), }, labels.EmptyLabels(), - labels.FromStrings("params", "[1,2,3]"), + labels.FromStrings("params", `[1,2,3,"string_value"]`), NoParserHints(), }, { @@ -357,7 +367,7 @@ func TestJSONExpressionParser(t *testing.T) { NewLabelExtractionExpr("deployment", `pod.deployment`), }, labels.EmptyLabels(), - labels.FromStrings("deployment", `{"ref":"foobar", "params": [1,2,3]}`), + labels.FromStrings("deployment", `{"ref":"foobar", "params": [1,2,3,"string_value"]}`), NoParserHints(), }, { diff --git a/pkg/logql/metrics.go b/pkg/logql/metrics.go index 67cfee24a056..d5a3f38a21f3 100644 --- a/pkg/logql/metrics.go +++ b/pkg/logql/metrics.go @@ -26,15 +26,13 @@ import ( ) const ( - QueryTypeMetric = "metric" - QueryTypeFilter = "filter" - QueryTypeLimited = "limited" - QueryTypeLabels = "labels" - QueryTypeSeries = "series" - QueryTypeIngesterStreams = "ingester_streams" - QueryTypeIngesterSeries = "ingester_series" - QueryTypeStats = "stats" - QueryTypeVolume = "volume" + QueryTypeMetric = "metric" + QueryTypeFilter = "filter" + QueryTypeLimited = "limited" + QueryTypeLabels = "labels" + QueryTypeSeries = "series" + QueryTypeStats = "stats" + QueryTypeVolume = "volume" latencyTypeSlow = "slow" latencyTypeFast = "fast" @@ -96,7 +94,8 @@ func RecordRangeAndInstantQueryMetrics( ) { var ( logger = fixLogger(ctx, log) - rt = string(GetRangeType(p)) + rangeType = GetRangeType(p) + rt = string(rangeType) latencyType = latencyTypeFast returnedLines = 0 ) @@ -105,6 +104,12 @@ func RecordRangeAndInstantQueryMetrics( level.Warn(logger).Log("msg", "error parsing query type", "err", err) } + resultCache := stats.Caches.Result + + if queryType == QueryTypeMetric && rangeType == InstantType { + resultCache = stats.Caches.InstantMetricResult + } + // Tag throughput metric by latency type based on a threshold. // Latency below the threshold is fast, above is slow. if stats.Summary.ExecTime > slowQueryThresholdSecond { @@ -116,13 +121,17 @@ func RecordRangeAndInstantQueryMetrics( } queryTags, _ := ctx.Value(httpreq.QueryTagsHTTPHeader).(string) // it's ok to be empty. + var ( + query = p.QueryString() + hashedQuery = util.HashedQuery(query) + ) logValues := make([]interface{}, 0, 50) logValues = append(logValues, []interface{}{ "latency", latencyType, // this can be used to filter log lines. - "query", p.QueryString(), - "query_hash", util.HashedQuery(p.QueryString()), + "query", query, + "query_hash", hashedQuery, "query_type", queryType, "range_type", rt, "length", p.End().Sub(p.Start()), @@ -133,9 +142,9 @@ func RecordRangeAndInstantQueryMetrics( "status", status, "limit", p.Limit(), "returned_lines", returnedLines, - "throughput", strings.Replace(humanize.Bytes(uint64(stats.Summary.BytesProcessedPerSecond)), " ", "", 1), - "total_bytes", strings.Replace(humanize.Bytes(uint64(stats.Summary.TotalBytesProcessed)), " ", "", 1), - "total_bytes_structured_metadata", strings.Replace(humanize.Bytes(uint64(stats.Summary.TotalStructuredMetadataBytesProcessed)), " ", "", 1), + "throughput", humanizeBytes(uint64(stats.Summary.BytesProcessedPerSecond)), + "total_bytes", humanizeBytes(uint64(stats.Summary.TotalBytesProcessed)), + "total_bytes_structured_metadata", humanizeBytes(uint64(stats.Summary.TotalStructuredMetadataBytesProcessed)), "lines_per_second", stats.Summary.LinesProcessedPerSecond, "total_lines", stats.Summary.TotalLinesProcessed, "post_filter_lines", stats.Summary.TotalPostFilterLines, @@ -160,10 +169,28 @@ func RecordRangeAndInstantQueryMetrics( "cache_volume_results_req", stats.Caches.VolumeResult.EntriesRequested, "cache_volume_results_hit", stats.Caches.VolumeResult.EntriesFound, "cache_volume_results_download_time", stats.Caches.VolumeResult.CacheDownloadTime(), - "cache_result_req", stats.Caches.Result.EntriesRequested, - "cache_result_hit", stats.Caches.Result.EntriesFound, - "cache_result_download_time", stats.Caches.Result.CacheDownloadTime(), - "cache_result_query_length_served", stats.Caches.Result.CacheQueryLengthServed(), + "cache_result_req", resultCache.EntriesRequested, + "cache_result_hit", resultCache.EntriesFound, + "cache_result_download_time", resultCache.CacheDownloadTime(), + "cache_result_query_length_served", resultCache.CacheQueryLengthServed(), + // The total of chunk reference fetched from index. + "ingester_chunk_refs", stats.Ingester.Store.GetTotalChunksRef(), + // Total number of chunks fetched. + "ingester_chunk_downloaded", stats.Ingester.Store.GetTotalChunksDownloaded(), + // Total of chunks matched by the query from ingesters. + "ingester_chunk_matches", stats.Ingester.GetTotalChunksMatched(), + // Total ingester reached for this query. + "ingester_requests", stats.Ingester.GetTotalReached(), + // Total bytes processed but was already in memory (found in the headchunk). Includes structured metadata bytes. + "ingester_chunk_head_bytes", humanizeBytes(uint64(stats.Ingester.Store.Chunk.GetHeadChunkBytes())), + // Total bytes of compressed chunks (blocks) processed. + "ingester_chunk_compressed_bytes", humanizeBytes(uint64(stats.Ingester.Store.Chunk.GetCompressedBytes())), + // Total bytes decompressed and processed from chunks. Includes structured metadata bytes. + "ingester_chunk_decompressed_bytes", humanizeBytes(uint64(stats.Ingester.Store.Chunk.GetDecompressedBytes())), + // Total lines post filtering. + "ingester_post_filter_lines", stats.Ingester.Store.Chunk.GetPostFilterLines(), + // Time spent being blocked on congestion control. + "congestion_control_latency", stats.CongestionControlLatency(), }...) logValues = append(logValues, tagsToKeyValues(queryTags)...) @@ -191,6 +218,10 @@ func RecordRangeAndInstantQueryMetrics( recordUsageStats(queryType, stats) } +func humanizeBytes(val uint64) string { + return strings.Replace(humanize.Bytes(val), " ", "", 1) +} + func RecordLabelQueryMetrics( ctx context.Context, log log.Logger, @@ -251,64 +282,6 @@ func PrintMatches(matches []string) string { return strings.Join(matches, ":") } -func RecordIngesterStreamsQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, status string, limit uint32, returnedLines int32, shards []string, stats logql_stats.Result) { - recordIngesterQueryMetrics(ctx, QueryTypeIngesterStreams, log, start, end, query, status, &limit, returnedLines, shards, stats) -} - -func RecordIngesterSeriesQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, status string, returnedLines int32, shards []string, stats logql_stats.Result) { - recordIngesterQueryMetrics(ctx, QueryTypeIngesterSeries, log, start, end, query, status, nil, returnedLines, shards, stats) -} - -func recordIngesterQueryMetrics(ctx context.Context, queryType string, log log.Logger, start, end time.Time, query string, status string, limit *uint32, returnedLines int32, shards []string, stats logql_stats.Result) { - var ( - logger = fixLogger(ctx, log) - latencyType = latencyTypeFast - ) - - // Tag throughput metric by latency type based on a threshold. - // Latency below the threshold is fast, above is slow. - if stats.Summary.ExecTime > slowQueryThresholdSecond { - latencyType = latencyTypeSlow - } - - logValues := make([]interface{}, 0, 23) - logValues = append(logValues, - "latency", latencyType, - "query_type", queryType, - "start", start.Format(time.RFC3339Nano), - "end", end.Format(time.RFC3339Nano), - "start_delta", time.Since(start), - "end_delta", time.Since(end), - "length", end.Sub(start), - "duration", time.Duration(int64(stats.Summary.ExecTime*float64(time.Second))), - "status", status, - "query", query, - "query_hash", util.HashedQuery(query), - "returned_lines", returnedLines, - "throughput", strings.Replace(humanize.Bytes(uint64(stats.Summary.BytesProcessedPerSecond)), " ", "", 1), - "total_bytes", strings.Replace(humanize.Bytes(uint64(stats.Summary.TotalBytesProcessed)), " ", "", 1), - "total_bytes_structured_metadata", strings.Replace(humanize.Bytes(uint64(stats.Summary.TotalStructuredMetadataBytesProcessed)), " ", "", 1), - "lines_per_second", stats.Summary.LinesProcessedPerSecond, - "total_lines", stats.Summary.TotalLinesProcessed, - "post_filter_lines", stats.Summary.TotalPostFilterLines, - "total_entries", stats.Summary.TotalEntriesReturned, - "chunk_refs_fetch_time", stats.ChunkRefsFetchTime()) - - if limit != nil { - logValues = append(logValues, - "limit", *limit) - } - shard := extractShard(shards) - if shard != nil { - logValues = append(logValues, - "shard_num", shard.Shard, - "shard_count", shard.Of, - ) - } - - level.Info(logger).Log(logValues...) -} - func RecordSeriesQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, match []string, status string, shards []string, stats logql_stats.Result) { var ( logger = fixLogger(ctx, log) diff --git a/pkg/logql/quantile_over_time_sketch.go b/pkg/logql/quantile_over_time_sketch.go index f9f05f99c997..24a8a05d89ed 100644 --- a/pkg/logql/quantile_over_time_sketch.go +++ b/pkg/logql/quantile_over_time_sketch.go @@ -14,7 +14,6 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/sketch" "github.com/grafana/loki/pkg/logqlmodel" - "github.com/grafana/loki/pkg/queue" ) const ( @@ -116,7 +115,6 @@ func (m ProbabilisticQuantileMatrix) Release() { for _, vec := range m { vec.Release() } - quantileVectorPool.Put(m) } func (m ProbabilisticQuantileMatrix) ToProto() *logproto.QuantileSketchMatrix { @@ -238,24 +236,20 @@ func probabilisticQuantileSampleFromProto(proto *logproto.QuantileSketchSample) type quantileSketchBatchRangeVectorIterator struct { *batchRangeVectorIterator - at []ProbabilisticQuantileSample } func (r *quantileSketchBatchRangeVectorIterator) At() (int64, StepResult) { - if r.at == nil { - r.at = make([]ProbabilisticQuantileSample, 0, len(r.window)) - } - r.at = r.at[:0] + at := make([]ProbabilisticQuantileSample, 0, len(r.window)) // convert ts from nano to milli seconds as the iterator work with nanoseconds ts := r.current/1e+6 + r.offset/1e+6 for _, series := range r.window { - r.at = append(r.at, ProbabilisticQuantileSample{ + at = append(at, ProbabilisticQuantileSample{ F: r.agg(series.Floats), T: ts, Metric: series.Metric, }) } - return ts, ProbabilisticQuantileVector(r.at) + return ts, ProbabilisticQuantileVector(at) } func (r *quantileSketchBatchRangeVectorIterator) agg(samples []promql.FPoint) sketch.QuantileSketch { @@ -268,23 +262,23 @@ func (r *quantileSketchBatchRangeVectorIterator) agg(samples []promql.FPoint) sk return s } -// quantileVectorPool slice of ProbabilisticQuantileVector [64, 128, 256, ..., 65536] -var quantileVectorPool = queue.NewSlicePool[ProbabilisticQuantileVector](1<<6, 1<<16, 2) - -// JoinQuantileSketchVector joins the results from stepEvaluator into a ProbabilisticQuantileMatrix. -func JoinQuantileSketchVector(next bool, r StepResult, stepEvaluator StepEvaluator, params Params) (promql_parser.Value, error) { +// MergeQuantileSketchVector joins the results from stepEvaluator into a ProbabilisticQuantileMatrix. +func MergeQuantileSketchVector(next bool, r StepResult, stepEvaluator StepEvaluator, params Params) (promql_parser.Value, error) { vec := r.QuantileSketchVec() if stepEvaluator.Error() != nil { return nil, stepEvaluator.Error() } + if GetRangeType(params) == InstantType { + return ProbabilisticQuantileMatrix{vec}, nil + } + stepCount := int(math.Ceil(float64(params.End().Sub(params.Start()).Nanoseconds()) / float64(params.Step().Nanoseconds()))) if stepCount <= 0 { stepCount = 1 } - // The result is released to the pool when the matrix is serialized. - result := quantileVectorPool.Get(stepCount) + result := make(ProbabilisticQuantileMatrix, 0, stepCount) for next { result = append(result, vec) @@ -295,7 +289,7 @@ func JoinQuantileSketchVector(next bool, r StepResult, stepEvaluator StepEvaluat } } - return ProbabilisticQuantileMatrix(result), stepEvaluator.Error() + return result, stepEvaluator.Error() } // QuantileSketchMatrixStepEvaluator steps through a matrix of quantile sketch diff --git a/pkg/logql/quantile_over_time_sketch_test.go b/pkg/logql/quantile_over_time_sketch_test.go index 4dcd079eeacc..488ebdec26f0 100644 --- a/pkg/logql/quantile_over_time_sketch_test.go +++ b/pkg/logql/quantile_over_time_sketch_test.go @@ -69,7 +69,7 @@ func TestJoinQuantileSketchVectorError(t *testing.T) { ev := errorStepEvaluator{ err: errors.New("could not evaluate"), } - _, err := JoinQuantileSketchVector(true, result, ev, LiteralParams{}) + _, err := MergeQuantileSketchVector(true, result, ev, LiteralParams{}) require.ErrorContains(t, err, "could not evaluate") } @@ -113,81 +113,33 @@ func (e errorStepEvaluator) Error() error { func (e errorStepEvaluator) Explain(Node) {} func BenchmarkJoinQuantileSketchVector(b *testing.B) { - results := make([]ProbabilisticQuantileVector, 100) - for i := range results { - results[i] = make(ProbabilisticQuantileVector, 10) - for j := range results[i] { - results[i][j] = ProbabilisticQuantileSample{ - T: int64(i), - F: newRandomSketch(), - Metric: []labels.Label{{Name: "foo", Value: fmt.Sprintf("bar-%d", j)}}, - } - } - } - ev := &sliceStepEvaluator{ - slice: results, - cur: 1, - } + selRange := (5 * time.Second).Nanoseconds() + step := (30 * time.Second) + offset := int64(0) + start := time.Unix(10, 0) + end := time.Unix(100, 0) // (end - start) / step == len(results) params := LiteralParams{ - start: time.Unix(0, 0), - end: time.Unix(int64(len(results)), 0), - step: time.Second, + start: start, + end: end, + step: step, } b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - // Reset step evaluator - ev.cur = 1 - r, err := JoinQuantileSketchVector(true, results[0], ev, params) + iter := newQuantileSketchIterator(newfakePeekingSampleIterator(samples), selRange, step.Nanoseconds(), start.UnixNano(), end.UnixNano(), offset) + ev := &QuantileSketchStepEvaluator{ + iter: iter, + } + _, _, r := ev.Next() + m, err := MergeQuantileSketchVector(true, r.QuantileSketchVec(), ev, params) require.NoError(b, err) - r.(ProbabilisticQuantileMatrix).Release() - } -} - -func newRandomSketch() sketch.QuantileSketch { - r := rand.New(rand.NewSource(42)) - s := sketch.NewDDSketch() - for i := 0; i < 1000; i++ { - _ = s.Add(r.Float64()) - } - return s -} - -type sliceStepEvaluator struct { - err error - slice []ProbabilisticQuantileVector - cur int -} - -// Close implements StepEvaluator. -func (*sliceStepEvaluator) Close() error { - return nil -} - -// Error implements StepEvaluator. -func (ev *sliceStepEvaluator) Error() error { - return ev.err -} - -// Explain implements StepEvaluator. -func (*sliceStepEvaluator) Explain(Node) {} - -// Next implements StepEvaluator. -func (ev *sliceStepEvaluator) Next() (ok bool, ts int64, r StepResult) { - if ev.cur >= len(ev.slice) { - return false, 0, nil + m.(ProbabilisticQuantileMatrix).Release() } - - r = ev.slice[ev.cur] - ts = ev.slice[ev.cur][0].T - ev.cur++ - ok = ev.cur < len(ev.slice) - return } func BenchmarkQuantileBatchRangeVectorIteratorAt(b *testing.B) { @@ -196,7 +148,9 @@ func BenchmarkQuantileBatchRangeVectorIteratorAt(b *testing.B) { }{ {numberSamples: 1}, {numberSamples: 1_000}, + {numberSamples: 10_000}, {numberSamples: 100_000}, + {numberSamples: 1_000_000}, } { b.Run(fmt.Sprintf("%d-samples", tc.numberSamples), func(b *testing.B) { r := rand.New(rand.NewSource(42)) diff --git a/pkg/logql/rangemapper.go b/pkg/logql/rangemapper.go index 975f63f4c952..f898e19d2ea1 100644 --- a/pkg/logql/rangemapper.go +++ b/pkg/logql/rangemapper.go @@ -57,6 +57,20 @@ type RangeMapper struct { splitByInterval time.Duration metrics *MapperMetrics stats *MapperStats + + splitAlignTs time.Time +} + +// NewRangeMapperWithSplitAlign is similar to `NewRangeMapper` except it accepts additional `splitAlign` argument and used to +// align the subqueries generated according to that. Look at `rangeSplitAlign` method for more information. +func NewRangeMapperWithSplitAlign(interval time.Duration, splitAlign time.Time, metrics *MapperMetrics, stats *MapperStats) (RangeMapper, error) { + rm, err := NewRangeMapper(interval, metrics, stats) + if err != nil { + return RangeMapper{}, err + } + rm.splitAlignTs = splitAlign + + return rm, nil } // NewRangeMapper creates a new RangeMapper instance with the given duration as @@ -327,6 +341,77 @@ func (m RangeMapper) getOriginalOffset(expr syntax.SampleExpr) (offset time.Dura // rangeInterval should be greater than m.splitByInterval, otherwise the resultant expression // will have an unnecessary aggregation operation func (m RangeMapper) mapConcatSampleExpr(expr syntax.SampleExpr, rangeInterval time.Duration, recorder *downstreamRecorder) syntax.SampleExpr { + if m.splitAlignTs.IsZero() { + return m.rangeSplit(expr, rangeInterval, recorder) + } + return m.rangeSplitAlign(expr, rangeInterval, recorder) +} + +// rangeSplitAlign try to split given `rangeInterval` into units of `m.splitByInterval` by making sure `rangeInterval` is aligned with `m.splitByInterval` for as much as the units as possible. +// Consider following example with real use case. +// Instant Query: `sum(rate({foo="bar"}[3h])` +// execTs: 12:34:00 +// splitBy: 1h +// Given above parameters, queries will be split into following +// 1. sum(rate({foo="bar"}[34m])) +// 2. sum(rate({foo="bar"}[1h] offset 34m)) +// 3. sum(rate({foo="bar"}[1h] offset 1h34m)) +// 4. sum(rate({foo="bar"}[26m] offset 2h34m)) +func (m RangeMapper) rangeSplitAlign( + expr syntax.SampleExpr, rangeInterval time.Duration, recorder *downstreamRecorder, +) syntax.SampleExpr { + if rangeInterval <= m.splitByInterval { + return expr + } + + originalOffset, err := m.getOriginalOffset(expr) + if err != nil { + return expr + } + + align := m.splitAlignTs.Sub(m.splitAlignTs.Truncate(m.splitByInterval)) // say, 12:34:00 - 12:00:00(truncated) = 34m + + if align == 0 { + return m.rangeSplit(expr, rangeInterval, recorder) // Don't have to align + } + + var ( + newRng = align + + // TODO(kavi): If the originalOffset is non-zero, there may be a edge case, where subqueries generated won't be aligned correctly. Handle this edge case in separate PR. + newOffset = originalOffset + downstreams *ConcatSampleExpr + pendingRangeInterval = rangeInterval + splits = 0 + ) + + // first subquery + downstreams = appendDownstream(downstreams, expr, newRng, newOffset) + splits++ + + newOffset += align // e.g: offset 34m + pendingRangeInterval -= newRng + newRng = m.splitByInterval // [1h] + + // Rest of the subqueries. + for pendingRangeInterval > 0 { + if pendingRangeInterval < m.splitByInterval { + newRng = pendingRangeInterval // last subquery + } + downstreams = appendDownstream(downstreams, expr, newRng, newOffset) + newOffset += m.splitByInterval + pendingRangeInterval -= newRng + splits++ + } + + // update stats and metrics + m.stats.AddSplitQueries(splits) + recorder.Add(splits, MetricsKey) + + return downstreams +} + +func (m RangeMapper) rangeSplit(expr syntax.SampleExpr, rangeInterval time.Duration, recorder *downstreamRecorder) syntax.SampleExpr { splitCount := int(math.Ceil(float64(rangeInterval) / float64(m.splitByInterval))) if splitCount <= 1 { return expr diff --git a/pkg/logql/rangemapper_test.go b/pkg/logql/rangemapper_test.go index 562ac0cd168e..5e95486a8c8e 100644 --- a/pkg/logql/rangemapper_test.go +++ b/pkg/logql/rangemapper_test.go @@ -93,6 +93,84 @@ func Test_SplitRangeInterval(t *testing.T) { } } +func Test_RangeMapperSplitAlign(t *testing.T) { + cases := []struct { + name string + expr string + queryTime time.Time + splityByInterval time.Duration + expected string + expectedSplits int + }{ + { + name: "query_time_aligned_with_split_by", + expr: `bytes_over_time({app="foo"}[3m])`, + expected: `sum without() ( + downstream> + ++ downstream> + ++ downstream> + )`, + queryTime: time.Unix(60, 0), // 1970 00:01:00 + splityByInterval: 1 * time.Minute, + expectedSplits: 3, + }, + { + name: "query_time_aligned_with_split_by_with_original_offset", + expr: `bytes_over_time({app="foo"}[3m] offset 20m10s)`, // NOTE: original query has offset, which should be considered in all the splits subquery + expected: `sum without() ( + downstream> + ++ downstream> + ++ downstream> + )`, + queryTime: time.Unix(60, 0), // 1970 00:01:00 + splityByInterval: 1 * time.Minute, + expectedSplits: 3, + }, + { + name: "query_time_not_aligned_with_split_by", + expr: `bytes_over_time({app="foo"}[3h])`, + expected: `sum without() ( + downstream> + ++ downstream> + ++ downstream> + ++ downstream> + )`, + queryTime: time.Date(0, 0, 0, 12, 54, 0, 0, time.UTC), // 1970 12:54:00 + splityByInterval: 1 * time.Hour, + expectedSplits: 4, + }, + { + name: "query_time_not_aligned_with_split_by_with_original_offset", + expr: `bytes_over_time({app="foo"}[3h] offset 1h2m20s)`, // NOTE: original query has offset, which should be considered in all the splits subquery + expected: `sum without() ( + downstream> + ++ downstream> + ++ downstream> + ++ downstream> + )`, + queryTime: time.Date(0, 0, 0, 12, 54, 0, 0, time.UTC), // 1970 12:54:00 + splityByInterval: 1 * time.Hour, + expectedSplits: 4, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mapperStats := NewMapperStats() + rvm, err := NewRangeMapperWithSplitAlign(tc.splityByInterval, tc.queryTime, nilShardMetrics, mapperStats) + require.NoError(t, err) + + noop, mappedExpr, err := rvm.Parse(syntax.MustParseExpr(tc.expr)) + require.NoError(t, err) + + require.Equal(t, removeWhiteSpace(tc.expected), removeWhiteSpace(mappedExpr.String())) + require.Equal(t, tc.expectedSplits, mapperStats.GetSplitQueries()) + require.False(t, noop) + + }) + } +} + func Test_SplitRangeVectorMapping(t *testing.T) { for _, tc := range []struct { expr string @@ -1675,7 +1753,7 @@ func Test_SplitRangeVectorMapping(t *testing.T) { // Non-splittable vector aggregators - should go deeper in the AST { `topk(2, count_over_time({app="foo"}[3m]))`, - `topk(2, + `topk(2, sum without () ( downstream> ++ downstream> @@ -1713,7 +1791,7 @@ func Test_SplitRangeVectorMapping(t *testing.T) { ++ downstream> ++ downstream> ) - ), + ), "x", "$1", "a", "(.*)" )`, 3, @@ -1727,7 +1805,7 @@ func Test_SplitRangeVectorMapping(t *testing.T) { ++ downstream> ++ downstream> ) - / 180), + / 180), "foo", "$1", "service", "(.*):.*" )`, 3, diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index 96955109a941..0e345291eed3 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -1598,3 +1598,32 @@ func TestStringTrimming(t *testing.T) { func float64p(v float64) *float64 { return &v } + +func TestShardTopk(t *testing.T) { + expr := `topk( + 10, + sum by (ip) ( + sum_over_time({job="foo"} | json | unwrap bytes(bytes)[1m]) + ) + )` + m := NewShardMapper(ConstantShards(5), nilShardMetrics, []string{ShardQuantileOverTime}) + _, _, mappedExpr, err := m.Parse(syntax.MustParseExpr(expr)) + require.NoError(t, err) + + expected := `topk( + 10, + sum by (ip)( + concat( + downstream + ++ + downstream + ++ + downstream + ++ + downstream + ++ ... + ) + ) +)` + require.Equal(t, expected, mappedExpr.Pretty(0)) +} diff --git a/pkg/logql/sketch/quantile.go b/pkg/logql/sketch/quantile.go index 8042ea53741e..3b8b0f22fc8e 100644 --- a/pkg/logql/sketch/quantile.go +++ b/pkg/logql/sketch/quantile.go @@ -47,7 +47,7 @@ const relativeAccuracy = 0.01 var ddsketchPool = sync.Pool{ New: func() any { m, _ := mapping.NewCubicallyInterpolatedMapping(relativeAccuracy) - return ddsketch.NewDDSketchFromStoreProvider(m, store.DefaultProvider) + return ddsketch.NewDDSketch(m, store.NewCollapsingLowestDenseStore(2048), store.NewCollapsingLowestDenseStore(2048)) }, } diff --git a/pkg/logql/syntax/ast.go b/pkg/logql/syntax/ast.go index cea41f4d95c5..26e77779c4b3 100644 --- a/pkg/logql/syntax/ast.go +++ b/pkg/logql/syntax/ast.go @@ -54,6 +54,22 @@ func MustClone[T Expr](e T) T { return copied } +func ExtractLineFilters(e Expr) []LineFilterExpr { + if e == nil { + return nil + } + var filters []LineFilterExpr + visitor := &DepthFirstTraversal{ + VisitLineFilterFn: func(v RootVisitor, e *LineFilterExpr) { + if e != nil { + filters = append(filters, *e) + } + }, + } + e.Accept(visitor) + return filters +} + // implicit holds default implementations type implicit struct{} diff --git a/pkg/logql/syntax/prettier.go b/pkg/logql/syntax/prettier.go index cf346e26c562..1b407453858f 100644 --- a/pkg/logql/syntax/prettier.go +++ b/pkg/logql/syntax/prettier.go @@ -35,8 +35,8 @@ import ( // var ( - // maxCharsPerLine is used to qualify whether some LogQL expressions are worth `splitting` into new lines. - maxCharsPerLine = 100 + // MaxCharsPerLine is used to qualify whether some LogQL expressions are worth `splitting` into new lines. + MaxCharsPerLine = 100 ) func Prettify(e Expr) string { @@ -51,8 +51,8 @@ func (e *MatchersExpr) Pretty(level int) string { // e.g: `{foo="bar"} | logfmt | level="error"` // Here, left = `{foo="bar"}` and multistages would collection of each stage in pipeline, here `logfmt` and `level="error"` func (e *PipelineExpr) Pretty(level int) string { - if !needSplit(e) { - return indent(level) + e.String() + if !NeedSplit(e) { + return Indent(level) + e.String() } s := fmt.Sprintf("%s\n", e.Left.Pretty(level)) @@ -73,8 +73,8 @@ func (e *PipelineExpr) Pretty(level int) string { // e.g: `|= "error" != "memcache" |= ip("192.168.0.1")` // NOTE: here `ip` is Op in this expression. func (e *LineFilterExpr) Pretty(level int) string { - if !needSplit(e) { - return indent(level) + e.String() + if !NeedSplit(e) { + return Indent(level) + e.String() } var s string @@ -90,7 +90,7 @@ func (e *LineFilterExpr) Pretty(level int) string { s += "\n" } - s += indent(level) + s += Indent(level) // We re-use LineFilterExpr's String() implementation to avoid duplication. // We create new LineFilterExpr without `Left`. @@ -153,7 +153,7 @@ func (e *LogfmtExpressionParser) Pretty(level int) string { // e.g: sum_over_time({foo="bar"} | logfmt | unwrap bytes_processed [5m]) func (e *UnwrapExpr) Pretty(level int) string { - s := indent(level) + s := Indent(level) if e.Operation != "" { s += fmt.Sprintf("%s %s %s(%s)", OpPipe, OpUnwrap, e.Operation, e.Identifier) @@ -161,7 +161,7 @@ func (e *UnwrapExpr) Pretty(level int) string { s += fmt.Sprintf("%s %s %s", OpPipe, OpUnwrap, e.Identifier) } for _, f := range e.PostFilters { - s += fmt.Sprintf("\n%s%s %s", indent(level), OpPipe, f) + s += fmt.Sprintf("\n%s%s %s", Indent(level), OpPipe, f) } return s } @@ -200,8 +200,8 @@ func (e *OffsetExpr) Pretty(_ int) string { // e.g: count_over_time({foo="bar"}[5m]) func (e *RangeAggregationExpr) Pretty(level int) string { - s := indent(level) - if !needSplit(e) { + s := Indent(level) + if !NeedSplit(e) { return s + e.String() } @@ -211,13 +211,13 @@ func (e *RangeAggregationExpr) Pretty(level int) string { // print args to the function. if e.Params != nil { - s = fmt.Sprintf("%s%s%s,", s, indent(level+1), fmt.Sprint(*e.Params)) + s = fmt.Sprintf("%s%s%s,", s, Indent(level+1), fmt.Sprint(*e.Params)) s += "\n" } s += e.Left.Pretty(level + 1) - s += "\n" + indent(level) + ")" + s += "\n" + Indent(level) + ")" if e.Grouping != nil { s += e.Grouping.Pretty(level) @@ -236,9 +236,9 @@ func (e *RangeAggregationExpr) Pretty(level int) string { // - vector on which aggregation is done. // [without|by (