From 43a058984fa15acd119131eda8bf357c443ac86e Mon Sep 17 00:00:00 2001 From: Piotr <17101802+thampiotr@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:30:48 +0000 Subject: [PATCH] Cherry picks for v0.40.3 (#6679) * loki.source.api: fix a bug where structured metadata is not passed downstream (#6508) * Update build image to Go 1.22.1 (#6650) * Upgrade Go to 1.22.1 (#6646) * Upgrade Go to 1.22.1 * update build images * Attempt to fix main: separate clean in windows docker images (#6671) * Change git operations from fetch to pull (#6668) * Update to using FETCH_HEAD since we are using FETCH which doesnt update HEAD. * Update to using FETCH_HEAD since we are using FETCH which doesnt update HEAD. * Use pull and not fetch * add comments * fix linting * Update CHANGELOG.md Co-authored-by: Robert Fratto --------- Co-authored-by: Robert Fratto --------- Co-authored-by: Mark Motliuk Co-authored-by: mattdurham Co-authored-by: Robert Fratto --- .drone/drone.yml | 120 +++++++++--------- .drone/pipelines/build_images.jsonnet | 4 +- .github/workflows/check-linux-build-image.yml | 4 +- CHANGELOG.md | 13 ++ build-image/Dockerfile | 2 +- build-image/windows/Dockerfile | 2 +- cmd/grafana-agent-operator/Dockerfile | 2 +- cmd/grafana-agent/Dockerfile | 2 +- cmd/grafana-agent/Dockerfile.windows | 9 +- cmd/grafana-agentctl/Dockerfile | 2 +- cmd/grafana-agentctl/Dockerfile.windows | 10 +- .../api/internal/lokipush/push_api_server.go | 4 +- .../internal/lokipush/push_api_server_test.go | 14 ++ docs/Makefile | 6 +- .../flow/reference/components/module.git.md | 12 +- go.mod | 2 +- .../configs/otel-metrics-gen/Dockerfile | 2 +- integration-tests/configs/prom-gen/Dockerfile | 2 +- internal/vcs/git.go | 25 ++-- pkg/flow/import_test.go | 96 ++++++++++++++ tools/make/build-container.mk | 2 +- 21 files changed, 237 insertions(+), 98 deletions(-) diff --git a/.drone/drone.yml b/.drone/drone.yml index 204767a96375..ce6249bf5fea 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -10,7 +10,7 @@ steps: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - - docker buildx build --build-arg="GO_RUNTIME=golang:1.22.0-bullseye" --push --platform + - docker buildx build --build-arg="GO_RUNTIME=golang:1.22.1-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image environment: DOCKER_LOGIN: @@ -44,7 +44,7 @@ steps: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - - docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye" + - docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22.1-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image environment: @@ -110,7 +110,7 @@ steps: - commands: - apt-get update -y && apt-get install -y libsystemd-dev - make lint - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Lint trigger: event: @@ -128,7 +128,7 @@ steps: - ERR_MSG="Dashboard definitions are out of date. Please run 'make generate-dashboards' and commit changes!" - if [ ! -z "$(git status --porcelain)" ]; then echo $ERR_MSG >&2; exit 1; fi - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Regenerate dashboards trigger: event: @@ -146,7 +146,7 @@ steps: - ERR_MSG="Custom Resource Definitions are out of date. Please run 'make generate-crds' and commit changes!" - if [ ! -z "$(git status --porcelain)" ]; then echo $ERR_MSG >&2; exit 1; fi - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Regenerate crds trigger: event: @@ -161,7 +161,7 @@ platform: steps: - commands: - make GO_TAGS="nodocker" test - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Run Go tests trigger: event: @@ -176,7 +176,7 @@ platform: steps: - commands: - K8S_USE_DOCKER_NETWORK=1 make test - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Run Go tests volumes: - name: docker @@ -199,7 +199,7 @@ platform: steps: - commands: - go test -tags="nodocker,nonetwork" ./... - image: grafana/agent-build-image:0.33.0-windows + image: grafana/agent-build-image:0.40.2-windows name: Run Go tests trigger: ref: @@ -214,7 +214,7 @@ platform: steps: - commands: - make agent-image - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build container volumes: - name: docker @@ -239,7 +239,7 @@ platform: steps: - commands: - make agentctl-image - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build container volumes: - name: docker @@ -264,7 +264,7 @@ platform: steps: - commands: - make operator-image - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build container volumes: - name: docker @@ -290,7 +290,7 @@ platform: steps: - commands: - '& "C:/Program Files/git/bin/bash.exe" ./tools/ci/docker-containers-windows agent' - image: grafana/agent-build-image:0.33.0-windows + image: grafana/agent-build-image:0.40.2-windows name: Build container volumes: - name: docker @@ -316,7 +316,7 @@ platform: steps: - commands: - '& "C:/Program Files/git/bin/bash.exe" ./tools/ci/docker-containers-windows agentctl' - image: grafana/agent-build-image:0.33.0-windows + image: grafana/agent-build-image:0.40.2-windows name: Build container volumes: - name: docker @@ -343,7 +343,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=amd64 GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -360,7 +360,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=arm64 GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -377,7 +377,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=ppc64le GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -394,7 +394,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=s390x GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -410,7 +410,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=amd64 GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -426,7 +426,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=arm64 GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -442,7 +442,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=windows GOARCH=amd64 GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -458,7 +458,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=freebsd GOARCH=amd64 GOARM= make agent - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -475,7 +475,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=amd64 GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -492,7 +492,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=arm64 GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -509,7 +509,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=ppc64le GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -526,7 +526,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=s390x GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -542,7 +542,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=amd64 GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -558,7 +558,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=arm64 GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -574,7 +574,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=windows GOARCH=amd64 GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -590,7 +590,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=freebsd GOARCH=amd64 GOARM= make agent-flow - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -607,7 +607,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=amd64 GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -624,7 +624,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=arm64 GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -641,7 +641,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=ppc64le GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -658,7 +658,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=s390x GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -674,7 +674,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=amd64 GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -690,7 +690,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=arm64 GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -706,7 +706,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=windows GOARCH=amd64 GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -722,7 +722,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=freebsd GOARCH=amd64 GOARM= make agentctl - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -739,7 +739,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=amd64 GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -756,7 +756,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=arm64 GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -773,7 +773,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=ppc64le GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -790,7 +790,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=s390x GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -806,7 +806,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=amd64 GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -822,7 +822,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=darwin GOARCH=arm64 GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -838,7 +838,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=windows GOARCH=amd64 GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -854,7 +854,7 @@ steps: - commands: - make generate-ui - GO_TAGS="builtinassets" GOOS=freebsd GOARCH=amd64 GOARM= make operator - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -871,7 +871,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=amd64 GOARM= GOEXPERIMENT=boringcrypto make agent-boringcrypto - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -888,7 +888,7 @@ steps: - make generate-ui - GO_TAGS="builtinassets promtail_journal_enabled" GOOS=linux GOARCH=arm64 GOARM= GOEXPERIMENT=boringcrypto make agent-boringcrypto - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Build trigger: event: @@ -904,7 +904,7 @@ steps: - commands: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes failure: ignore - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Configure QEMU volumes: - name: docker @@ -924,7 +924,7 @@ steps: from_secret: docker_password GCR_CREDS: from_secret: gcr_admin - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Publish container volumes: - name: docker @@ -948,7 +948,7 @@ steps: - commands: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes failure: ignore - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Configure QEMU volumes: - name: docker @@ -968,7 +968,7 @@ steps: from_secret: docker_password GCR_CREDS: from_secret: gcr_admin - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Publish container volumes: - name: docker @@ -992,7 +992,7 @@ steps: - commands: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes failure: ignore - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Configure QEMU volumes: - name: docker @@ -1012,7 +1012,7 @@ steps: from_secret: docker_password GCR_CREDS: from_secret: gcr_admin - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Publish container volumes: - name: docker @@ -1036,7 +1036,7 @@ steps: - commands: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes failure: ignore - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Configure QEMU volumes: - name: docker @@ -1056,7 +1056,7 @@ steps: from_secret: docker_password GCR_CREDS: from_secret: gcr_admin - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Publish container volumes: - name: docker @@ -1085,7 +1085,7 @@ steps: from_secret: docker_login DOCKER_PASSWORD: from_secret: docker_password - image: grafana/agent-build-image:0.33.0-windows + image: grafana/agent-build-image:0.40.2-windows name: Build containers volumes: - name: docker @@ -1114,7 +1114,7 @@ steps: from_secret: docker_login DOCKER_PASSWORD: from_secret: docker_password - image: grafana/agent-build-image:0.33.0-windows + image: grafana/agent-build-image:0.40.2-windows name: Build containers volumes: - name: docker @@ -1231,7 +1231,7 @@ steps: from_secret: gpg_private_key GPG_PUBLIC_KEY: from_secret: gpg_public_key - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Publish release volumes: - name: docker @@ -1256,7 +1256,7 @@ steps: - DOCKER_OPTS="" make dist/grafana-agentctl-linux-amd64 - DOCKER_OPTS="" make dist.temp/grafana-agent-flow-linux-amd64 - DOCKER_OPTS="" make test-packages - image: grafana/agent-build-image:0.33.0 + image: grafana/agent-build-image:0.40.2 name: Test Linux system packages volumes: - name: docker @@ -1352,6 +1352,6 @@ kind: secret name: updater_private_key --- kind: signature -hmac: 251627e59296f743a58112edccf6a03ba1e226a57d59720e3894d2ff9c7e520d +hmac: b2dba2c3561451244eb5145b946c01d3c30336828cc0a60aee247946608ceb18 ... diff --git a/.drone/pipelines/build_images.jsonnet b/.drone/pipelines/build_images.jsonnet index 328c7fc34443..3d19ebc1b470 100644 --- a/.drone/pipelines/build_images.jsonnet +++ b/.drone/pipelines/build_images.jsonnet @@ -32,7 +32,7 @@ local locals = { 'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD', 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes', 'docker buildx create --name multiarch --driver docker-container --use', - 'docker buildx build --build-arg="GO_RUNTIME=golang:1.22.0-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image', + 'docker buildx build --build-arg="GO_RUNTIME=golang:1.22.1-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image', ], }], volumes: [{ @@ -55,7 +55,7 @@ local locals = { 'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD', 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes', 'docker buildx create --name multiarch --driver docker-container --use', - 'docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image', + 'docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22.1-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image', ], }], volumes: [{ diff --git a/.github/workflows/check-linux-build-image.yml b/.github/workflows/check-linux-build-image.yml index 32737b0b6afd..300d203d3678 100644 --- a/.github/workflows/check-linux-build-image.yml +++ b/.github/workflows/check-linux-build-image.yml @@ -25,7 +25,7 @@ jobs: push: false tags: grafana/agent-build-image:latest build-args: | - GO_RUNTIME=golang:1.22.0-bullseye + GO_RUNTIME=golang:1.22.1-bullseye - name: Create test Linux build image for boring crypto uses: docker/build-push-action@v5 @@ -34,4 +34,4 @@ jobs: push: false tags: grafana/agent-build-image:latest build-args: | - GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye \ No newline at end of file + GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22.1-bullseye \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 888c219e08dc..132cab106c04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ This document contains a historical list of changes between releases. Only changes that impact end-user behavior are listed; changes to documentation or internal API changes are not present. +v0.40.3 (2024-03-14) +-------------------- + +### Bugfixes + +- Fix a bug where structured metadata and parsed field are not passed further in `loki.source.api` (@marchellodev) + +- Change `import.git` to use Git pulls rather than fetches to fix scenarios where the local code did not get updated. (@mattdurham) + +### Other changes + +- Upgrade to Go 1.22.1 (@thampiotr) + v0.40.2 (2024-03-05) -------------------- diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 602e7cbd1aeb..9691fd02b2fb 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -4,7 +4,7 @@ # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set # in environment variables. -# NOTE: The GO_RUNTIME is used to switch between the default google go runtime and mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye which is a microsoft +# NOTE: The GO_RUNTIME is used to switch between the default Google go runtime and mcr.microsoft.com/oss/go/microsoft/golang:1.22.1-bullseye which is a Microsoft # fork of go that allows using windows crypto instead of boring crypto. Details at https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips ARG GO_RUNTIME=mustoverride diff --git a/build-image/windows/Dockerfile b/build-image/windows/Dockerfile index ddd3448e2c09..3827b073f931 100644 --- a/build-image/windows/Dockerfile +++ b/build-image/windows/Dockerfile @@ -1,4 +1,4 @@ -FROM library/golang:1.22.0-windowsservercore-1809 +FROM library/golang:1.22.1-windowsservercore-1809 SHELL ["powershell", "-command"] diff --git a/cmd/grafana-agent-operator/Dockerfile b/cmd/grafana-agent-operator/Dockerfile index 6ba2a6218cd7..7e4f98f09270 100644 --- a/cmd/grafana-agent-operator/Dockerfile +++ b/cmd/grafana-agent-operator/Dockerfile @@ -4,7 +4,7 @@ # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set # in environment variables. -FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.33.0 as build +FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.40.2 as build ARG BUILDPLATFORM ARG TARGETPLATFORM ARG TARGETOS diff --git a/cmd/grafana-agent/Dockerfile b/cmd/grafana-agent/Dockerfile index 132d1317495c..36ff4e98f799 100644 --- a/cmd/grafana-agent/Dockerfile +++ b/cmd/grafana-agent/Dockerfile @@ -4,7 +4,7 @@ # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set # in environment variables. -FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.33.0 as build +FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.40.2 as build ARG BUILDPLATFORM ARG TARGETPLATFORM ARG TARGETOS diff --git a/cmd/grafana-agent/Dockerfile.windows b/cmd/grafana-agent/Dockerfile.windows index 6f99a6138fdb..1ebc3346f174 100644 --- a/cmd/grafana-agent/Dockerfile.windows +++ b/cmd/grafana-agent/Dockerfile.windows @@ -1,4 +1,4 @@ -FROM grafana/agent-build-image:0.33.0-windows as builder +FROM grafana/agent-build-image:0.40.2-windows as builder ARG VERSION ARG RELEASE_BUILD=1 @@ -10,7 +10,12 @@ SHELL ["cmd", "/S", "/C"] # Creating new layers can be really slow on Windows so we clean up any caches # we can before moving on to the next step. RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} make generate-ui && rm -rf web/ui/node_modules && yarn cache clean --all"" -RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} GO_TAGS='builtinassets' make agent && go clean -cache -modcache"" + +RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} GO_TAGS='builtinassets' make agent"" +# In this case, we're separating the clean command from make agent to avoid an issue where access to some mod cache +# files is denied immediately after make agent, for example: +# "go: remove C:\go\pkg\mod\golang.org\toolchain@v0.0.1-go1.22.1.windows-amd64\bin\go.exe: Access is denied." +RUN ""C:\Program Files\git\bin\bash.exe" -c "go clean -cache -modcache"" # Use the smallest container possible for the final image FROM mcr.microsoft.com/windows/nanoserver:1809 diff --git a/cmd/grafana-agentctl/Dockerfile b/cmd/grafana-agentctl/Dockerfile index 67fe64c6f3bf..f43179d418cd 100644 --- a/cmd/grafana-agentctl/Dockerfile +++ b/cmd/grafana-agentctl/Dockerfile @@ -4,7 +4,7 @@ # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set # in environment variables. -FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.33.0 as build +FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.40.2 as build ARG BUILDPLATFORM ARG TARGETPLATFORM ARG TARGETOS diff --git a/cmd/grafana-agentctl/Dockerfile.windows b/cmd/grafana-agentctl/Dockerfile.windows index 8cf3c34a0ace..77df750b9d7e 100644 --- a/cmd/grafana-agentctl/Dockerfile.windows +++ b/cmd/grafana-agentctl/Dockerfile.windows @@ -1,4 +1,4 @@ -FROM grafana/agent-build-image:0.33.0-windows as builder +FROM grafana/agent-build-image:0.40.2-windows as builder ARG VERSION ARG RELEASE_BUILD=1 @@ -7,9 +7,11 @@ WORKDIR /src/agent SHELL ["cmd", "/S", "/C"] -# Creating new layers can be really slow on Windows so we clean up any caches -# we can before moving on to the next step. -RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} make agentctl && go clean -cache -modcache"" +RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} make agentctl"" +# We're separating the clean command from make agent to avoid an issue where access to some mod cache +# files is denied immediately after make agentctl, for example: +# "go: remove C:\go\pkg\mod\golang.org\toolchain@v0.0.1-go1.22.1.windows-amd64\bin\go.exe: Access is denied." +RUN ""C:\Program Files\git\bin\bash.exe" -c "go clean -cache -modcache"" # Use the smallest container possible for the final image FROM mcr.microsoft.com/windows/nanoserver:1809 diff --git a/component/loki/source/api/internal/lokipush/push_api_server.go b/component/loki/source/api/internal/lokipush/push_api_server.go index 9a9a20f75920..5998ca958f35 100644 --- a/component/loki/source/api/internal/lokipush/push_api_server.go +++ b/component/loki/source/api/internal/lokipush/push_api_server.go @@ -185,7 +185,9 @@ func (s *PushAPIServer) handleLoki(w http.ResponseWriter, r *http.Request) { e := loki.Entry{ Labels: filtered.Clone(), Entry: logproto.Entry{ - Line: entry.Line, + Line: entry.Line, + StructuredMetadata: entry.StructuredMetadata, + Parsed: entry.Parsed, }, } if keepTimestamp { diff --git a/component/loki/source/api/internal/lokipush/push_api_server_test.go b/component/loki/source/api/internal/lokipush/push_api_server_test.go index 8cc9e827ade6..01be759c9ea7 100644 --- a/component/loki/source/api/internal/lokipush/push_api_server_test.go +++ b/component/loki/source/api/internal/lokipush/push_api_server_test.go @@ -23,6 +23,7 @@ import ( frelabel "github.com/grafana/agent/component/common/relabel" "github.com/grafana/dskit/flagext" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/push" "github.com/grafana/river" "github.com/phayes/freeport" "github.com/prometheus/client_golang/prometheus" @@ -79,6 +80,10 @@ regex = "dropme" Entry: logproto.Entry{ Timestamp: time.Unix(int64(i), 0), Line: "line" + strconv.Itoa(i), + StructuredMetadata: push.LabelsAdapter{ + {Name: "i", Value: strconv.Itoa(i)}, + {Name: "anotherMetaData", Value: "val"}, + }, }, } } @@ -98,9 +103,18 @@ regex = "dropme" "pushserver": "pushserver1", "stream": "stream1", } + + expectedStructuredMetadata := push.LabelsAdapter{ + {Name: "i", Value: strconv.Itoa(0)}, + {Name: "anotherMetaData", Value: "val"}, + } + // Spot check the first value in the result to make sure relabel rules were applied properly require.Equal(t, expectedLabels, eh.Received()[0].Labels) + // Spot check the first value in the result to make sure structured metadata was received properly + require.Equal(t, expectedStructuredMetadata, eh.Received()[0].StructuredMetadata) + // With keep timestamp enabled, verify timestamp require.Equal(t, time.Unix(99, 0).Unix(), eh.Received()[99].Timestamp.Unix()) diff --git a/docs/Makefile b/docs/Makefile index 473f6bab03d7..9c60214b2768 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -11,11 +11,11 @@ include docs.mk docs: check-cloudwatch-integration check-cloudwatch-integration: - $(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md - $(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md + $(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md + $(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md generate-cloudwatch-integration: - $(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go generate + $(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go generate sources/assets/hierarchy.svg: sources/operator/hierarchy.dot cat $< | $(PODMAN) run --rm -i nshine/dot dot -Tsvg > $@ diff --git a/docs/sources/flow/reference/components/module.git.md b/docs/sources/flow/reference/components/module.git.md index 44bdee36a034..90b8dae130c1 100644 --- a/docs/sources/flow/reference/components/module.git.md +++ b/docs/sources/flow/reference/components/module.git.md @@ -41,12 +41,12 @@ module.git "LABEL" { The following arguments are supported: -Name | Type | Description | Default | Required ----- | ---- | ----------- | ------- | -------- -`repository` | `string` | The Git repository address to retrieve the module from. | | yes -`revision` | `string` | The Git revision to retrieve the module from. | `"HEAD"` | no -`path` | `string` | The path in the repository where the module is stored. | | yes -`pull_frequency` | `duration` | The frequency to pull the repository for updates. | `"60s"` | no +Name | Type | Description | Default | Required +-----------------|------------|---------------------------------------------------------|----------|--------- +`repository` | `string` | The Git repository address to retrieve the module from. | | yes +`revision` | `string` | The Git revision to retrieve the module from. | `"HEAD"` | no +`path` | `string` | The path in the repository where the module is stored. | | yes +`pull_frequency` | `duration` | The frequency to pull the repository for updates. | `"60s"` | no The `repository` attribute must be set to a repository address that would be recognized by Git with a `git clone REPOSITORY_ADDRESS` command, such as diff --git a/go.mod b/go.mod index bc0d6ade408f..152232302a98 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/grafana/agent -go 1.21.0 +go 1.22.1 require ( cloud.google.com/go/pubsub v1.33.0 diff --git a/integration-tests/configs/otel-metrics-gen/Dockerfile b/integration-tests/configs/otel-metrics-gen/Dockerfile index 8bf00387660e..6424b617725b 100644 --- a/integration-tests/configs/otel-metrics-gen/Dockerfile +++ b/integration-tests/configs/otel-metrics-gen/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21 as build +FROM golang:1.22.1 as build WORKDIR /app/ COPY go.mod go.sum ./ RUN go mod download diff --git a/integration-tests/configs/prom-gen/Dockerfile b/integration-tests/configs/prom-gen/Dockerfile index 7c395748689a..5e273dccb571 100644 --- a/integration-tests/configs/prom-gen/Dockerfile +++ b/integration-tests/configs/prom-gen/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21 as build +FROM golang:1.22.1 as build WORKDIR /app/ COPY go.mod go.sum ./ RUN go mod download diff --git a/internal/vcs/git.go b/internal/vcs/git.go index dece43c10b2f..78857e56d87c 100644 --- a/internal/vcs/git.go +++ b/internal/vcs/git.go @@ -29,7 +29,7 @@ type GitRepo struct { // managed at storagePath. // // If storagePath is empty on disk, NewGitRepo initializes GitRepo by cloning -// the repository. Otherwise, NewGitRepo will do a fetch. +// the repository. Otherwise, NewGitRepo will do a pull. // // After GitRepo is initialized, it checks out to the Revision specified in // GitRepoOptions. @@ -57,13 +57,20 @@ func NewGitRepo(ctx context.Context, storagePath string, opts GitRepoOptions) (* } } - // Fetch the latest contents. This may be a no-op if we just did a clone. - fetchRepoErr := repo.FetchContext(ctx, &git.FetchOptions{ + // Pulls the latest contents. This may be a no-op if we just did a clone. + wt, err := repo.Worktree() + if err != nil { + return nil, DownloadFailedError{ + Repository: opts.Repository, + Inner: err, + } + } + pullRepoErr := wt.PullContext(ctx, &git.PullOptions{ RemoteName: "origin", Force: true, Auth: opts.Auth.Convert(), }) - if fetchRepoErr != nil && !errors.Is(fetchRepoErr, git.NoErrAlreadyUpToDate) { + if pullRepoErr != nil && !errors.Is(pullRepoErr, git.NoErrAlreadyUpToDate) { workTree, err := repo.Worktree() if err != nil { return nil, err @@ -74,7 +81,7 @@ func NewGitRepo(ctx context.Context, storagePath string, opts GitRepoOptions) (* workTree: workTree, }, UpdateFailedError{ Repository: opts.Repository, - Inner: fetchRepoErr, + Inner: pullRepoErr, } } @@ -108,19 +115,19 @@ func isRepoCloned(dir string) bool { return dirError == nil && len(fi) > 0 } -// Update updates the repository by fetching new content and re-checking out to +// Update updates the repository by pulling new content and re-checking out to // latest version of Revision. func (repo *GitRepo) Update(ctx context.Context) error { var err error - fetchRepoErr := repo.repo.FetchContext(ctx, &git.FetchOptions{ + pullRepoErr := repo.workTree.PullContext(ctx, &git.PullOptions{ RemoteName: "origin", Force: true, Auth: repo.opts.Auth.Convert(), }) - if fetchRepoErr != nil && !errors.Is(fetchRepoErr, git.NoErrAlreadyUpToDate) { + if pullRepoErr != nil && !errors.Is(pullRepoErr, git.NoErrAlreadyUpToDate) { return UpdateFailedError{ Repository: repo.opts.Repository, - Inner: fetchRepoErr, + Inner: pullRepoErr, } } diff --git a/pkg/flow/import_test.go b/pkg/flow/import_test.go index fd95eacfc949..768b353b76a9 100644 --- a/pkg/flow/import_test.go +++ b/pkg/flow/import_test.go @@ -4,6 +4,7 @@ import ( "context" "io/fs" "os" + "os/exec" "path/filepath" "strings" "sync" @@ -163,6 +164,101 @@ func TestImportError(t *testing.T) { } } +func TestPullUpdating(t *testing.T) { + // Previously we used fetch instead of pull, which would set the FETCH_HEAD but not HEAD + // This caused changes not to propagate if there were changes, since HEAD was pinned to whatever it was on the initial download. + // Switching to pull removes this problem at the expense of network bandwidth. + // Tried switching to FETCH_HEAD but FETCH_HEAD is only set on fetch and not initial repo clone so we would need to + // remember to always call fetch after clone. + // + // This test ensures we can pull the correct values down if they update no matter what, it works by creating a local + // file based git repo then committing a file, running the component, then updating the file in the repo. + testRepo := t.TempDir() + + contents := `declare "add" { + argument "a" {} + argument "b" {} + + export "sum" { + value = argument.a.value + argument.b.value + } +}` + main := ` +import.git "testImport" { + repository = "` + testRepo + `" + path = "math.river" + pull_frequency = "5s" +} + +testImport.add "cc" { + a = 1 + b = 1 +} +` + init := exec.Command("git", "init", testRepo) + err := init.Run() + require.NoError(t, err) + math := filepath.Join(testRepo, "math.river") + err = os.WriteFile(math, []byte(contents), 0666) + require.NoError(t, err) + add := exec.Command("git", "add", ".") + add.Dir = testRepo + err = add.Run() + require.NoError(t, err) + commit := exec.Command("git", "commit", "-m \"test\"") + commit.Dir = testRepo + err = commit.Run() + require.NoError(t, err) + + defer verifyNoGoroutineLeaks(t) + ctrl, f := setup(t, main) + err = ctrl.LoadSource(f, nil) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + defer func() { + cancel() + wg.Wait() + }() + + wg.Add(1) + go func() { + defer wg.Done() + ctrl.Run(ctx) + }() + + // Check for initial condition + require.Eventually(t, func() bool { + export := getExport[map[string]interface{}](t, ctrl, "", "testImport.add.cc") + return export["sum"] == 2 + }, 3*time.Second, 10*time.Millisecond) + + contentsMore := `declare "add" { + argument "a" {} + argument "b" {} + + export "sum" { + value = argument.a.value + argument.b.value + 1 + } +}` + err = os.WriteFile(math, []byte(contentsMore), 0666) + require.NoError(t, err) + add2 := exec.Command("git", "add", ".") + add2.Dir = testRepo + add2.Run() + + commit2 := exec.Command("git", "commit", "-m \"test2\"") + commit2.Dir = testRepo + commit2.Run() + + // Check for final condition. + require.Eventually(t, func() bool { + export := getExport[map[string]interface{}](t, ctrl, "", "testImport.add.cc") + return export["sum"] == 3 + }, 20*time.Second, 1*time.Millisecond) +} + func testConfig(t *testing.T, config string, reloadConfig string, update func()) { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, config) diff --git a/tools/make/build-container.mk b/tools/make/build-container.mk index 409d284e1f10..0daf5becb047 100644 --- a/tools/make/build-container.mk +++ b/tools/make/build-container.mk @@ -34,7 +34,7 @@ # variable names should be passed through to the container. USE_CONTAINER ?= 0 -BUILD_IMAGE_VERSION ?= 0.33.0 +BUILD_IMAGE_VERSION ?= 0.40.2 BUILD_IMAGE ?= grafana/agent-build-image:$(BUILD_IMAGE_VERSION) DOCKER_OPTS ?= -it