Cherry picks for v0.40.3 (#6679)

* loki.source.api: fix a bug where structured metadata is not passed downstream (#6508)

* Update build image to Go 1.22.1 (#6650)

* Upgrade Go to 1.22.1 (#6646)

* Upgrade Go to 1.22.1

* update build images

* Attempt to fix main: separate clean in windows docker images (#6671)

* Change git operations from fetch to pull (#6668)

* Update to using FETCH_HEAD since we are using FETCH which doesnt update HEAD.

* Update to using FETCH_HEAD since we are using FETCH which doesnt update HEAD.

* Use pull and not fetch

* add comments

* fix linting

* Update CHANGELOG.md

Co-authored-by: Robert Fratto <[email protected]>

---------

Co-authored-by: Robert Fratto <[email protected]>

---------

Co-authored-by: Mark Motliuk <[email protected]>
Co-authored-by: mattdurham <[email protected]>
Co-authored-by: Robert Fratto <[email protected]>

.drone/pipelines/build_images.jsonnet

@@ -32,7 +32,7 @@ local locals = {
         'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD',
         'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes',
         'docker buildx create --name multiarch --driver docker-container --use',
-        'docker buildx build --build-arg="GO_RUNTIME=golang:1.22.0-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image',
+        'docker buildx build --build-arg="GO_RUNTIME=golang:1.22.1-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image',
       ],
     }],
     volumes: [{
@@ -55,7 +55,7 @@ local locals = {
         'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD',
         'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes',
         'docker buildx create --name multiarch --driver docker-container --use',
-        'docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image',
+        'docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22.1-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image',
       ],
     }],
     volumes: [{

.github/workflows/check-linux-build-image.yml

@@ -25,7 +25,7 @@ jobs:
           push: false
           tags: grafana/agent-build-image:latest
           build-args: |
-            GO_RUNTIME=golang:1.22.0-bullseye
+            GO_RUNTIME=golang:1.22.1-bullseye
 
       - name: Create test Linux build image for boring crypto
         uses: docker/build-push-action@v5
@@ -34,4 +34,4 @@ jobs:
           push: false
           tags: grafana/agent-build-image:latest
           build-args: |
-            GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye
+            GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22.1-bullseye

CHANGELOG.md

@@ -7,6 +7,19 @@ This document contains a historical list of changes between releases. Only
 changes that impact end-user behavior are listed; changes to documentation or
 internal API changes are not present.
 
+v0.40.3 (2024-03-14)
+--------------------
+
+### Bugfixes
+
+- Fix a bug where structured metadata and parsed field are not passed further in `loki.source.api` (@marchellodev)
+
+- Change `import.git` to use Git pulls rather than fetches to fix scenarios where the local code did not get updated. (@mattdurham)
+
+### Other changes
+
+- Upgrade to Go 1.22.1 (@thampiotr)
+
 v0.40.2 (2024-03-05)
 --------------------
 

build-image/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-# NOTE: The GO_RUNTIME is used to switch between the default google go runtime and  mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye which is a microsoft
+# NOTE: The GO_RUNTIME is used to switch between the default Google go runtime and mcr.microsoft.com/oss/go/microsoft/golang:1.22.1-bullseye which is a Microsoft
 # fork of go that allows using windows crypto instead of boring crypto. Details at https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips
 ARG GO_RUNTIME=mustoverride
 

build-image/windows/Dockerfile

@@ -1,4 +1,4 @@
-FROM library/golang:1.22.0-windowsservercore-1809
+FROM library/golang:1.22.1-windowsservercore-1809
 
 SHELL ["powershell", "-command"]
 

cmd/grafana-agent-operator/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.33.0 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.40.2 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

cmd/grafana-agent/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.33.0 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.40.2 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

cmd/grafana-agent/Dockerfile.windows

@@ -1,4 +1,4 @@
-FROM grafana/agent-build-image:0.33.0-windows as builder
+FROM grafana/agent-build-image:0.40.2-windows as builder
 ARG VERSION
 ARG RELEASE_BUILD=1
 
@@ -10,7 +10,12 @@ SHELL ["cmd", "/S", "/C"]
 # Creating new layers can be really slow on Windows so we clean up any caches
 # we can before moving on to the next step.
 RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} make generate-ui && rm -rf web/ui/node_modules && yarn cache clean --all""
-RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} GO_TAGS='builtinassets' make agent && go clean -cache -modcache""
+
+RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} GO_TAGS='builtinassets' make agent""
+# In this case, we're separating the clean command from make agent to avoid an issue where access to some mod cache
+# files is denied immediately after make agent, for example:
+# "go: remove C:\go\pkg\mod\golang.org\[email protected]\bin\go.exe: Access is denied."
+RUN ""C:\Program Files\git\bin\bash.exe" -c "go clean -cache -modcache""
 
 # Use the smallest container possible for the final image
 FROM mcr.microsoft.com/windows/nanoserver:1809

cmd/grafana-agentctl/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.33.0 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.40.2 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

cmd/grafana-agentctl/Dockerfile.windows

@@ -1,4 +1,4 @@
-FROM grafana/agent-build-image:0.33.0-windows as builder
+FROM grafana/agent-build-image:0.40.2-windows as builder
 ARG VERSION
 ARG RELEASE_BUILD=1
 
@@ -7,9 +7,11 @@ WORKDIR /src/agent
 
 SHELL ["cmd", "/S", "/C"]
 
-# Creating new layers can be really slow on Windows so we clean up any caches
-# we can before moving on to the next step.
-RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} make agentctl && go clean -cache -modcache""
+RUN ""C:\Program Files\git\bin\bash.exe" -c "RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} make agentctl""
+# We're separating the clean command from make agent to avoid an issue where access to some mod cache
+# files is denied immediately after make agentctl, for example:
+# "go: remove C:\go\pkg\mod\golang.org\[email protected]\bin\go.exe: Access is denied."
+RUN ""C:\Program Files\git\bin\bash.exe" -c "go clean -cache -modcache""
 
 # Use the smallest container possible for the final image
 FROM mcr.microsoft.com/windows/nanoserver:1809

component/loki/source/api/internal/lokipush/push_api_server.go

@@ -185,7 +185,9 @@ func (s *PushAPIServer) handleLoki(w http.ResponseWriter, r *http.Request) {
 			e := loki.Entry{
 				Labels: filtered.Clone(),
 				Entry: logproto.Entry{
-					Line: entry.Line,
+					Line:               entry.Line,
+					StructuredMetadata: entry.StructuredMetadata,
+					Parsed:             entry.Parsed,
 				},
 			}
 			if keepTimestamp {

component/loki/source/api/internal/lokipush/push_api_server_test.go

@@ -23,6 +23,7 @@ import (
 	frelabel "github.com/grafana/agent/component/common/relabel"
 	"github.com/grafana/dskit/flagext"
 	"github.com/grafana/loki/pkg/logproto"
+	"github.com/grafana/loki/pkg/push"
 	"github.com/grafana/river"
 	"github.com/phayes/freeport"
 	"github.com/prometheus/client_golang/prometheus"
@@ -79,6 +80,10 @@ regex = "dropme"
 			Entry: logproto.Entry{
 				Timestamp: time.Unix(int64(i), 0),
 				Line:      "line" + strconv.Itoa(i),
+				StructuredMetadata: push.LabelsAdapter{
+					{Name: "i", Value: strconv.Itoa(i)},
+					{Name: "anotherMetaData", Value: "val"},
+				},
 			},
 		}
 	}
@@ -98,9 +103,18 @@ regex = "dropme"
 		"pushserver": "pushserver1",
 		"stream":     "stream1",
 	}
+
+	expectedStructuredMetadata := push.LabelsAdapter{
+		{Name: "i", Value: strconv.Itoa(0)},
+		{Name: "anotherMetaData", Value: "val"},
+	}
+
 	// Spot check the first value in the result to make sure relabel rules were applied properly
 	require.Equal(t, expectedLabels, eh.Received()[0].Labels)
 
+	// Spot check the first value in the result to make sure structured metadata was received properly
+	require.Equal(t, expectedStructuredMetadata, eh.Received()[0].StructuredMetadata)
+
 	// With keep timestamp enabled, verify timestamp
 	require.Equal(t, time.Unix(99, 0).Unix(), eh.Received()[99].Timestamp.Unix())
 

docs/Makefile

@@ -11,11 +11,11 @@ include docs.mk
 docs: check-cloudwatch-integration
 
 check-cloudwatch-integration:
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md
 
 generate-cloudwatch-integration:
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.21-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go generate
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run pkg/integrations/cloudwatch_exporter/docs/doc.go generate
 
 sources/assets/hierarchy.svg: sources/operator/hierarchy.dot
 	cat $< | $(PODMAN) run --rm -i nshine/dot dot -Tsvg > $@

docs/sources/flow/reference/components/module.git.md

@@ -41,12 +41,12 @@ module.git "LABEL" {
 
 The following arguments are supported:
 
-Name | Type | Description | Default | Required
----- | ---- | ----------- | ------- | --------
-`repository` | `string` | The Git repository address to retrieve the module from. | | yes
-`revision` | `string` | The Git revision to retrieve the module from. | `"HEAD"` | no
-`path` | `string` | The path in the repository where the module is stored. | | yes
-`pull_frequency` | `duration` | The frequency to pull the repository for updates. | `"60s"` | no
+Name             | Type       | Description                                             | Default  | Required
+-----------------|------------|---------------------------------------------------------|----------|---------
+`repository`     | `string`   | The Git repository address to retrieve the module from. |          | yes
+`revision`       | `string`   | The Git revision to retrieve the module from.           | `"HEAD"` | no
+`path`           | `string`   | The path in the repository where the module is stored.  |          | yes
+`pull_frequency` | `duration` | The frequency to pull the repository for updates.       | `"60s"`  | no
 
 The `repository` attribute must be set to a repository address that would be
 recognized by Git with a `git clone REPOSITORY_ADDRESS` command, such as

go.mod

@@ -1,6 +1,6 @@
 module github.com/grafana/agent
 
-go 1.21.0
+go 1.22.1
 
 require (
 	cloud.google.com/go/pubsub v1.33.0

integration-tests/configs/otel-metrics-gen/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21 as build
+FROM golang:1.22.1 as build
 WORKDIR /app/
 COPY go.mod go.sum ./
 RUN go mod download

integration-tests/configs/prom-gen/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21 as build
+FROM golang:1.22.1 as build
 WORKDIR /app/
 COPY go.mod go.sum ./
 RUN go mod download

internal/vcs/git.go

@@ -29,7 +29,7 @@ type GitRepo struct {
 // managed at storagePath.
 //
 // If storagePath is empty on disk, NewGitRepo initializes GitRepo by cloning
-// the repository. Otherwise, NewGitRepo will do a fetch.
+// the repository. Otherwise, NewGitRepo will do a pull.
 //
 // After GitRepo is initialized, it checks out to the Revision specified in
 // GitRepoOptions.
@@ -57,13 +57,20 @@ func NewGitRepo(ctx context.Context, storagePath string, opts GitRepoOptions) (*
 		}
 	}
 
-	// Fetch the latest contents. This may be a no-op if we just did a clone.
-	fetchRepoErr := repo.FetchContext(ctx, &git.FetchOptions{
+	// Pulls the latest contents. This may be a no-op if we just did a clone.
+	wt, err := repo.Worktree()
+	if err != nil {
+		return nil, DownloadFailedError{
+			Repository: opts.Repository,
+			Inner:      err,
+		}
+	}
+	pullRepoErr := wt.PullContext(ctx, &git.PullOptions{
 		RemoteName: "origin",
 		Force:      true,
 		Auth:       opts.Auth.Convert(),
 	})
-	if fetchRepoErr != nil && !errors.Is(fetchRepoErr, git.NoErrAlreadyUpToDate) {
+	if pullRepoErr != nil && !errors.Is(pullRepoErr, git.NoErrAlreadyUpToDate) {
 		workTree, err := repo.Worktree()
 		if err != nil {
 			return nil, err
@@ -74,7 +81,7 @@ func NewGitRepo(ctx context.Context, storagePath string, opts GitRepoOptions) (*
 				workTree: workTree,
 			}, UpdateFailedError{
 				Repository: opts.Repository,
-				Inner:      fetchRepoErr,
+				Inner:      pullRepoErr,
 			}
 	}
 
@@ -108,19 +115,19 @@ func isRepoCloned(dir string) bool {
 	return dirError == nil && len(fi) > 0
 }
 
-// Update updates the repository by fetching new content and re-checking out to
+// Update updates the repository by pulling new content and re-checking out to
 // latest version of Revision.
 func (repo *GitRepo) Update(ctx context.Context) error {
 	var err error
-	fetchRepoErr := repo.repo.FetchContext(ctx, &git.FetchOptions{
+	pullRepoErr := repo.workTree.PullContext(ctx, &git.PullOptions{
 		RemoteName: "origin",
 		Force:      true,
 		Auth:       repo.opts.Auth.Convert(),
 	})
-	if fetchRepoErr != nil && !errors.Is(fetchRepoErr, git.NoErrAlreadyUpToDate) {
+	if pullRepoErr != nil && !errors.Is(pullRepoErr, git.NoErrAlreadyUpToDate) {
 		return UpdateFailedError{
 			Repository: repo.opts.Repository,
-			Inner:      fetchRepoErr,
+			Inner:      pullRepoErr,
 		}
 	}