Merge branch 'main' into otel-awsfirehose

.drone/pipelines/crosscompile.jsonnet

@@ -22,6 +22,7 @@ local os_arch_tuples = [
   // Windows
   { name: 'Windows amd64', os: 'windows', arch: 'amd64' },
 
+
   // FreeBSD
   { name: 'FreeBSD amd64', os: 'freebsd', arch: 'amd64' },
 ];
@@ -37,15 +38,22 @@ local targets = [
 local targets_boringcrypto = [
   'agent-boringcrypto',
 ];
+local targets_boringcrypto_windows = [
+  'agent-flow-windows-boringcrypto',
+];
+
 
 local os_arch_types_boringcrypto = [
   // Linux boringcrypto
   { name: 'Linux amd64 boringcrypto', os: 'linux', arch: 'amd64', experiment: 'boringcrypto' },
   { name: 'Linux arm64 boringcrypto', os: 'linux', arch: 'arm64', experiment: 'boringcrypto' },
 ];
+local windows_os_arch_types_boringcrypto = [
+  // Windows boringcrypto
+  { name: 'Windows amd64', os: 'windows', arch: 'amd64', experiment: 'cngcrypto' },
+];
 
-
-std.flatMap(function(target) (
+local build_environments(targets, tuples, image) = std.flatMap(function(target) (
   std.map(function(platform) (
     pipelines.linux('Build %s (%s)' % [target, platform.name]) {
       local env = {
@@ -56,47 +64,26 @@ std.flatMap(function(target) (
         target: target,
 
         tags: go_tags[platform.os],
-      },
+      } + (if 'experiment' in platform then { GOEXPERIMENT: platform.experiment } else { }),
 
       trigger: {
         event: ['pull_request'],
       },
-      steps: [{
-        name: 'Build',
-        image: build_image.linux,
-        commands: [
-          'make generate-ui',
-          'GO_TAGS="%(tags)s" GOOS=%(GOOS)s GOARCH=%(GOARCH)s GOARM=%(GOARM)s make %(target)s' % env,
-        ],
-      }],
-    }
-  ), os_arch_tuples)
-), targets) +
-std.flatMap(function(target) (
-  std.map(function(platform) (
-    pipelines.linux('Build %s (%s)' % [target, platform.name]) {
-      local env = {
-        GOOS: platform.os,
-        GOARCH: platform.arch,
-        GOARM: if 'arm' in platform then platform.arm else '',
-        GOEXPERIMENT: platform.experiment,
-
-        target: target,
-
-        tags: go_tags[platform.os],
-      },
 
-      trigger: {
-        event: ['pull_request'],
-      },
       steps: [{
         name: 'Build',
-        image: build_image.linux,
+        image: image,
         commands: [
           'make generate-ui',
-          'GO_TAGS="%(tags)s" GOOS=%(GOOS)s GOARCH=%(GOARCH)s GOARM=%(GOARM)s GOEXPERIMENT=%(GOEXPERIMENT)s make %(target)s' % env,
+          (if 'GOEXPERIMENT' in env 
+           then 'GO_TAGS="%(tags)s" GOOS=%(GOOS)s GOARCH=%(GOARCH)s GOARM=%(GOARM)s GOEXPERIMENT=%(GOEXPERIMENT)s make %(target)s' % env
+           else 'GO_TAGS="%(tags)s" GOOS=%(GOOS)s GOARCH=%(GOARCH)s GOARM=%(GOARM)s make %(target)s') % env,
         ],
       }],
     }
-  ), os_arch_types_boringcrypto)
-), targets_boringcrypto)
+  ), tuples)
+), targets);
+
+build_environments(targets, os_arch_tuples, build_image.linux) +
+build_environments(targets_boringcrypto, os_arch_types_boringcrypto, build_image.linux) +
+build_environments(targets_boringcrypto_windows, windows_os_arch_types_boringcrypto, build_image.boringcrypto)

.gitattributes

@@ -1 +1,3 @@
 pkg/operator/crds/*.yaml linguist-generated=true
+
+*.txtar text eol=lf

.github/workflows/autolock.yml

@@ -0,0 +1,25 @@
+name: Lock closed issues and PRs
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: '0 0 * * *'
+
+permissions:
+  issues: write
+  pull-requests: write
+  discussions: write
+
+concurrency:
+  group: lock-threads
+
+jobs:
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/lock-threads@v5
+        with:
+          pr-inactive-days: 30
+          issue-inactive-days: 30
+          add-issue-labels: 'frozen-due-to-age'
+          add-pr-labels: 'frozen-due-to-age'
+          process-only: 'issues, prs'

CHANGELOG.md

@@ -10,6 +10,16 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
+### Features
+
+- Added a new CLI flag `--stability.level` which defines the minimum stability
+  level required for the features that the agent is allowed to use. Default is `experimental`. (@thampiotr)
+
+- A new `loki.rules.kubernetes` component that discovers `PrometheusRule` Kubernetes resources and loads them into a Loki Ruler instance. (@EStork09)
+
+v0.40.0 (2024-02-27)
+--------------------
+
 ### Breaking changes
 
 - Prohibit the configuration of services within modules. (@wildum)
@@ -18,8 +28,15 @@ Main (unreleased)
 
 - Rename component `prometheus.exporter.agent` to `prometheus.exporter.self` to clear up ambiguity. (@hainenber)
 
+### Deprecations
+
+- Module components have been deprecated in favor of import and declare configuration blocks. These deprecated components will be removed in the next release. (@wildum)
+
 ### Features
 
+- Modules have been redesigned to split the import logic from the instantiation.
+  You can now define custom components via the `declare` config block and import modules via `import.git`, `import.http`, `import.string`, `import.file`. (@wildum)
+
 - A new `discovery.process` component for discovering Linux OS processes on the current host. (@korniltsev)
 
 - A new `pyroscope.java` component for profiling Java processes using async-profiler. (@korniltsev)
@@ -33,8 +50,12 @@ Main (unreleased)
 - Expose track_timestamps_staleness on Prometheus scraping, to fix the issue where container metrics live for 5 minutes after the container disappears. (@ptodev)
 
 - Introduce the `remotecfg` service that enables loading configuration from a
-  remote endpoint. (@tpaschalis) 
-
+  remote endpoint. (@tpaschalis)
+
+- Add `otelcol.connector.host_info` component to gather usage metrics for cloud users. (@rlankfo, @jcreixell)
+
+- Add Windows boringcrypto build and executable. (@mattdurham)
+
 ### Enhancements
 
 - Include line numbers in profiles produced by `pyrsocope.java` component. (@korniltsev)
@@ -70,6 +91,8 @@ Main (unreleased)
 
 - Batch staleness tracking to reduce mutex contention and increase performance. (@mattdurham)
 
+- Python profiling using eBPF is now aggregated now by kernel space. [PR](https://github.com/grafana/pyroscope/pull/2996) (@korniltsev)
+
 ### Bugfixes
 
 - Fix an issue in `remote.s3` where the exported content of an object would be an empty string if `remote.s3` failed to fully retrieve
@@ -88,15 +111,28 @@ Main (unreleased)
 - Fix an issue with static integrations-next marshaling where non singletons
   would cause `/-/config` to fail to marshal. (@erikbaranowski)
 
-- Fix divide-by-zero issue when sharding targets. (@hainenber) 
+- Fix an issue where agent logs are emitted before the logging format
+  is correctly determined. (@hainenber)
+
+- Fix divide-by-zero issue when sharding targets. (@hainenber)
 
 - Fix bug where custom headers were not actually being set in loki client. (@captncraig)
 
+- Fix missing measurement type field in the KeyVal() conversion function for measurments. @vanugrah)
+
 - Fix `ResolveEndpointV2 not found` for AWS-related components. (@hainenber)
 
 - Fix OTEL metrics not getting collected after reload. (@hainenber)
 
-- Fix bug in `pyroscope.ebpf` component when elf's PT_LOAD section is not page aligned . [PR](https://github.com/grafana/pyroscope/pull/2983)  (@korniltsev)
+- Fix bug in `pyroscope.ebpf` component when elf's PT_LOAD section is not page aligned. [PR](https://github.com/grafana/pyroscope/pull/2983)  (@korniltsev)
+
+- Pyroscope eBPF profiling now respects the PID namespace Grafana Agent is running in. [PR](https://github.com/grafana/pyroscope/pull/3008) (@simonswine)
+
+- Fix an issue where the configuration of the `http` and `remotecfg` blocks get ignored after loading a module. (@erikbaranowski)
+
+- Fix an issue where changing the configuration of `loki.write` would cause a panic. (@rfratto)
+
+- Fix issue where registry was not being properly deleted. (@mattdurham)
 
 ### Other changes
 

Makefile

@@ -21,13 +21,14 @@
 ##
 ## Targets for building binaries:
 ##
-##   binaries                Compiles all binaries.
-##   agent                   Compiles cmd/grafana-agent to $(AGENT_BINARY)
-##   agent-boringcrypto      Compiles cmd/grafana-agent with GOEXPERIMENT=boringcrypto to $(AGENT_BORINGCRYPTO_BINARY)
-##   agent-flow              Compiles cmd/grafana-agent-flow to $(FLOW_BINARY)
-##   agent-service           Compiles cmd/grafana-agent-service to $(SERVICE_BINARY)
-##   agentctl                Compiles cmd/grafana-agentctl to $(AGENTCTL_BINARY)
-##   operator                Compiles cmd/grafana-agent-operator to $(OPERATOR_BINARY)
+##   binaries                        Compiles all binaries.
+##   agent                           Compiles cmd/grafana-agent to $(AGENT_BINARY)
+##   agent-boringcrypto              Compiles cmd/grafana-agent with GOEXPERIMENT=boringcrypto to $(AGENT_BORINGCRYPTO_BINARY)
+##   agent-flow                      Compiles cmd/grafana-agent-flow to $(FLOW_BINARY)
+##   agent-flow-windows-boringcrypto Compiles cmd/grafana-agent-flow to $(FLOW_BINARY)-windows-boringcrypto
+##   agent-service                   Compiles cmd/grafana-agent-service to $(SERVICE_BINARY)
+##   agentctl                        Compiles cmd/grafana-agentctl to $(AGENTCTL_BINARY)
+##   operator                        Compiles cmd/grafana-agent-operator to $(OPERATOR_BINARY)
 ##
 ## Targets for building Docker images:
 ##
@@ -98,6 +99,7 @@ AGENTCTL_IMAGE                          ?= grafana/agentctl:latest
 OPERATOR_IMAGE                          ?= grafana/agent-operator:latest
 AGENT_BINARY                            ?= build/grafana-agent
 AGENT_BORINGCRYPTO_BINARY               ?= build/grafana-agent-boringcrypto
+AGENT_BORINGCRYPTO_WINDOWS_BINARY       ?= build/agent-flow-windows-boringcrypto.exe
 FLOW_BINARY                             ?= build/grafana-agent-flow
 SERVICE_BINARY                          ?= build/grafana-agent-service
 AGENTCTL_BINARY                         ?= build/grafana-agentctl
@@ -192,6 +194,13 @@ else
 	GOEXPERIMENT=boringcrypto $(GO_ENV) go build $(GO_FLAGS) -o $(AGENT_BORINGCRYPTO_BINARY) ./cmd/grafana-agent
 endif
 
+agent-flow-windows-boringcrypto:
+ifeq ($(USE_CONTAINER),1)
+	$(RERUN_IN_CONTAINER)
+else
+	GOEXPERIMENT=cngcrypto $(GO_ENV) go build $(GO_FLAGS) -tags cngcrypto -o $(AGENT_BORINGCRYPTO_WINDOWS_BINARY) ./cmd/grafana-agent-flow
+endif
+
 
 agent-flow:
 ifeq ($(USE_CONTAINER),1)

cmd/grafana-agent-operator/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.4 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.32.0 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

cmd/grafana-agent/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.4 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.32.0 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

cmd/grafana-agent/Dockerfile.windows

@@ -1,4 +1,4 @@
-FROM grafana/agent-build-image:0.30.4-windows as builder
+FROM grafana/agent-build-image:0.32.0-windows as builder
 ARG VERSION
 ARG RELEASE_BUILD=1
 

cmd/grafana-agentctl/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.4 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.32.0 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

cmd/grafana-agentctl/Dockerfile.windows

@@ -1,4 +1,4 @@
-FROM grafana/agent-build-image:0.30.4-windows as builder
+FROM grafana/agent-build-image:0.32.0-windows as builder
 ARG VERSION
 ARG RELEASE_BUILD=1
 

cmd/internal/flowmode/cmd_run.go

@@ -21,6 +21,7 @@ import (
 	"github.com/grafana/agent/converter"
 	convert_diag "github.com/grafana/agent/converter/diag"
 	"github.com/grafana/agent/internal/agentseed"
+	"github.com/grafana/agent/internal/featuregate"
 	"github.com/grafana/agent/pkg/boringcrypto"
 	"github.com/grafana/agent/pkg/config/instrumentation"
 	"github.com/grafana/agent/pkg/flow"
@@ -51,6 +52,7 @@ func runCommand() *cobra.Command {
 		inMemoryAddr:          "agent.internal:12345",
 		httpListenAddr:        "127.0.0.1:12345",
 		storagePath:           "data-agent/",
+		minStability:          featuregate.StabilityExperimental,
 		uiPrefix:              "/",
 		disableReporting:      false,
 		enablePprof:           true,
@@ -97,13 +99,15 @@ depending on the nature of the reload error.
 		},
 	}
 
+	// Server flags
 	cmd.Flags().
 		StringVar(&r.httpListenAddr, "server.http.listen-addr", r.httpListenAddr, "Address to listen for HTTP traffic on")
 	cmd.Flags().StringVar(&r.inMemoryAddr, "server.http.memory-addr", r.inMemoryAddr, "Address to listen for in-memory HTTP traffic on. Change if it collides with a real address")
-	cmd.Flags().StringVar(&r.storagePath, "storage.path", r.storagePath, "Base directory where components can store data")
 	cmd.Flags().StringVar(&r.uiPrefix, "server.http.ui-path-prefix", r.uiPrefix, "Prefix to serve the HTTP UI at")
 	cmd.Flags().
 		BoolVar(&r.enablePprof, "server.http.enable-pprof", r.enablePprof, "Enable /debug/pprof profiling endpoints.")
+
+	// Cluster flags
 	cmd.Flags().
 		BoolVar(&r.clusterEnabled, "cluster.enabled", r.clusterEnabled, "Start in clustered mode")
 	cmd.Flags().
@@ -122,18 +126,25 @@ depending on the nature of the reload error.
 		IntVar(&r.ClusterMaxJoinPeers, "cluster.max-join-peers", r.ClusterMaxJoinPeers, "Number of peers to join from the discovered set")
 	cmd.Flags().
 		StringVar(&r.clusterName, "cluster.name", r.clusterName, "The name of the cluster to join")
-	cmd.Flags().
-		BoolVar(&r.disableReporting, "disable-reporting", r.disableReporting, "Disable reporting of enabled components to Grafana.")
+
+	// Config flags
 	cmd.Flags().StringVar(&r.configFormat, "config.format", r.configFormat, fmt.Sprintf("The format of the source file. Supported formats: %s.", supportedFormatsList()))
 	cmd.Flags().BoolVar(&r.configBypassConversionErrors, "config.bypass-conversion-errors", r.configBypassConversionErrors, "Enable bypassing errors when converting")
 	cmd.Flags().StringVar(&r.configExtraArgs, "config.extra-args", r.configExtraArgs, "Extra arguments from the original format used by the converter. Multiple arguments can be passed by separating them with a space.")
+
+	// Misc flags
+	cmd.Flags().
+		BoolVar(&r.disableReporting, "disable-reporting", r.disableReporting, "Disable reporting of enabled components to Grafana.")
+	cmd.Flags().StringVar(&r.storagePath, "storage.path", r.storagePath, "Base directory where components can store data")
+	cmd.Flags().Var(&r.minStability, "stability.level", fmt.Sprintf("Minimum stability level of features to enable. Supported values: %s", strings.Join(featuregate.AllowedValues(), ", ")))
 	return cmd
 }
 
 type flowRun struct {
 	inMemoryAddr                 string
 	httpListenAddr               string
 	storagePath                  string
+	minStability                 featuregate.Stability
 	uiPrefix                     string
 	enablePprof                  bool
 	disableReporting             bool
@@ -162,7 +173,8 @@ func (fr *flowRun) Run(configPath string) error {
 		return fmt.Errorf("path argument not provided")
 	}
 
-	l, err := logging.New(os.Stderr, logging.DefaultOptions)
+	// Buffer logs until log format has been determined
+	l, err := logging.NewDeferred(os.Stderr)
 	if err != nil {
 		return fmt.Errorf("building logger: %w", err)
 	}
@@ -264,10 +276,11 @@ func (fr *flowRun) Run(configPath string) error {
 	agentseed.Init(fr.storagePath, l)
 
 	f := flow.New(flow.Options{
-		Logger:   l,
-		Tracer:   t,
-		DataPath: fr.storagePath,
-		Reg:      reg,
+		Logger:       l,
+		Tracer:       t,
+		DataPath:     fr.storagePath,
+		Reg:          reg,
+		MinStability: fr.minStability,
 		Services: []service.Service{
 			httpService,
 			uiService,

component/all/all.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/grafana/agent/component/loki/echo"                                // Import loki.echo
 	_ "github.com/grafana/agent/component/loki/process"                             // Import loki.process
 	_ "github.com/grafana/agent/component/loki/relabel"                             // Import loki.relabel
+	_ "github.com/grafana/agent/component/loki/rules/kubernetes"                    // Import loki.rules.kubernetes
 	_ "github.com/grafana/agent/component/loki/source/api"                          // Import loki.source.api
 	_ "github.com/grafana/agent/component/loki/source/aws_firehose"                 // Import loki.source.awsfirehose
 	_ "github.com/grafana/agent/component/loki/source/azure_event_hubs"             // Import loki.source.azure_event_hubs
@@ -65,6 +66,7 @@ import (
 	_ "github.com/grafana/agent/component/otelcol/auth/headers"                     // Import otelcol.auth.headers
 	_ "github.com/grafana/agent/component/otelcol/auth/oauth2"                      // Import otelcol.auth.oauth2
 	_ "github.com/grafana/agent/component/otelcol/auth/sigv4"                       // Import otelcol.auth.sigv4
+	_ "github.com/grafana/agent/component/otelcol/connector/host_info"              // Import otelcol.connector.host_info
 	_ "github.com/grafana/agent/component/otelcol/connector/servicegraph"           // Import otelcol.connector.servicegraph
 	_ "github.com/grafana/agent/component/otelcol/connector/spanlogs"               // Import otelcol.connector.spanlogs
 	_ "github.com/grafana/agent/component/otelcol/connector/spanmetrics"            // Import otelcol.connector.spanmetrics