Merge branch 'main' into bump-go-1.21.6

CHANGELOG.md

@@ -10,21 +10,24 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
-### Security fixes
+### Breaking changes
 
-- Fixes following vulnerabilities (@hainenber)
-  - [GO-2023-2409](https://github.com/advisories/GHSA-mhpq-9638-x6pw)
-  - [GO-2023-2412](https://github.com/advisories/GHSA-7ww5-4wqc-m92c)
-  - [CVE-2023-49568](https://github.com/advisories/GHSA-mw99-9chc-xw7r)
+- Prohibit the configuration of services within modules. (@wildum)
+
+- For `otelcol.exporter` components, change the default value of `disable_high_cardinality_metrics` to `true`. (@ptodev)
 
 ### Features
 
 - A new `discovery.process` component for discovering Linux OS processes on the current host. (@korniltsev)
 
 - A new `pyroscope.java` component for profiling Java processes using async-profiler. (@korniltsev)
 
+- A new `otelcol.processor.resourcedetection` component which inserts resource attributes 
+  to OTLP telemetry based on the host on which Grafana Agent is running. (@ptodev)
+
 ### Enhancements
 
+- Include line numbers in profiles produced by `pyrsocope.java` component. (@korniltsev)
 - Add an option to the windows static mode installer for expanding environment vars in the yaml config. (@erikbaranowski)
 - Add authentication support to `loki.source.awsfirehose` (@sberz)
 
@@ -33,6 +36,21 @@ Main (unreleased)
 - Expose `physical_disk` collector from `windows_exporter` v0.24.0 to 
   Flow configuration. (@hainenber)
 
+- Renamed Grafana Agent Mixin's "prometheus.remote_write" dashboard to
+  "Prometheus Components" and added charts for `prometheus.scrape` success rate
+  and duration metrics. (@thampiotr)
+
+- Removed `ClusterLamportClockDrift` and `ClusterLamportClockStuck` alerts from
+  Grafana Agent Mixin to focus on alerting on symptoms. (@thampiotr)
+
+- Increased clustering alert periods to 10 minutes to improve the
+  signal-to-noise ratio in Grafana Agent Mixin. (@thampiotr)
+
+- `mimir.rules.kubernetes` has a new `prometheus_http_prefix` argument to configure 
+  the HTTP endpoint on which to connect to Mimir's API. (@hainenber)
+
+- `service_name` label is inferred from discovery meta labels in `pyroscope.java` (@korniltsev)
+
 ### Bugfixes
 
 - Fix an issue in `remote.s3` where the exported content of an object would be an empty string if `remote.s3` failed to fully retrieve
@@ -43,6 +61,14 @@ Main (unreleased)
 - Fix a duplicate metrics registration panic when sending metrics to an static
   mode metric instance's write handler. (@tpaschalis)
 
+- Fix issue causing duplicate logs when a docker target is restarted. (@captncraig)
+
+- Fix an issue where blocks having the same type and the same label across
+  modules could result in missed updates. (@thampiotr)
+
+- Fix an issue with static integrations-next marshaling where non singletons
+  would cause `/-/config` to fail to marshal. (@erikbaranowski)
+
 ### Other changes
 
 - Removed support for Windows 2012 in line with Microsoft end of life. (@mattdurham)
@@ -53,6 +79,32 @@ Main (unreleased)
 
 - Use Go 1.21.6 for builds. (@hainenber)
 
+v0.39.2 (2024-1-31)
+--------------------
+
+### Bugfixes
+
+- Fix error introduced in v0.39.0 preventing remote write to Amazon Managed Prometheus. (@captncraig)
+
+- An error will be returned in the converter from Static to Flow when `scrape_integration` is set
+  to `true` but no `remote_write` is defined. (@erikbaranowski)
+
+
+v0.39.1 (2024-01-19)
+--------------------
+
+### Security fixes
+
+- Fixes following vulnerabilities (@hainenber)
+  - [GO-2023-2409](https://github.com/advisories/GHSA-mhpq-9638-x6pw)
+  - [GO-2023-2412](https://github.com/advisories/GHSA-7ww5-4wqc-m92c)
+  - [CVE-2023-49568](https://github.com/advisories/GHSA-mw99-9chc-xw7r)
+
+### Bugfixes
+
+- Fix issue where installing the Windows Agent Flow installer would hang then crash. (@mattdurham)
+
+
 v0.39.0 (2024-01-09)
 --------------------
 

cmd/grafana-agent-operator/DEVELOPERS.md

@@ -74,7 +74,7 @@ running.
 ### Apply the CRDs
 
 Generated CRDs used by the operator can be found in [the Production
-folder](../../production/operator/crds). Deploy them from the root of the
+folder](../../operations/agent-static-operator/crds). Deploy them from the root of the
 repository with:
 
 ```

cmd/internal/flowmode/cmd_run.go

@@ -360,7 +360,7 @@ func getEnabledComponentsFunc(f *flow.Flow) func() map[string]interface{} {
 		components := component.GetAllComponents(f, component.InfoOptions{})
 		componentNames := map[string]struct{}{}
 		for _, c := range components {
-			componentNames[c.Registration.Name] = struct{}{}
+			componentNames[c.ComponentName] = struct{}{}
 		}
 		return map[string]interface{}{"enabled-components": maps.Keys(componentNames)}
 	}

component/all/all.go

@@ -82,6 +82,7 @@ import (
 	_ "github.com/grafana/agent/component/otelcol/processor/k8sattributes"          // Import otelcol.processor.k8sattributes
 	_ "github.com/grafana/agent/component/otelcol/processor/memorylimiter"          // Import otelcol.processor.memory_limiter
 	_ "github.com/grafana/agent/component/otelcol/processor/probabilistic_sampler"  // Import otelcol.processor.probabilistic_sampler
+	_ "github.com/grafana/agent/component/otelcol/processor/resourcedetection"      // Import otelcol.processor.resourcedetection
 	_ "github.com/grafana/agent/component/otelcol/processor/span"                   // Import otelcol.processor.span
 	_ "github.com/grafana/agent/component/otelcol/processor/tail_sampling"          // Import otelcol.processor.tail_sampling
 	_ "github.com/grafana/agent/component/otelcol/processor/transform"              // Import otelcol.processor.transform

component/component_provider.go

@@ -93,8 +93,8 @@ type Info struct {
 	// this component depends on, or is depended on by, respectively.
 	References, ReferencedBy []string
 
-	Registration Registration // Component registration.
-	Health       Health       // Current component health.
+	ComponentName string // Name of the component.
+	Health        Health // Current component health.
 
 	Arguments Arguments   // Current arguments value of the component.
 	Exports   Exports     // Current exports value of the component.
@@ -157,7 +157,7 @@ func (info *Info) MarshalJSON() ([]byte, error) {
 	}
 
 	return json.Marshal(&componentDetailJSON{
-		Name:         info.Registration.Name,
+		Name:         info.ComponentName,
 		Type:         "block",
 		ModuleID:     info.ID.ModuleID,
 		LocalID:      info.ID.LocalID,

component/loki/source/docker/internal/dockertarget/target.go

@@ -219,6 +219,7 @@ func (t *Target) process(r io.Reader, logStreamLset model.LabelSet) {
 		// labels (e.g. duplicated and relabeled), but this shouldn't be the
 		// case anyway.
 		t.positions.Put(positions.CursorKey(t.containerName), t.labelsStr, ts.Unix())
+		t.since = ts.Unix()
 	}
 }
 

component/loki/source/docker/internal/dockertarget/target_test.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -24,14 +23,21 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestDockerTarget(t *testing.T) {
 	h := func(w http.ResponseWriter, r *http.Request) {
 		switch path := r.URL.Path; {
 		case strings.HasSuffix(path, "/logs"):
-			dat, err := os.ReadFile("testdata/flog.log")
+			var filePath string
+			if strings.Contains(r.URL.RawQuery, "since=0") {
+				filePath = "testdata/flog.log"
+			} else {
+				filePath = "testdata/flog_after_restart.log"
+			}
+			dat, err := os.ReadFile(filePath)
 			require.NoError(t, err)
 			_, err = w.Write(dat)
 			require.NoError(t, err)
@@ -76,25 +82,56 @@ func TestDockerTarget(t *testing.T) {
 	require.NoError(t, err)
 	tgt.StartIfNotRunning()
 
-	require.Eventually(t, func() bool {
-		return len(entryHandler.Received()) >= 5
-	}, 5*time.Second, 100*time.Millisecond)
-
-	received := entryHandler.Received()
-	sort.Slice(received, func(i, j int) bool {
-		return received[i].Timestamp.Before(received[j].Timestamp)
-	})
-
 	expectedLines := []string{
 		"5.3.69.55 - - [09/Dec/2021:09:15:02 +0000] \"HEAD /brand/users/clicks-and-mortar/front-end HTTP/2.0\" 503 27087",
 		"101.54.183.185 - - [09/Dec/2021:09:15:03 +0000] \"POST /next-generation HTTP/1.0\" 416 11468",
 		"69.27.137.160 - runolfsdottir2670 [09/Dec/2021:09:15:03 +0000] \"HEAD /content/visionary/engineer/cultivate HTTP/1.1\" 302 2975",
 		"28.104.242.74 - - [09/Dec/2021:09:15:03 +0000] \"PATCH /value-added/cultivate/systems HTTP/2.0\" 405 11843",
 		"150.187.51.54 - satterfield1852 [09/Dec/2021:09:15:03 +0000] \"GET /incentivize/deliver/innovative/cross-platform HTTP/1.1\" 301 13032",
 	}
-	actualLines := make([]string, 0, 5)
-	for _, entry := range received[:5] {
-		actualLines = append(actualLines, entry.Line)
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		assertExpectedLog(c, entryHandler, expectedLines)
+	}, 5*time.Second, 100*time.Millisecond, "Expected log lines were not found within the time limit.")
+
+	tgt.Stop()
+	entryHandler.Clear()
+	// restart target to simulate container restart
+	tgt.StartIfNotRunning()
+	expectedLinesAfterRestart := []string{
+		"243.115.12.215 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /morph/exploit/granular HTTP/1.0\" 500 26468",
+		"221.41.123.237 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /user-centric/whiteboard HTTP/2.0\" 205 22487",
+		"89.111.144.144 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /open-source/e-commerce HTTP/1.0\" 401 11092",
+		"62.180.191.187 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /cultivate/integrate/technologies HTTP/2.0\" 302 12979",
+		"156.249.2.192 - - [09/Dec/2023:09:16:57 +0000] \"POST /revolutionize/mesh/metrics HTTP/2.0\" 401 5297",
+	}
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		assertExpectedLog(c, entryHandler, expectedLinesAfterRestart)
+	}, 5*time.Second, 100*time.Millisecond, "Expected log lines after restart were not found within the time limit.")
+}
+
+// assertExpectedLog will verify that all expectedLines were received, in any order, without duplicates.
+func assertExpectedLog(c *assert.CollectT, entryHandler *fake.Client, expectedLines []string) {
+	logLines := entryHandler.Received()
+	testLogLines := make(map[string]int)
+	for _, l := range logLines {
+		if containsString(expectedLines, l.Line) {
+			testLogLines[l.Line] += 1
+		}
+	}
+	// assert that all log lines were received
+	assert.Len(c, testLogLines, len(expectedLines))
+	// assert that there are no duplicated log lines
+	for _, v := range testLogLines {
+		assert.Equal(c, v, 1)
+	}
+}
+
+func containsString(slice []string, str string) bool {
+	for _, item := range slice {
+		if item == str {
+			return true
+		}
 	}
-	require.ElementsMatch(t, actualLines, expectedLines)
+	return false
 }

component/mimir/rules/kubernetes/rules.go

@@ -261,10 +261,11 @@ func (c *Component) init() error {
 	httpClient := c.args.HTTPClientConfig.Convert()
 
 	c.mimirClient, err = mimirClient.New(c.log, mimirClient.Config{
-		ID:               c.args.TenantID,
-		Address:          c.args.Address,
-		UseLegacyRoutes:  c.args.UseLegacyRoutes,
-		HTTPClientConfig: *httpClient,
+		ID:                   c.args.TenantID,
+		Address:              c.args.Address,
+		UseLegacyRoutes:      c.args.UseLegacyRoutes,
+		PrometheusHTTPPrefix: c.args.PrometheusHTTPPrefix,
+		HTTPClientConfig:     *httpClient,
 	}, c.metrics.mimirClientTiming)
 	if err != nil {
 		return err

component/mimir/rules/kubernetes/types.go

@@ -11,6 +11,7 @@ type Arguments struct {
 	Address              string                  `river:"address,attr"`
 	TenantID             string                  `river:"tenant_id,attr,optional"`
 	UseLegacyRoutes      bool                    `river:"use_legacy_routes,attr,optional"`
+	PrometheusHTTPPrefix string                  `river:"prometheus_http_prefix,attr,optional"`
 	HTTPClientConfig     config.HTTPClientConfig `river:",squash"`
 	SyncInterval         time.Duration           `river:"sync_interval,attr,optional"`
 	MimirNameSpacePrefix string                  `river:"mimir_namespace_prefix,attr,optional"`
@@ -23,6 +24,7 @@ var DefaultArguments = Arguments{
 	SyncInterval:         30 * time.Second,
 	MimirNameSpacePrefix: "agent",
 	HTTPClientConfig:     config.DefaultHTTPClientConfig,
+	PrometheusHTTPPrefix: "/prometheus",
 }
 
 // SetToDefault implements river.Defaulter.

component/module/git/git.go

@@ -12,7 +12,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/module"
-	"github.com/grafana/agent/component/module/git/internal/vcs"
+	"github.com/grafana/agent/internal/vcs"
 	"github.com/grafana/agent/pkg/flow/logging/level"
 )
 

component/otelcol/config_debug_metrics.go

@@ -7,7 +7,7 @@ type DebugMetricsArguments struct {
 
 // DefaultDebugMetricsArguments holds default settings for DebugMetricsArguments.
 var DefaultDebugMetricsArguments = DebugMetricsArguments{
-	DisableHighCardinalityMetrics: false,
+	DisableHighCardinalityMetrics: true,
 }
 
 // SetToDefault implements river.Defaulter.

component/otelcol/config_k8s.go

@@ -0,0 +1,35 @@
+package otelcol
+
+import "fmt"
+
+const (
+	KubernetesAPIConfig_AuthType_None           = "none"
+	KubernetesAPIConfig_AuthType_ServiceAccount = "serviceAccount"
+	KubernetesAPIConfig_AuthType_KubeConfig     = "kubeConfig"
+	KubernetesAPIConfig_AuthType_TLS            = "tls"
+)
+
+// KubernetesAPIConfig contains options relevant to connecting to the K8s API
+type KubernetesAPIConfig struct {
+	// How to authenticate to the K8s API server.  This can be one of `none`
+	// (for no auth), `serviceAccount` (to use the standard service account
+	// token provided to the agent pod), or `kubeConfig` to use credentials
+	// from `~/.kube/config`.
+	AuthType string `river:"auth_type,attr,optional"`
+
+	// When using auth_type `kubeConfig`, override the current context.
+	Context string `river:"context,attr,optional"`
+}
+
+// Validate returns an error if the config is invalid.
+func (c *KubernetesAPIConfig) Validate() error {
+	switch c.AuthType {
+	case KubernetesAPIConfig_AuthType_None,
+		KubernetesAPIConfig_AuthType_ServiceAccount,
+		KubernetesAPIConfig_AuthType_KubeConfig,
+		KubernetesAPIConfig_AuthType_TLS:
+		return nil
+	default:
+		return fmt.Errorf("invalid auth_type %q", c.AuthType)
+	}
+}

component/otelcol/exporter/loadbalancing/loadbalancing.go

@@ -59,7 +59,8 @@ var (
 		Protocol: Protocol{
 			OTLP: DefaultOTLPConfig,
 		},
-		RoutingKey: "traceID",
+		RoutingKey:   "traceID",
+		DebugMetrics: otelcol.DefaultDebugMetricsArguments,
 	}
 
 	DefaultOTLPConfig = OtlpConfig{