Merge branch 'main' into hackathon-xray-web-stream

.drone/drone.yml

@@ -119,24 +119,6 @@ trigger:
 type: docker
 ---
 kind: pipeline
-name: Test manifests
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - make generate-manifests
-  - ERR_MSG="The environment manifests are out of date. Please run 'make generate-manifests'
-    and commit changes!"
-  - if [ ! -z "$(git status --porcelain)" ]; then echo $ERR_MSG >&2; exit 1; fi
-  image: grafana/agent-build-image:0.30.4
-  name: Regenerate environment manifests
-trigger:
-  event:
-  - pull_request
-type: docker
----
-kind: pipeline
 name: Test
 platform:
   arch: amd64
@@ -1055,94 +1037,6 @@ volumes:
   name: docker
 ---
 kind: pipeline
-name: Publish Linux smoke container
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-  failure: ignore
-  image: grafana/agent-build-image:0.30.4
-  name: Configure QEMU
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-- commands:
-  - mkdir -p $HOME/.docker
-  - printenv GCR_CREDS > $HOME/.docker/config.json
-  - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
-  - docker buildx create --name multiarch-agent-smoke-${DRONE_COMMIT_SHA} --driver
-    docker-container --use
-  - ./tools/ci/docker-containers smoke
-  - docker buildx rm multiarch-agent-smoke-${DRONE_COMMIT_SHA}
-  environment:
-    DOCKER_LOGIN:
-      from_secret: docker_login
-    DOCKER_PASSWORD:
-      from_secret: docker_password
-    GCR_CREDS:
-      from_secret: gcr_admin
-  image: grafana/agent-build-image:0.30.4
-  name: Publish container
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-trigger:
-  ref:
-  - refs/heads/main
-  - refs/tags/v*
-type: docker
-volumes:
-- host:
-    path: /var/run/docker.sock
-  name: docker
----
-kind: pipeline
-name: Publish Linux crow container
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-  failure: ignore
-  image: grafana/agent-build-image:0.30.4
-  name: Configure QEMU
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-- commands:
-  - mkdir -p $HOME/.docker
-  - printenv GCR_CREDS > $HOME/.docker/config.json
-  - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
-  - docker buildx create --name multiarch-agent-crow-${DRONE_COMMIT_SHA} --driver
-    docker-container --use
-  - ./tools/ci/docker-containers crow
-  - docker buildx rm multiarch-agent-crow-${DRONE_COMMIT_SHA}
-  environment:
-    DOCKER_LOGIN:
-      from_secret: docker_login
-    DOCKER_PASSWORD:
-      from_secret: docker_password
-    GCR_CREDS:
-      from_secret: gcr_admin
-  image: grafana/agent-build-image:0.30.4
-  name: Publish container
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-trigger:
-  ref:
-  - refs/heads/main
-  - refs/tags/v*
-type: docker
-volumes:
-- host:
-    path: /var/run/docker.sock
-  name: docker
----
-kind: pipeline
 name: Publish Windows agent container
 platform:
   arch: amd64
@@ -1205,8 +1099,6 @@ depends_on:
 - Publish Linux agent-boringcrypto container
 - Publish Linux agentctl container
 - Publish Linux agent-operator container
-- Publish Linux smoke container
-- Publish Linux crow container
 image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
@@ -1266,8 +1158,6 @@ depends_on:
 - Publish Linux agent-boringcrypto container
 - Publish Linux agentctl container
 - Publish Linux agent-operator container
-- Publish Linux smoke container
-- Publish Linux crow container
 - Publish Windows agent container
 - Publish Windows agentctl container
 image_pull_secrets:
@@ -1427,6 +1317,6 @@ kind: secret
 name: updater_private_key
 ---
 kind: signature
-hmac: 28ba52df6f22c10bf77a95386a49aff65a1c372127f7d89489ac2d3ee02ce618
+hmac: b4b3bb3578124bba1758f323695216281365054c623738d38e51cc37125277ae
 
 ...

.drone/pipelines/publish.jsonnet

@@ -6,7 +6,7 @@ local ghTokenFilename = '/drone/src/gh-token.txt';
 // job_names gets the list of job names for use in depends_on.
 local job_names = function(jobs) std.map(function(job) job.name, jobs);
 
-local linux_containers = ['agent','agent-boringcrypto', 'agentctl', 'agent-operator', 'smoke', 'crow'];
+local linux_containers = ['agent', 'agent-boringcrypto', 'agentctl', 'agent-operator'];
 local linux_containers_jobs = std.map(function(container) (
   pipelines.linux('Publish Linux %s container' % container) {
     trigger: {
@@ -163,8 +163,8 @@ linux_containers_jobs + windows_containers_jobs + [
           GITHUB_APP_PRIVATE_KEY: secrets.updater_private_key.fromSecret,
         },
         commands: [
-          '/usr/bin/github-app-external-token > %s' % ghTokenFilename
-        ]
+          '/usr/bin/github-app-external-token > %s' % ghTokenFilename,
+        ],
       },
       {
         name: 'Publish release',
@@ -188,7 +188,7 @@ linux_containers_jobs + windows_containers_jobs + [
             VERSION=${DRONE_TAG} RELEASE_DOC_TAG=$(echo ${DRONE_TAG} | awk -F '.' '{print $1"."$2}') ./tools/release
           |||,
         ],
-      }
+      },
     ],
     volumes: [{
       name: 'docker',

.drone/pipelines/test.jsonnet

@@ -50,23 +50,6 @@ local pipelines = import '../util/pipelines.jsonnet';
     }],
   },
 
-  pipelines.linux('Test manifests') {
-    trigger: {
-      event: ['pull_request'],
-    },
-    steps: [{
-      name: 'Regenerate environment manifests',
-      image: build_image.linux,
-
-      commands: [
-        'make generate-manifests',
-        'ERR_MSG="The environment manifests are out of date. Please run \'make generate-manifests\' and commit changes!"',
-        // "git status --porcelain" reports if there's any new, modified, or deleted files.
-        'if [ ! -z "$(git status --porcelain)" ]; then echo $ERR_MSG >&2; exit 1; fi',
-      ],
-    }],
-  },
-
   pipelines.linux('Test') {
     trigger: {
       event: ['pull_request'],

.github/workflows/helm-test.yml

@@ -51,7 +51,7 @@ jobs:
           version: v3.10.3
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           check-latest: true

.github/workflows/integration-tests.yml

@@ -14,7 +14,7 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v4
     - name: Setup Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: "1.21"
     - name: Set OTEL Exporter Endpoint

.github/workflows/needs-attention.yml

@@ -10,7 +10,7 @@ jobs:
   needs-attention:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           days-before-stale: 30
           days-before-close: -1 # never close automatically

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v4
     - name: Set up Go 1.21
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: "1.21"
         cache: true

.github/workflows/trivy.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@2b6a709cf9c4025c5438138008beaddbb02086f0
+        uses: aquasecurity/trivy-action@d43c1f16c00cfd3978dde6c07f4bbcf9eb6993ca
         with:
           image-ref: 'grafana/agent:main'
           format: 'template'
@@ -35,6 +35,6 @@ jobs:
           severity: 'CRITICAL,HIGH,MEDIUM,LOW'
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results.sarif'

CHANGELOG.md

@@ -10,12 +10,50 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
+### Security fixes
+
+- Fixes following vulnerabilities (@hainenber)
+  - [GO-2023-2409](https://github.com/advisories/GHSA-mhpq-9638-x6pw)
+  - [GO-2023-2412](https://github.com/advisories/GHSA-7ww5-4wqc-m92c)
+  - [CVE-2023-49568](https://github.com/advisories/GHSA-mw99-9chc-xw7r)
+
+### Enhancements
+
+- Add an option to the windows static mode installer for expanding environment vars in the yaml config. (@erikbaranowski)
+- Add authentication support to `loki.source.awsfirehose` (@sberz)
+
+- Sort kubelet endpoint to reduce pressure on K8s's API server and watcher endpoints. (@hainenber)
+
+- Expose `physical_disk` collector from `windows_exporter` v0.24.0 to 
+  Flow configuration. (@hainenber)
+
+### Bugfixes
+
+- Fix an issue in `remote.s3` where the exported content of an object would be an empty string if `remote.s3` failed to fully retrieve
+  the file in a single read call. (@grafana/agent-squad)
+
+- Utilize the `instance` Argument of `prometheus.exporter.kafka` when set. (@akhmatov-s)
+
+- Fix a duplicate metrics registration panic when sending metrics to an static
+  mode metric instance's write handler. (@tpaschalis)
+
+### Other changes
+
+- Removed support for Windows 2012 in line with Microsoft end of life. (@mattdurham)
+
+- Split instance ID and component groupings into separate panels for `remote write active series by component` in the Flow mixin. (@tristanburgess)
+
+- Updated dependency to add support for Go 1.22 (@stefanb)
+
+v0.39.0 (2024-01-09)
+--------------------
+
 ### Breaking changes
 
 - `otelcol.receiver.prometheus` will drop all `otel_scope_info` metrics when converting them to OTLP. (@wildum)
   - If the `otel_scope_info` metric has labels `otel_scope_name` and `otel_scope_version`,
-    their values will be used to set OTLP Instrumentation Scope name and  version respectively. 
-  - Labels of `otel_scope_info` metrics other than `otel_scope_name` and `otel_scope_version` 
+    their values will be used to set OTLP Instrumentation Scope name and  version respectively.
+  - Labels of `otel_scope_info` metrics other than `otel_scope_name` and `otel_scope_version`
     are added as scope attributes with the matching name and version.
 
 - The `target` block in `prometheus.exporter.blackbox` requires a mandatory `name`
@@ -25,6 +63,12 @@ Main (unreleased)
   - This change will not break any existing configurations and you can opt in to validation via the `validate_dimensions` configuration option.
   - Before this change, pulling metrics for azure resources with variable dimensions required one configuration per metric + dimension combination to avoid an error.
   - After this change, you can include all metrics and dimensions in a single configuration and the Azure APIs will only return dimensions which are valid for the various metrics.
+
+### Features
+
+- A new `discovery.ovhcloud` component for discovering scrape targets on OVHcloud. (@ptodev)
+
+- Allow specifying additional containers to run. (@juangom)
 
 ### Enhancements
 
@@ -53,11 +97,12 @@ Main (unreleased)
 - `otelcol.receiver.prometheus` does not drop histograms without buckets anymore. (@wildum)
 
 - Added exemplars support to `otelcol.receiver.prometheus`. (@wildum)
+
 - `mimir.rules.kubernetes` may now retry its startup on failure. (@hainenber)
 
 - Added links between compatible components in the documentation to make it
   easier to discover them. (@thampiotr)
-  
+
 - Allow defining `HTTPClientConfig` for `discovery.ec2`. (@cmbrad)
 
 - The `remote.http` component can optionally define a request body. (@tpaschalis)
@@ -71,21 +116,29 @@ Main (unreleased)
 - Added 'country' mmdb-type to log pipeline-stage geoip. (@superstes)
 
 - Azure exporter enhancements for flow and static mode, (@kgeckhart)
-  - Allows for pulling metrics at the Azure subscription level instead of resource by resource 
-  - Disable dimension validation by default to reduce the number of exporter instances needed for full dimension coverage 
+  - Allows for pulling metrics at the Azure subscription level instead of resource by resource
+  - Disable dimension validation by default to reduce the number of exporter instances needed for full dimension coverage
 
 - Add `max_cache_size` to `prometheus.relabel` to allow configurability instead of hard coded 100,000. (@mattdurham)
 
 - Add support for `http_sd_config` within a `scrape_config` for prometheus to flow config conversion. (@erikbaranowski)
 
+- `discovery.lightsail` now supports additional parameters for configuring HTTP client settings. (@ptodev)
+- Add `sample_age_limit` to remote_write config to drop samples older than a specified duration. (@marctc)
+
+- Handle paths in the Kubelet URL for `discovery.kubelet`. (@petewall)
+
+- `loki.source.docker` now deduplicates targets which report the same container
+  ID. (@tpaschalis)
+
 ### Bugfixes
 
 - Update `pyroscope.ebpf` to fix a logical bug causing to profile to many kthreads instead of regular processes https://github.com/grafana/pyroscope/pull/2778 (@korniltsev)
- 
+
 - Update `pyroscope.ebpf` to produce more optimal pprof profiles for python processes https://github.com/grafana/pyroscope/pull/2788 (@korniltsev)
 
 - In Static mode's `traces` subsystem, `spanmetrics` used to be generated prior to load balancing.
-  This could lead to inaccurate metrics. This issue only affects Agents using both `spanmetrics` and 
+  This could lead to inaccurate metrics. This issue only affects Agents using both `spanmetrics` and
   `load_balancing`, when running in a load balanced cluster with more than one Agent instance. (@ptodev)
 
 - Fixes `loki.source.docker` a behavior that synced an incomplete list of targets to the tailer manager. (@FerdinandvHagen)
@@ -94,11 +147,21 @@ Main (unreleased)
 
 - Add staleness tracking to labelstore to reduce memory usage. (@mattdurham)
 
+- Fix issue where `prometheus.exporter.kafka` would crash when configuring `sasl_password`. (@rfratto)
+
+- Fix performance issue where perf lib where clause was not being set, leading to timeouts in collecting metrics for windows_exporter. (@mattdurham)
+
+- Fix nil panic when using the process collector with the windows exporter. (@mattdurham)
+
 ### Other changes
 
 - Bump github.com/IBM/sarama from v1.41.2 to v1.42.1
 
-- Attatch unique Agent ID header to remote-write requests. (@captncraig)
+- Attach unique Agent ID header to remote-write requests. (@captncraig)
+
+- Update to v2.48.1 of `github.com/prometheus/prometheus`.
+  Previously, a custom fork of v2.47.2 was used. 
+  The custom fork of v2.47.2 also contained prometheus#12729 and prometheus#12677.
 
 v0.38.1 (2023-11-30)
 --------------------