Add buildkite pipeline to build elastic-agent binary artifacts (#3046)

* Add DRA pipeline

* Fix mage and go check

* Force trigger

* Change permissions

* source bootstrap

* ignore existing folder

* Generate dependencies report

* Split in various steps and move creds to pre-command

* Fix script name

* Use agent to upload artifacts

* Expose env var to force publish for testing

* Generalize dra-publish.sh

* Use version for dependencies file

* Fix dry run

* Run test dra publish

* Fix version

* Only upload distributions

* Fix glob

* Only add spanshot on build

* Fix permissions on downloaded artifacts

* Remove test env vars

* Stop calling buildkite-agent in scripts

* Add vars to test

* Add correct command to staging

* creat local var

* Create local var

* Remove testing force var

* Remove test vars

(cherry picked from commit 30dff79)

# Conflicts:
#	.buildkite/pipeline.elastic-agent-binary-dra.yml
#	.buildkite/pipeline.yml
#	catalog-info.yaml

.buildkite/hooks/pre-command

@@ -2,27 +2,65 @@
 
 set -euo pipefail
 
+function retry {
+    local retries=$1
+    shift
 
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
+
+DOCKER_REGISTRY="docker.elastic.co"
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+CI_DRA_ROLE_PATH=kv/ci-shared/release/dra-role
+CI_GCP_OBS_PATH=kv/ci-shared/observability-ingest/cloud/gcp
+CI_AGENT_QA_OBS_PATH=kv/ci-shared/observability-ingest/elastic-agent-ess-qa
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "package_elastic-agent" ]]; then
-    export DOCKER_USERNAME_SECRET=$(vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
-    unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
-  fi
-fi
+# Secrets must be redacted
+# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then
   # Set GCP credentials
-  export GOOGLE_APPLICATION_GCP_SECRET=$(vault kv get -format=json -field=data kv/ci-shared/observability-ingest/cloud/gcp)
+  export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH})
   echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json
   export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json)
   export TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE=$(realpath ./gcp.json)
 
   # ESS credentials
-  export API_KEY_TOKEN=$(vault kv get -field api_key kv/ci-shared/observability-ingest/elastic-agent-ess-qa)
+  export API_KEY_TOKEN=$(vault kv get -field api_key ${CI_AGENT_QA_OBS_PATH})
   echo ${API_KEY_TOKEN} > ./apiKey
   export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
- fi 
+fi
+
+if [[ ("$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" && "$BUILDKITE_STEP_KEY" == "package_elastic-agent") || "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then
+  if command -v docker &>/dev/null; then
+    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+    unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
+  else
+    echo "+++ docker not found"
+  fi
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" && ("$BUILDKITE_STEP_KEY" == "publish-dra-snapshot" || "$BUILDKITE_STEP_KEY" == "publish-dra-staging") ]]; then
+  echo "+++ Setting DRA params"
+  # Shared secret path containing the dra creds for project teams
+  DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
+fi

.buildkite/hooks/pre-exit

@@ -3,17 +3,18 @@ set -eo pipefail
 if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
   if test -f "$GOOGLE_APPLICATION_CREDENTIALS"; then
     rm $GOOGLE_APPLICATION_CREDENTIALS
-  fi  
+  fi
 fi
 
 if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then
   if test -f "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE"; then
     rm $TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE
-  fi  
+  fi
 fi
 
 unset GOOGLE_APPLICATION_GCP_SECRET API_KEY_TOKEN
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then
+if command -v docker &>/dev/null; then
+  DOCKER_REGISTRY="docker.elastic.co"
   docker logout $DOCKER_REGISTRY
-fi
+fi

.buildkite/pipeline.elastic-agent-binary-dra.yml

@@ -0,0 +1,64 @@
+env:
+  DRA_PROJECT: "elastic-agent-core"
+  DRA_ARTIFACT_SET: "agent-core"
+steps:
+  - group: ":beats: DRA Elastic-Agent Core Snapshot :beats:"
+    key: "dra-core-snapshot"
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
+    steps:
+    - label: ":package: Build Elastic-Agent Core Snapshot"
+      commands:
+        - .buildkite/scripts/steps/build-agent-core.sh
+      key: "build-dra-snapshot"
+      artifact_paths:
+        - "build/distributions/**/*"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        WORKFLOW: "snapshot"
+
+    - wait
+
+    - label: ":hammer: DRA Publish Elastic-Agent Core Snapshot"
+      command: |
+        buildkite-agent artifact download "build/**/*" .
+        .buildkite/scripts/steps/dra-publish.sh
+      key: "publish-dra-snapshot"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        WORKFLOW: "snapshot"
+
+  - group: ":beats: DRA Elastic-Agent Core Staging :beats:"
+    key: "dra-core-staging"
+    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+    steps:
+    - label: ":package: Build Elastic-Agent Core staging"
+      commands:
+        - .buildkite/scripts/steps/build-agent-core.sh
+      key: "build-dra-staging"
+      artifact_paths:
+        - "build/distributions/**/*"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        WORKFLOW: "staging"
+
+    - wait
+
+    - label: ":hammer: DRA Publish Elastic-Agent Core staging"
+      command: |
+        buildkite-agent artifact download "build/**/*" .
+        .buildkite/scripts/steps/dra-publish.sh
+      key: "publish-dra-staging"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        WORKFLOW: "staging"
+
+notify:
+  - slack: "#ingest-notifications"

.buildkite/pipeline.yml

@@ -6,8 +6,12 @@ env:
 steps:
   - label: "Integration tests"
     key: "integration-tests"
+<<<<<<< HEAD
     command: ".buildkite/scripts/integration_tests.sh"
     branches: "main"
+=======
+    command: ".buildkite/scripts/steps/integration_tests.sh"
+>>>>>>> 30dff79483 (Add buildkite pipeline to build elastic-agent binary artifacts (#3046))
     artifact_paths:
       - "build/TEST-**"
     agents:

.buildkite/scripts/bootstrap.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# this is required in order to allow the build process to override the default PWD of the BEAT_NAME.
+export BEAT_NAME="elastic-agent"
+
+if [[ -z "${WORKSPACE-""}" ]]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+    export WORKSPACE
+fi
+
+# Retrieve version value - will match versions like 8.8.0 and also 8.8.0-er1
+export BEAT_VERSION=`grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' ${WORKSPACE}/version/version.go`
+export BRANCH="${BUILDKITE_BRANCH}"
+
+# Install Go TODO: move to makefile
+if ! command -v go &>/dev/null; then
+  echo "Go is not installed. Installing Go..."
+  export GO_VERSION=`cat .go-version`
+  curl -O https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz
+  sudo tar -xf go$GO_VERSION.linux-amd64.tar.gz -C /usr/local
+  mkdir -p $HOME/go/bin
+  export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin
+  echo "Go has been installed."
+else
+  echo "Go is already installed."
+fi
+
+# Install mage
+make mage

.buildkite/scripts/steps/build-agent-core.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/bootstrap.sh
+
+echo "+++ Build Agent artifacts"
+SNAPSHOT=""
+BEAT_VERSION_FULL=$BEAT_VERSION
+if [ "$WORKFLOW" == "snapshot" ]; then
+    SNAPSHOT="true"
+    BEAT_VERSION_FULL="${BEAT_VERSION}-SNAPSHOT"
+fi
+
+SNAPSHOT=$SNAPSHOT mage packageAgentCore
+chmod -R 777 build/distributions
+
+echo  "+++ Generate dependencies report"
+./dev-tools/dependencies-report
+mkdir -p build/distributions/reports
+mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv"

.buildkite/scripts/steps/dra-publish.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -uo pipefail
+
+source .buildkite/scripts/bootstrap.sh
+
+# Publish DRA artifacts
+function run_release_manager() {
+    echo "+++ Publishing $BUILDKITE_BRANCH ${WORKFLOW} DRA artifacts..."
+    dry_run=""
+    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
+        dry_run="--dry-run"
+        # force main branch on PR's or it won't execute
+        # because the PR branch does not have a project folder in release-manager
+        BRANCH=main
+    fi
+    docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli collect \
+        --project $DRA_PROJECT \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set $DRA_ARTIFACT_SET \
+        $dry_run
+}
+
+chmod -R 777 build/distributions
+
+run_release_manager
+RM_EXIT_CODE=$?
+
+exit $RM_EXIT_CODE

.buildkite/scripts/steps/integration_tests.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source .buildkite/scripts/bootstrap.sh
+
+# PACKAGE
+DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package
+
+# Run integration tests
+set +e
+SNAPSHOT=true mage integration:test
+TESTS_EXIT_STATUS=$?
+set -e
+
+# HTML report
+go install github.com/alexec/junit2html@latest
+junit2html < build/TEST-go-integration.xml > build/TEST-report.html
+
+exit $TESTS_EXIT_STATUS

Makefile

@@ -3,7 +3,7 @@ COVERAGE_DIR=$(BUILD_DIR)/coverage
 BEATS?=elastic-agent
 PROJECTS= $(BEATS)
 PYTHON_ENV?=$(BUILD_DIR)/python-env
-MAGE_VERSION     ?= v1.13.0
+MAGE_VERSION     ?= v1.14.0
 MAGE_PRESENT     := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
 MAGE_IMPORT_PATH ?= github.com/magefile/mage
 export MAGE_IMPORT_PATH
@@ -18,7 +18,6 @@ ifndef MAGE_PRESENT
 endif
 	@true
 
-
 ## help : Show this help.
 help: Makefile
 	@printf "Usage: make [target] [VARIABLE=value]\nTargets:\n"