Add buildkite pipeline to build elastic-agent binary artifacts

.buildkite/hooks/pre-command

@@ -2,16 +2,15 @@
 
 set -euo pipefail
 
-
-DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "package_elastic-agent" ]]; then
-    export DOCKER_USERNAME_SECRET=$(vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
-    unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
-  fi
+if command -v docker &>/dev/null; then
+  DOCKER_REGISTRY="docker.elastic.co"
+  DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+  export DOCKER_USERNAME_SECRET=$(vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+  export DOCKER_PASSWORD_SECRET=$(vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+  docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+  unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
+else
+  echo "+++ docker not found"
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then
@@ -25,4 +24,4 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" ==
   export API_KEY_TOKEN=$(vault kv get -field api_key kv/ci-shared/observability-ingest/elastic-agent-ess-qa)
   echo ${API_KEY_TOKEN} > ./apiKey
   export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
- fi 
+ fi

.buildkite/hooks/pre-exit

@@ -3,17 +3,18 @@ set -eo pipefail
 if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
   if test -f "$GOOGLE_APPLICATION_CREDENTIALS"; then
     rm $GOOGLE_APPLICATION_CREDENTIALS
-  fi  
+  fi
 fi
 
 if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then
   if test -f "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE"; then
     rm $TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE
-  fi  
+  fi
 fi
 
 unset GOOGLE_APPLICATION_GCP_SECRET API_KEY_TOKEN
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then
+if command -v docker &>/dev/null; then
+  DOCKER_REGISTRY="docker.elastic.co"
   docker logout $DOCKER_REGISTRY
-fi
+fi

.buildkite/pipeline.elastic-agent-binary-dra.yml

@@ -1,3 +1,34 @@
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - group: ":beats: DRA Elastic-Agent Core Snapshot :beats:"
+    key: "dra-core-snapshot"
+    steps:
+    - label: ":hammer::package: Build and DRA Publish Elastic-Agent Core Snapshot"
+      if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
+      commands:
+        - .buildkite/scripts/steps/dra-build.sh
+        - .buildkite/scripts/steps/dra-publish.sh
+      key: "build-and-publish-dra-snapshot"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        WORKFLOW: "snapshot"
+  - group: ":beats: DRA Elastic-Agent Core Staging :beats:"
+    key: "dra-core-staging"
+    steps:
+    - label: ":hammer: Build Elastic-Agent Core staging"
+      if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+      commands:
+        - .buildkite/scripts/steps/dra-build.sh
+        - .buildkite/scripts/steps/dra-publish.sh
+      key: "build-and-publish-dra-staging"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        WORKFLOW: "staging"
+
+notify:
+  - slack: "#elastic-agent"
+    # only notify on main and release branches
+    if: build.pull_request.id == 'null'

.buildkite/pipeline.yml

@@ -6,7 +6,7 @@ env:
 steps:
   - label: "Integration tests"
     key: "integration-tests"
-    command: ".buildkite/scripts/integration_tests.sh"
+    command: ".buildkite/scripts/steps/integration_tests.sh"
     artifact_paths:
       - "build/TEST-**"
     agents:

.buildkite/scripts/bootstrap.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# this is required in order to allow the build process to override the default PWD of the BEAT_NAME.
+export BEAT_NAME="elastic-agent"
+
+if [[ -z "${WORKSPACE-""}" ]]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+    export WORKSPACE
+fi
+
+if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then
+    SETUP_MAGE_VERSION="1.14.0"
+fi
+
+if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then
+    SETUP_GVM_VERSION="v0.5.0"
+fi
+
+if [[ -z "${GO_VERSION-""}" ]]; then
+    GO_VERSION=$(cat "${WORKSPACE}/.go-version")
+fi
+
+# Retrieve version value
+export BEAT_VERSION=$(grep -oe "\d.\d.\d[-\w\d]*" ${WORKSPACE}/version/version.go)
+export BRANCH="${BUILDKITE_BRANCH}"
+
+# Wrapper function for executing mage
+mage() {
+    go version
+    if ! [ -x "$(type -p mage | sed 's/mage is //g')" ];
+    then
+        echo "+++ Installing mage ${SETUP_MAGE_VERSION}"
+        make mage
+    fi
+    pushd "$WORKSPACE"
+    command "mage" "$@"
+    popd
+}
+
+# Wrapper function for executing go
+go(){
+    # Search for the go in the Path
+    if ! [ -x "$(type -p go | sed 's/go is //g')" ];
+    then
+        local _bin="${WORKSPACE}/bin"
+        mkdir -p "${_bin}"
+        retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
+        chmod +x "${_bin}/gvm"
+        eval "$(command "${_bin}/gvm" "${GO_VERSION}" )"
+        export GOPATH=$(command go env GOPATH)
+        export PATH="${PATH}:${GOPATH}/bin"
+    fi
+    pushd "$WORKSPACE"
+    command go "$@"
+    popd
+}

.buildkite/scripts/steps/dra-build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo "+++ Build Agent artifacts"
+mage packageAgentCore
+chmod -R 777 build/distributions

.buildkite/scripts/steps/dra-publish.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -uo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo "+++ Setting DRA params"
+# Shared secret path containing the dra creds for project teams
+DRA_CREDS=$(vault kv get -field=data -format=json kv/ci-shared/release/dra-role)
+VAULT_ADDR=$(echo $DRA_CREDS | jq -r '.vault_addr')
+VAULT_ROLE_ID=$(echo $DRA_CREDS | jq -r '.role_id')
+VAULT_SECRET_ID=$(echo $DRA_CREDS | jq -r '.secret_id')
+export VAULT_ADDR VAULT_ROLE_ID VAULT_SECRET_ID
+
+# Publish DRA artifacts
+function run_release_manager() {
+    echo "+++ Publishing $BUILDKITE_BRANCH ${WORKFLOW} DRA artifacts..."
+    dry_run=""
+    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
+        dry_run="--dry-run"
+    fi
+    docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET_ID}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli collect \
+        --project agent-core \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set main \
+        $dry_run \
+        #
+}
+
+run_release_manager
+RM_EXIT_CODE=$?
+
+exit $RM_EXIT_CODE

.buildkite/scripts/steps/integration_tests.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+# PACKAGE
+DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package
+
+# Run integration tests
+set +e
+SNAPSHOT=true mage integration:test
+TESTS_EXIT_STATUS=$?
+set -e
+
+# HTML report
+go install github.com/alexec/junit2html@latest
+junit2html < build/TEST-go-integration.xml > build/TEST-report.html
+
+exit $TESTS_EXIT_STATUS

catalog-info.yaml

@@ -138,6 +138,15 @@ spec:
         filter_condition: >-
           build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       repository: elastic/elastic-agent
+      # schedules:
+      #   Daily 8_8:
+      #     branch: '8.8'
+      #     cronline: '@daily'
+      #     message: Builds daily `8.8` dra
+      #   Daily main:
+      #     branch: main
+      #     cronline: '@daily'
+      #     message: Builds daily `main` dra
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ