Initial config for artifacts pipeline

.buildkite/artifacts.yml

@@ -0,0 +1,8 @@
+steps:
+  - command: .buildkite/scripts/steps/artifacts/build.sh
+    label: Build Artifacts
+
+  - wait
+
+  - command: .buildkite/scripts/steps/artifacts/publish.sh
+    label: 'Publish Artifacts'

.buildkite/hooks/pre-command

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+export DOCKER_USERNAME_SECRET=$(vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+export DOCKER_PASSWORD_SECRET=$(vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET

.buildkite/hooks/pre-exit

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+docker logout ${DOCKER_REGISTRY}

.buildkite/scripts/bootstrap.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+buildkite-agent annotate "Package version: \\${BEAT_VERSION}" --style 'info'

.buildkite/scripts/common/download_artifact.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "$0")/util.sh"
+download_artifact "$@"

.buildkite/scripts/common/env.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+export FORCE_COLOR=1
+export DOCKER_REGISTRY="docker.elastic.co"
+export SETUP_GVM_VERSION='v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+export SETUP_MAGE_VERSION='1.14.0'
+if [[ -z "${WORKSPACE-""}" ]]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+fi
+export WORKSPACE
+if [[ -z "${GO_VERSION-""}" ]]; then
+    GO_VERSION=$(cat "${WORKSPACE}/.go-version")
+fi
+export GO_VERSION
+export BEAT_VERSION=$(grep -oe "\d.\d.\d[-\w\d]*" ${WORKSPACE}/version/version.go)
+export ELASTIC_AGENT_BASE_BRANCH=main

.buildkite/scripts/common/util.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -exuo pipefail
+
+source "$(dirname "$0")/env.sh"
+
+# Wrapper function for executing mage
+mage() {
+    go version
+    if ! [ -x "$(type -p mage | sed 's/mage is //g')" ];
+    then
+        echo "installing mage ${SETUP_MAGE_VERSION}"
+        make mage
+    fi
+    pushd "$WORKSPACE"
+    command "mage" "$@"
+    popd
+}
+
+# Wrapper function for executing go
+go(){
+    # Search for the go in the Path
+    if ! [ -x "$(type -p go | sed 's/go is //g')" ];
+    then
+        local _bin="${WORKSPACE}/bin"
+        mkdir -p "${_bin}"
+        retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
+        chmod +x "${_bin}/gvm"
+        eval "$(command "${_bin}/gvm" "${GO_VERSION}" )"
+        export GOPATH=$(command go env GOPATH)
+        export PATH="${PATH}:${GOPATH}/bin"
+    fi
+    pushd "$WORKSPACE"
+    command go "$@"
+    popd
+}
+
+google_cloud_auth() {
+    local keyFile=$1
+
+    gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null
+
+    export GOOGLE_APPLICATIONS_CREDENTIALS=${secretFileLocation}
+}
+
+retry() {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
+
+# Download an artifact using the buildkite-agent, takes the same arguments as https://buildkite.com/docs/agent/v3/cli-artifact#downloading-artifacts-usage
+# times-out after 60 seconds and retries up to 3 times
+download_artifact() {
+  retry 3 1 timeout 3m buildkite-agent artifact download "$@"
+}

.buildkite/scripts/steps/artifacts/build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "--- Build Agent artifacts"
+mage packageAgentBinary
+
+echo "--- Upload Agent Artifacts"
+cd build/distributions
+# buildkite-agent artifact upload 'elastic-agent-*'
+cd -

.buildkite/scripts/steps/artifacts/env.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RELEASE_BUILD="${RELEASE_BUILD:="false"}"
+
+if [[ "$RELEASE_BUILD" == "true" ]]; then
+  WORKFLOW="staging"
+else
+  WORKFLOW="snapshot"
+fi
+
+ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+ARTIFACTS_MANIFEST_FQDN="https://$ARTIFACTS_SUBDOMAIN.elastic.co"
+ELASTIC_AGENT_MANIFEST_LATEST="$ARTIFACTS_MANIFEST_FQDN/elastic-agent/latest/$FULL_VERSION.json"
+
+export WORKFLOW
+export ELASTIC_AGENT_MANIFEST_LATEST

.buildkite/scripts/steps/artifacts/publish.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "--- Download and verify artifacts"
+function download {
+  download_artifact "$1" . --build "${ELASTIC_AGENT_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact "$1.sha512.txt" . --build "${ELASTIC_AGENT_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  sha512sum -c "$1.sha512.txt"
+  rm "$1.sha512.txt"
+}
+
+mkdir -p build/distributions
+cd build/distributions
+
+download "elastic-agent-$BEAT_VERSION-darwin-aarch64.tar.gz"
+download "elastic-agent-$BEAT_VERSION-darwin-x86_64.tar.gz"
+download "elastic-agent-$BEAT_VERSION-linux-arm64.tar.gz"
+download "elastic-agent-$BEAT_VERSION-linux-x86_64.tar.gz"
+download "elastic-agent-$BEAT_VERSION-windows-x86_64.zip"
+
+cd -
+
+echo "--- Set artifact permissions"
+chmod -R a+r build/distributions/*
+chmod -R a+w build/distributions
+
+echo "--- Pull latest Release Manager CLI"
+docker pull docker.elastic.co/infra/release-manager:latest
+
+echo "--- Publish artifacts"
+if [[ "$BUILDKITE_BRANCH" == "$ELASTIC_AGENT_BASE_BRANCH" ]]; then
+  export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=elastic-agent-buildkite-vault-role-id)"
+  export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=elastic-agent-buildkite-vault-secret-id)"
+  export VAULT_ADDR="https://secrets.elastic.co:8200"
+
+  # todo: change to collect command when ready
+  docker run --rm \
+    --name release-manager \
+    -e VAULT_ADDR \
+    -e VAULT_ROLE_ID \
+    -e VAULT_SECRET_ID \
+    --mount type=bind,readonly=false,src="$PWD/build/distributions",target=/build/distributions \
+    docker.elastic.co/infra/release-manager:latest \
+      cli list \
+        --project elastic-agent \
+        --branch "$ELASTIC_AGENT_BASE_BRANCH" \
+        --commit "$GIT_COMMIT" \
+        --workflow "$WORKFLOW" \
+        --version "$BASE_VERSION" \
+        --artifact-set main
+
+  ELASTIC_AGENT_SUMMARY=$(curl -s "$ELASTIC_AGENT_MANIFEST_LATEST")
+
+  cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
+
+  ### Artifacts Summary
+
+  $ELASTIC_AGENT_SUMMARY
+EOF
+
+else
+  echo "Skipping publish for untracked branch $BUILDKITE_BRANCH"
+fi