From 425edc382885d991e3747d9569c7d28df5406480 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 8 Jun 2023 12:49:23 +0200 Subject: [PATCH] Initial config for artifacts pipeline --- .buildkite/artifacts.yml | 8 +++ .buildkite/hooks/pre-command | 9 +++ .buildkite/hooks/pre-exit | 5 ++ .buildkite/scripts/bootstrap.sh | 7 ++ .../scripts/common/download_artifact.sh | 6 ++ .buildkite/scripts/common/env.sh | 16 +++++ .buildkite/scripts/common/util.sh | 70 +++++++++++++++++++ .buildkite/scripts/steps/artifacts/build.sh | 15 ++++ .buildkite/scripts/steps/artifacts/env.sh | 18 +++++ .buildkite/scripts/steps/artifacts/publish.sh | 67 ++++++++++++++++++ 10 files changed, 221 insertions(+) create mode 100644 .buildkite/artifacts.yml create mode 100755 .buildkite/hooks/pre-command create mode 100755 .buildkite/hooks/pre-exit create mode 100755 .buildkite/scripts/bootstrap.sh create mode 100755 .buildkite/scripts/common/download_artifact.sh create mode 100755 .buildkite/scripts/common/env.sh create mode 100755 .buildkite/scripts/common/util.sh create mode 100644 .buildkite/scripts/steps/artifacts/build.sh create mode 100755 .buildkite/scripts/steps/artifacts/env.sh create mode 100644 .buildkite/scripts/steps/artifacts/publish.sh diff --git a/.buildkite/artifacts.yml b/.buildkite/artifacts.yml new file mode 100644 index 00000000000..26855c63165 --- /dev/null +++ b/.buildkite/artifacts.yml @@ -0,0 +1,8 @@ +steps: + - command: .buildkite/scripts/steps/artifacts/build.sh + label: Build Artifacts + + - wait + + - command: .buildkite/scripts/steps/artifacts/publish.sh + label: 'Publish Artifacts' diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 00000000000..88edbd0fd20 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" +export DOCKER_USERNAME_SECRET=$(vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") +export DOCKER_PASSWORD_SECRET=$(vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") +docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null +unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100755 index 00000000000..2b03f707ab6 --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +docker logout ${DOCKER_REGISTRY} diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh new file mode 100755 index 00000000000..52371e27c62 --- /dev/null +++ b/.buildkite/scripts/bootstrap.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +buildkite-agent annotate "Package version: \\${BEAT_VERSION}" --style 'info' diff --git a/.buildkite/scripts/common/download_artifact.sh b/.buildkite/scripts/common/download_artifact.sh new file mode 100755 index 00000000000..09f2d9e978a --- /dev/null +++ b/.buildkite/scripts/common/download_artifact.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "$(dirname "$0")/util.sh" +download_artifact "$@" diff --git a/.buildkite/scripts/common/env.sh b/.buildkite/scripts/common/env.sh new file mode 100755 index 00000000000..5141253efa1 --- /dev/null +++ b/.buildkite/scripts/common/env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +export FORCE_COLOR=1 +export DOCKER_REGISTRY="docker.elastic.co" +export SETUP_GVM_VERSION='v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 +export SETUP_MAGE_VERSION='1.14.0' +if [[ -z "${WORKSPACE-""}" ]]; then + WORKSPACE=$(git rev-parse --show-toplevel) +fi +export WORKSPACE +if [[ -z "${GO_VERSION-""}" ]]; then + GO_VERSION=$(cat "${WORKSPACE}/.go-version") +fi +export GO_VERSION +export BEAT_VERSION=$(grep -oe "\d.\d.\d[-\w\d]*" ${WORKSPACE}/version/version.go) +export ELASTIC_AGENT_BASE_BRANCH=main diff --git a/.buildkite/scripts/common/util.sh b/.buildkite/scripts/common/util.sh new file mode 100755 index 00000000000..61a20832f47 --- /dev/null +++ b/.buildkite/scripts/common/util.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +set -exuo pipefail + +source "$(dirname "$0")/env.sh" + +# Wrapper function for executing mage +mage() { + go version + if ! [ -x "$(type -p mage | sed 's/mage is //g')" ]; + then + echo "installing mage ${SETUP_MAGE_VERSION}" + make mage + fi + pushd "$WORKSPACE" + command "mage" "$@" + popd +} + +# Wrapper function for executing go +go(){ + # Search for the go in the Path + if ! [ -x "$(type -p go | sed 's/go is //g')" ]; + then + local _bin="${WORKSPACE}/bin" + mkdir -p "${_bin}" + retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64" + chmod +x "${_bin}/gvm" + eval "$(command "${_bin}/gvm" "${GO_VERSION}" )" + export GOPATH=$(command go env GOPATH) + export PATH="${PATH}:${GOPATH}/bin" + fi + pushd "$WORKSPACE" + command go "$@" + popd +} + +google_cloud_auth() { + local keyFile=$1 + + gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null + + export GOOGLE_APPLICATIONS_CREDENTIALS=${secretFileLocation} +} + +retry() { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +# Download an artifact using the buildkite-agent, takes the same arguments as https://buildkite.com/docs/agent/v3/cli-artifact#downloading-artifacts-usage +# times-out after 60 seconds and retries up to 3 times +download_artifact() { + retry 3 1 timeout 3m buildkite-agent artifact download "$@" +} diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh new file mode 100644 index 00000000000..54b053ab387 --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +.buildkite/scripts/bootstrap.sh + +source .buildkite/scripts/steps/artifacts/env.sh + +echo "--- Build Agent artifacts" +mage packageAgentBinary + +echo "--- Upload Agent Artifacts" +cd build/distributions +# buildkite-agent artifact upload 'elastic-agent-*' +cd - diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh new file mode 100755 index 00000000000..4f8a9038aca --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +RELEASE_BUILD="${RELEASE_BUILD:="false"}" + +if [[ "$RELEASE_BUILD" == "true" ]]; then + WORKFLOW="staging" +else + WORKFLOW="snapshot" +fi + +ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW" +ARTIFACTS_MANIFEST_FQDN="https://$ARTIFACTS_SUBDOMAIN.elastic.co" +ELASTIC_AGENT_MANIFEST_LATEST="$ARTIFACTS_MANIFEST_FQDN/elastic-agent/latest/$FULL_VERSION.json" + +export WORKFLOW +export ELASTIC_AGENT_MANIFEST_LATEST diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh new file mode 100644 index 00000000000..67524fc020f --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/artifacts/env.sh + +echo "--- Download and verify artifacts" +function download { + download_artifact "$1" . --build "${ELASTIC_AGENT_BUILD_ID:-$BUILDKITE_BUILD_ID}" + download_artifact "$1.sha512.txt" . --build "${ELASTIC_AGENT_BUILD_ID:-$BUILDKITE_BUILD_ID}" + sha512sum -c "$1.sha512.txt" + rm "$1.sha512.txt" +} + +mkdir -p build/distributions +cd build/distributions + +download "elastic-agent-$BEAT_VERSION-darwin-aarch64.tar.gz" +download "elastic-agent-$BEAT_VERSION-darwin-x86_64.tar.gz" +download "elastic-agent-$BEAT_VERSION-linux-arm64.tar.gz" +download "elastic-agent-$BEAT_VERSION-linux-x86_64.tar.gz" +download "elastic-agent-$BEAT_VERSION-windows-x86_64.zip" + +cd - + +echo "--- Set artifact permissions" +chmod -R a+r build/distributions/* +chmod -R a+w build/distributions + +echo "--- Pull latest Release Manager CLI" +docker pull docker.elastic.co/infra/release-manager:latest + +echo "--- Publish artifacts" +if [[ "$BUILDKITE_BRANCH" == "$ELASTIC_AGENT_BASE_BRANCH" ]]; then + export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=elastic-agent-buildkite-vault-role-id)" + export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=elastic-agent-buildkite-vault-secret-id)" + export VAULT_ADDR="https://secrets.elastic.co:8200" + + # todo: change to collect command when ready + docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/build/distributions",target=/build/distributions \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project elastic-agent \ + --branch "$ELASTIC_AGENT_BASE_BRANCH" \ + --commit "$GIT_COMMIT" \ + --workflow "$WORKFLOW" \ + --version "$BASE_VERSION" \ + --artifact-set main + + ELASTIC_AGENT_SUMMARY=$(curl -s "$ELASTIC_AGENT_MANIFEST_LATEST") + + cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary + + ### Artifacts Summary + + $ELASTIC_AGENT_SUMMARY +EOF + +else + echo "Skipping publish for untracked branch $BUILDKITE_BRANCH" +fi