Skip to content

Commit

Permalink
Add buildkite pipeline to build elastic-agent binary artifacts (#3046)
Browse files Browse the repository at this point in the history
* Add DRA pipeline

* Fix mage and go check

* Force trigger

* Change permissions

* source bootstrap

* ignore existing folder

* Generate dependencies report

* Split in various steps and move creds to pre-command

* Fix script name

* Use agent to upload artifacts

* Expose env var to force publish for testing

* Generalize dra-publish.sh

* Use version for dependencies file

* Fix dry run

* Run test dra publish

* Fix version

* Only upload distributions

* Fix glob

* Only add spanshot on build

* Fix permissions on downloaded artifacts

* Remove test env vars

* Stop calling buildkite-agent in scripts

* Add vars to test

* Add correct command to staging

* creat local var

* Create local var

* Remove testing force var

* Remove test vars

(cherry picked from commit 30dff79)

# Conflicts:
#	.buildkite/pipeline.elastic-agent-binary-dra.yml
#	.buildkite/pipeline.yml
#	catalog-info.yaml
  • Loading branch information
marc-gr authored and mergify[bot] committed Jul 26, 2023
1 parent 9f42f86 commit 662201a
Show file tree
Hide file tree
Showing 11 changed files with 283 additions and 53 deletions.
60 changes: 49 additions & 11 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,65 @@

set -euo pipefail

function retry {
local retries=$1
shift

local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ $count -lt "$retries" ]; then
>&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
sleep $wait
else
>&2 echo "Retry $count/$retries exited $exit, no more retries left."
return $exit
fi
done
return 0
}

DOCKER_REGISTRY="docker.elastic.co"
DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
CI_DRA_ROLE_PATH=kv/ci-shared/release/dra-role
CI_GCP_OBS_PATH=kv/ci-shared/observability-ingest/cloud/gcp
CI_AGENT_QA_OBS_PATH=kv/ci-shared/observability-ingest/elastic-agent-ess-qa

if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "package_elastic-agent" ]]; then
export DOCKER_USERNAME_SECRET=$(vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
export DOCKER_PASSWORD_SECRET=$(vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
fi
fi
# Secrets must be redacted
# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables

if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then
# Set GCP credentials
export GOOGLE_APPLICATION_GCP_SECRET=$(vault kv get -format=json -field=data kv/ci-shared/observability-ingest/cloud/gcp)
export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH})
echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json
export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json)
export TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE=$(realpath ./gcp.json)

# ESS credentials
export API_KEY_TOKEN=$(vault kv get -field api_key kv/ci-shared/observability-ingest/elastic-agent-ess-qa)
export API_KEY_TOKEN=$(vault kv get -field api_key ${CI_AGENT_QA_OBS_PATH})
echo ${API_KEY_TOKEN} > ./apiKey
export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
fi
fi

if [[ ("$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" && "$BUILDKITE_STEP_KEY" == "package_elastic-agent") || "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then
if command -v docker &>/dev/null; then
export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
else
echo "+++ docker not found"
fi
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" && ("$BUILDKITE_STEP_KEY" == "publish-dra-snapshot" || "$BUILDKITE_STEP_KEY" == "publish-dra-staging") ]]; then
echo "+++ Setting DRA params"
# Shared secret path containing the dra creds for project teams
DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
fi
9 changes: 5 additions & 4 deletions .buildkite/hooks/pre-exit
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,18 @@ set -eo pipefail
if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
if test -f "$GOOGLE_APPLICATION_CREDENTIALS"; then
rm $GOOGLE_APPLICATION_CREDENTIALS
fi
fi
fi

if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then
if test -f "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE"; then
rm $TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE
fi
fi
fi

unset GOOGLE_APPLICATION_GCP_SECRET API_KEY_TOKEN

if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then
if command -v docker &>/dev/null; then
DOCKER_REGISTRY="docker.elastic.co"
docker logout $DOCKER_REGISTRY
fi
fi
64 changes: 64 additions & 0 deletions .buildkite/pipeline.elastic-agent-binary-dra.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
env:
DRA_PROJECT: "elastic-agent-core"
DRA_ARTIFACT_SET: "agent-core"
steps:
- group: ":beats: DRA Elastic-Agent Core Snapshot :beats:"
key: "dra-core-snapshot"
if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
steps:
- label: ":package: Build Elastic-Agent Core Snapshot"
commands:
- .buildkite/scripts/steps/build-agent-core.sh
key: "build-dra-snapshot"
artifact_paths:
- "build/distributions/**/*"
agents:
provider: "gcp"
machineType: "c2-standard-16"
env:
WORKFLOW: "snapshot"

- wait

- label: ":hammer: DRA Publish Elastic-Agent Core Snapshot"
command: |
buildkite-agent artifact download "build/**/*" .
.buildkite/scripts/steps/dra-publish.sh
key: "publish-dra-snapshot"
agents:
provider: "gcp"
machineType: "c2-standard-16"
env:
WORKFLOW: "snapshot"

- group: ":beats: DRA Elastic-Agent Core Staging :beats:"
key: "dra-core-staging"
if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
steps:
- label: ":package: Build Elastic-Agent Core staging"
commands:
- .buildkite/scripts/steps/build-agent-core.sh
key: "build-dra-staging"
artifact_paths:
- "build/distributions/**/*"
agents:
provider: "gcp"
machineType: "c2-standard-16"
env:
WORKFLOW: "staging"

- wait

- label: ":hammer: DRA Publish Elastic-Agent Core staging"
command: |
buildkite-agent artifact download "build/**/*" .
.buildkite/scripts/steps/dra-publish.sh
key: "publish-dra-staging"
agents:
provider: "gcp"
machineType: "c2-standard-16"
env:
WORKFLOW: "staging"

notify:
- slack: "#ingest-notifications"
4 changes: 4 additions & 0 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,12 @@ env:
steps:
- label: "Integration tests"
key: "integration-tests"
<<<<<<< HEAD
command: ".buildkite/scripts/integration_tests.sh"
branches: "main"
=======
command: ".buildkite/scripts/steps/integration_tests.sh"
>>>>>>> 30dff79483 (Add buildkite pipeline to build elastic-agent binary artifacts (#3046))
artifact_paths:
- "build/TEST-**"
agents:
Expand Down
30 changes: 30 additions & 0 deletions .buildkite/scripts/bootstrap.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
set -euxo pipefail

# this is required in order to allow the build process to override the default PWD of the BEAT_NAME.
export BEAT_NAME="elastic-agent"

if [[ -z "${WORKSPACE-""}" ]]; then
WORKSPACE=$(git rev-parse --show-toplevel)
export WORKSPACE
fi

# Retrieve version value - will match versions like 8.8.0 and also 8.8.0-er1
export BEAT_VERSION=`grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' ${WORKSPACE}/version/version.go`
export BRANCH="${BUILDKITE_BRANCH}"

# Install Go TODO: move to makefile
if ! command -v go &>/dev/null; then
echo "Go is not installed. Installing Go..."
export GO_VERSION=`cat .go-version`
curl -O https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz
sudo tar -xf go$GO_VERSION.linux-amd64.tar.gz -C /usr/local
mkdir -p $HOME/go/bin
export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin
echo "Go has been installed."
else
echo "Go is already installed."
fi

# Install mage
make mage
36 changes: 0 additions & 36 deletions .buildkite/scripts/integration_tests.sh

This file was deleted.

21 changes: 21 additions & 0 deletions .buildkite/scripts/steps/build-agent-core.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/bootstrap.sh

echo "+++ Build Agent artifacts"
SNAPSHOT=""
BEAT_VERSION_FULL=$BEAT_VERSION
if [ "$WORKFLOW" == "snapshot" ]; then
SNAPSHOT="true"
BEAT_VERSION_FULL="${BEAT_VERSION}-SNAPSHOT"
fi

SNAPSHOT=$SNAPSHOT mage packageAgentCore
chmod -R 777 build/distributions

echo "+++ Generate dependencies report"
./dev-tools/dependencies-report
mkdir -p build/distributions/reports
mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv"
39 changes: 39 additions & 0 deletions .buildkite/scripts/steps/dra-publish.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/bash

set -uo pipefail

source .buildkite/scripts/bootstrap.sh

# Publish DRA artifacts
function run_release_manager() {
echo "+++ Publishing $BUILDKITE_BRANCH ${WORKFLOW} DRA artifacts..."
dry_run=""
if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
dry_run="--dry-run"
# force main branch on PR's or it won't execute
# because the PR branch does not have a project folder in release-manager
BRANCH=main
fi
docker run --rm \
--name release-manager \
-e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
-e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
-e VAULT_SECRET_ID="${VAULT_SECRET}" \
--mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
docker.elastic.co/infra/release-manager:latest \
cli collect \
--project $DRA_PROJECT \
--branch "${BRANCH}" \
--commit "${BUILDKITE_COMMIT}" \
--workflow "${WORKFLOW}" \
--version "${BEAT_VERSION}" \
--artifact-set $DRA_ARTIFACT_SET \
$dry_run
}

chmod -R 777 build/distributions

run_release_manager
RM_EXIT_CODE=$?

exit $RM_EXIT_CODE
19 changes: 19 additions & 0 deletions .buildkite/scripts/steps/integration_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -euxo pipefail

source .buildkite/scripts/bootstrap.sh

# PACKAGE
DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package

# Run integration tests
set +e
SNAPSHOT=true mage integration:test
TESTS_EXIT_STATUS=$?
set -e

# HTML report
go install github.com/alexec/junit2html@latest
junit2html < build/TEST-go-integration.xml > build/TEST-report.html

exit $TESTS_EXIT_STATUS
3 changes: 1 addition & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ COVERAGE_DIR=$(BUILD_DIR)/coverage
BEATS?=elastic-agent
PROJECTS= $(BEATS)
PYTHON_ENV?=$(BUILD_DIR)/python-env
MAGE_VERSION ?= v1.13.0
MAGE_VERSION ?= v1.14.0
MAGE_PRESENT := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
MAGE_IMPORT_PATH ?= github.com/magefile/mage
export MAGE_IMPORT_PATH
Expand All @@ -18,7 +18,6 @@ ifndef MAGE_PRESENT
endif
@true


## help : Show this help.
help: Makefile
@printf "Usage: make [target] [VARIABLE=value]\nTargets:\n"
Expand Down
Loading

0 comments on commit 662201a

Please sign in to comment.