diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0212a51cdaa..84fca8c0fd1 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,27 +2,65 @@ set -euo pipefail +function retry { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +DOCKER_REGISTRY="docker.elastic.co" DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" +CI_DRA_ROLE_PATH=kv/ci-shared/release/dra-role +CI_GCP_OBS_PATH=kv/ci-shared/observability-ingest/cloud/gcp +CI_AGENT_QA_OBS_PATH=kv/ci-shared/observability-ingest/elastic-agent-ess-qa -if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then - if [[ "$BUILDKITE_STEP_KEY" == "package_elastic-agent" ]]; then - export DOCKER_USERNAME_SECRET=$(vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") - export DOCKER_PASSWORD_SECRET=$(vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") - docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null - unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET - fi -fi +# Secrets must be redacted +# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then # Set GCP credentials - export GOOGLE_APPLICATION_GCP_SECRET=$(vault kv get -format=json -field=data kv/ci-shared/observability-ingest/cloud/gcp) + export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) export TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE=$(realpath ./gcp.json) # ESS credentials - export API_KEY_TOKEN=$(vault kv get -field api_key kv/ci-shared/observability-ingest/elastic-agent-ess-qa) + export API_KEY_TOKEN=$(vault kv get -field api_key ${CI_AGENT_QA_OBS_PATH}) echo ${API_KEY_TOKEN} > ./apiKey export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey) - fi \ No newline at end of file +fi + +if [[ ("$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" && "$BUILDKITE_STEP_KEY" == "package_elastic-agent") || "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then + if command -v docker &>/dev/null; then + export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") + export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null + unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET + else + echo "+++ docker not found" + fi +fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" && ("$BUILDKITE_STEP_KEY" == "publish-dra-snapshot" || "$BUILDKITE_STEP_KEY" == "publish-dra-staging") ]]; then + echo "+++ Setting DRA params" + # Shared secret path containing the dra creds for project teams + DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index af9062f4072..ab80d4c76be 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -3,17 +3,18 @@ set -eo pipefail if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then if test -f "$GOOGLE_APPLICATION_CREDENTIALS"; then rm $GOOGLE_APPLICATION_CREDENTIALS - fi + fi fi if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then if test -f "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE"; then rm $TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE - fi + fi fi unset GOOGLE_APPLICATION_GCP_SECRET API_KEY_TOKEN -if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then +if command -v docker &>/dev/null; then + DOCKER_REGISTRY="docker.elastic.co" docker logout $DOCKER_REGISTRY -fi \ No newline at end of file +fi diff --git a/.buildkite/pipeline.elastic-agent-binary-dra.yml b/.buildkite/pipeline.elastic-agent-binary-dra.yml new file mode 100644 index 00000000000..dbdcd4f1c8c --- /dev/null +++ b/.buildkite/pipeline.elastic-agent-binary-dra.yml @@ -0,0 +1,64 @@ +env: + DRA_PROJECT: "elastic-agent-core" + DRA_ARTIFACT_SET: "agent-core" +steps: + - group: ":beats: DRA Elastic-Agent Core Snapshot :beats:" + key: "dra-core-snapshot" + if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true" + steps: + - label: ":package: Build Elastic-Agent Core Snapshot" + commands: + - .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-snapshot" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + WORKFLOW: "snapshot" + + - wait + + - label: ":hammer: DRA Publish Elastic-Agent Core Snapshot" + command: | + buildkite-agent artifact download "build/**/*" . + .buildkite/scripts/steps/dra-publish.sh + key: "publish-dra-snapshot" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + WORKFLOW: "snapshot" + + - group: ":beats: DRA Elastic-Agent Core Staging :beats:" + key: "dra-core-staging" + if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true" + steps: + - label: ":package: Build Elastic-Agent Core staging" + commands: + - .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-staging" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + WORKFLOW: "staging" + + - wait + + - label: ":hammer: DRA Publish Elastic-Agent Core staging" + command: | + buildkite-agent artifact download "build/**/*" . + .buildkite/scripts/steps/dra-publish.sh + key: "publish-dra-staging" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + WORKFLOW: "staging" + +notify: + - slack: "#ingest-notifications" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 73a63548eeb..c6bb78d335c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,8 +6,12 @@ env: steps: - label: "Integration tests" key: "integration-tests" +<<<<<<< HEAD command: ".buildkite/scripts/integration_tests.sh" branches: "main" +======= + command: ".buildkite/scripts/steps/integration_tests.sh" +>>>>>>> 30dff79483 (Add buildkite pipeline to build elastic-agent binary artifacts (#3046)) artifact_paths: - "build/TEST-**" agents: diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh new file mode 100755 index 00000000000..9b9519feae6 --- /dev/null +++ b/.buildkite/scripts/bootstrap.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# this is required in order to allow the build process to override the default PWD of the BEAT_NAME. +export BEAT_NAME="elastic-agent" + +if [[ -z "${WORKSPACE-""}" ]]; then + WORKSPACE=$(git rev-parse --show-toplevel) + export WORKSPACE +fi + +# Retrieve version value - will match versions like 8.8.0 and also 8.8.0-er1 +export BEAT_VERSION=`grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' ${WORKSPACE}/version/version.go` +export BRANCH="${BUILDKITE_BRANCH}" + +# Install Go TODO: move to makefile +if ! command -v go &>/dev/null; then + echo "Go is not installed. Installing Go..." + export GO_VERSION=`cat .go-version` + curl -O https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz + sudo tar -xf go$GO_VERSION.linux-amd64.tar.gz -C /usr/local + mkdir -p $HOME/go/bin + export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin + echo "Go has been installed." +else + echo "Go is already installed." +fi + +# Install mage +make mage diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh deleted file mode 100755 index 2accd2bb7ed..00000000000 --- a/.buildkite/scripts/integration_tests.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -# Install Go TODO: mode to makefile -if ! command -v go &>/dev/null; then - echo "Go is not installed. Installing Go..." - export GO_VERSION=`cat .go-version` - curl -O https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz - sudo tar -xf go$GO_VERSION.linux-amd64.tar.gz -C /usr/local - echo 'export PATH=$PATH:/usr/local/go/bin' >> ~/.bashrc - source ~/.bashrc - mkdir $HOME/go - mkdir $HOME/go/bin - export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin - echo "Go has been installed." -else - echo "Go is already installed." -fi - -# Install mage -make mage - -# PACKAGE -DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package - -# Run integration tests -set +e -TEST_INTEG_TIMESTAMP=true SNAPSHOT=true mage integration:test -TESTS_EXIT_STATUS=$? -set -e - -# HTML report -go install github.com/alexec/junit2html@latest -junit2html < build/TEST-go-integration.xml > build/TEST-report.html - -exit $TESTS_EXIT_STATUS diff --git a/.buildkite/scripts/steps/build-agent-core.sh b/.buildkite/scripts/steps/build-agent-core.sh new file mode 100755 index 00000000000..40d400f06cc --- /dev/null +++ b/.buildkite/scripts/steps/build-agent-core.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/bootstrap.sh + +echo "+++ Build Agent artifacts" +SNAPSHOT="" +BEAT_VERSION_FULL=$BEAT_VERSION +if [ "$WORKFLOW" == "snapshot" ]; then + SNAPSHOT="true" + BEAT_VERSION_FULL="${BEAT_VERSION}-SNAPSHOT" +fi + +SNAPSHOT=$SNAPSHOT mage packageAgentCore +chmod -R 777 build/distributions + +echo "+++ Generate dependencies report" +./dev-tools/dependencies-report +mkdir -p build/distributions/reports +mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv" diff --git a/.buildkite/scripts/steps/dra-publish.sh b/.buildkite/scripts/steps/dra-publish.sh new file mode 100755 index 00000000000..142b4b146b0 --- /dev/null +++ b/.buildkite/scripts/steps/dra-publish.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -uo pipefail + +source .buildkite/scripts/bootstrap.sh + +# Publish DRA artifacts +function run_release_manager() { + echo "+++ Publishing $BUILDKITE_BRANCH ${WORKFLOW} DRA artifacts..." + dry_run="" + if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then + dry_run="--dry-run" + # force main branch on PR's or it won't execute + # because the PR branch does not have a project folder in release-manager + BRANCH=main + fi + docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project $DRA_PROJECT \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set $DRA_ARTIFACT_SET \ + $dry_run +} + +chmod -R 777 build/distributions + +run_release_manager +RM_EXIT_CODE=$? + +exit $RM_EXIT_CODE diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh new file mode 100755 index 00000000000..7f67252d518 --- /dev/null +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euxo pipefail + +source .buildkite/scripts/bootstrap.sh + +# PACKAGE +DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package + +# Run integration tests +set +e +SNAPSHOT=true mage integration:test +TESTS_EXIT_STATUS=$? +set -e + +# HTML report +go install github.com/alexec/junit2html@latest +junit2html < build/TEST-go-integration.xml > build/TEST-report.html + +exit $TESTS_EXIT_STATUS diff --git a/Makefile b/Makefile index 030ff00a8cd..bc04d6c4383 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ COVERAGE_DIR=$(BUILD_DIR)/coverage BEATS?=elastic-agent PROJECTS= $(BEATS) PYTHON_ENV?=$(BUILD_DIR)/python-env -MAGE_VERSION ?= v1.13.0 +MAGE_VERSION ?= v1.14.0 MAGE_PRESENT := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION)) MAGE_IMPORT_PATH ?= github.com/magefile/mage export MAGE_IMPORT_PATH @@ -18,7 +18,6 @@ ifndef MAGE_PRESENT endif @true - ## help : Show this help. help: Makefile @printf "Usage: make [target] [VARIABLE=value]\nTargets:\n" diff --git a/catalog-info.yaml b/catalog-info.yaml index 78544e96f3a..126a02b229e 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -104,3 +104,54 @@ spec: access_level: MANAGE_BUILD_AND_READ everyone: access_level: READ_ONLY +<<<<<<< HEAD +======= + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-elastic-agent-binary-dra + description: Buildkite pipeline for packaging Elastic Agent core binary and publish it to DRA + links: + - title: Pipeline + url: https://buildkite.com/elastic/elastic-agent + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: elastic-agent-binary-dra + description: Buildkite pipeline for packaging Elastic Agent core binary and publish it to DRA + spec: + pipeline_file: ".buildkite/pipeline.elastic-agent-binary-dra.yml" + provider_settings: + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + publish_commit_status: false # do not update status of commits for this pipeline + build_tags: false + build_branches: false + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/elastic-agent + schedules: + Daily 8_9: + branch: '8.9' + cronline: '@daily' + message: Builds daily `8.9` dra + Daily main: + branch: main + cronline: '@daily' + message: Builds daily `main` dra + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ +>>>>>>> 30dff79483 (Add buildkite pipeline to build elastic-agent binary artifacts (#3046))