[8.9](backport #3573) [CI] Pull requests migration to buildkite

.buildkite/hooks/post-checkout.ps1

@@ -0,0 +1,60 @@
+# Set error handling
+$ErrorActionPreference = "Stop"
+
+# Define a function to checkout and merge
+function Checkout-Merge {
+    param (
+        [string]$targetBranch,
+        [string]$prCommit,
+        [string]$mergeBranch
+    )
+
+    if (-not $targetBranch) {
+        Write-Host "No pull request target branch"
+        exit 1
+    }
+
+    git fetch -v origin $targetBranch
+    git checkout FETCH_HEAD
+    Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Create a temporary branch to merge the PR with the target branch
+    git checkout -b $mergeBranch
+    Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Set author identity so it can be used for git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit $prCommit
+
+    if ($LASTEXITCODE -ne 0) {
+        $mergeResult = $LASTEXITCODE
+        Write-Host "Merge failed: $mergeResult"
+        git merge --abort
+        exit $mergeResult
+    }
+}
+
+$pullRequest = $env:BUILDKITE_PULL_REQUEST
+
+if ($pullRequest -eq "false") {
+    Write-Host "Not a pull request, skipping"
+    exit 0
+}
+
+$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
+$prCommit = $env:BUILDKITE_COMMIT
+$prId = $env:BUILDKITE_PULL_REQUEST
+$mergeBranch = "pr_merge_$prId"
+
+Checkout-Merge $targetBranch $prCommit $mergeBranch
+
+Write-Host "Commit information"
+git --no-pager log --format=%B -n 1
+
+Write-Host "Fixing CRLF in git checkout --"
+git config core.autocrlf true
+
+# Ensure Buildkite groups are rendered
+Write-Host ""

.buildkite/hooks/pre-command.ps1

@@ -0,0 +1,25 @@
+# Install gcc TODO: Move to the VM image
+choco install mingw
+Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
+refreshenv
+
+# Install gvm and go
+# TODO: Move GVM download to the base VM image
+$env:GvmVersion = "0.5.2"
+[Net.ServicePointManager]::SecurityProtocol = "tls12"
+$env:GoVersion = Get-Content -Path .go-version
+Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
+gvm --format=powershell $env:GoVersion | Invoke-Expression
+go version
+
+$GOPATH = $(go env GOPATH)
+$env:Path = "$GOPATH\bin;" + $env:Path
+[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
+[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
+
+# Install tools
+go install github.com/magefile/mage
+go install github.com/elastic/go-licenser
+go install golang.org/x/tools/cmd/goimports
+go install github.com/jstemmer/go-junit-report
+go install gotest.tools/gotestsum

.buildkite/pipeline.yml

@@ -4,6 +4,148 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
+  - label: "Unit tests - Ubuntu 22.04"
+    key: "unit-tests-2204"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Ubuntu 22.04 ARM64"
+    key: "unit-tests-2204-arm64"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "aws"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 200
+      instanceType: "m6g.4xlarge"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Windows 2022"
+    key: "unit-tests-win2022"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2022"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Windows 2016"
+    key: "unit-tests-win2016"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2016"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - MacOS 13 ARM"
+    key: "unit-tests-macos-13-arm"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: orka
+      imagePrefix: generic-13-ventura-arm
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - MacOS 13"
+    key: "unit-tests-macos-13"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: orka
+      imagePrefix: generic-13-ventura-x64
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Merge coverage reports"
+    key: "merge-coverage"
+    env:
+      BUILDKITE_REPO: ""
+    command: "
+      .buildkite/scripts/steps/merge.sh
+      unit-tests-2204
+      unit-tests-2204-arm64
+      unit-tests-win2016
+      unit-tests-win2022
+      unit-tests-macos-13
+      unit-tests-macos-13-arm
+      "
+    artifact_paths:
+      - "build/TEST-**"
+    agents:
+      image: "golang:1.20.10"
+    depends_on:
+      - unit-tests-2204
+      - unit-tests-2204-arm64
+      - unit-tests-win2022
+      - unit-tests-win2016
+      - unit-tests-macos-13
+      - unit-tests-macos-13-arm
+    allow_dependency_failure: true
+
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command:
+      - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
+      - "/scan-source-code.sh"
+    depends_on:
+      - "merge-coverage"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Serverless integration test"
+    key: "serverless-integration-tests"
+    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
   - label: "Integration tests"
     key: "integration-tests"
     command: ".buildkite/scripts/integration_tests.sh"
@@ -13,3 +155,12 @@ steps:
     agents:
       provider: "gcp"
       machineType: "n1-standard-8"
+
+  - wait: ~
+    continue_on_failure: true
+  - label: "Processing test results"
+    agents:
+      provider: "gcp"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: build/TEST-go-integration*.xml

.buildkite/scripts/common.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ -z "${WORKSPACE-""}" ]]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+    export WORKSPACE
+fi
+PIPELINE="${WORKSPACE}/.buildkite/pipeline.elastic-agent-package.yml"
+if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then
+    SETUP_MAGE_VERSION=$(grep -oe "SETUP_MAGE_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" )
+fi
+if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then
+    SETUP_GVM_VERSION=$(grep -oe "SETUP_GVM_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" )
+fi
+BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go")
+export BEAT_VERSION
+
+getOSOptions() {
+  case $(uname | tr '[:upper:]' '[:lower:]') in
+    linux*)
+      export AGENT_OS_NAME=linux
+      ;;
+    darwin*)
+      export AGENT_OS_NAME=darwin
+      ;;
+    msys*)
+      export AGENT_OS_NAME=windows
+      ;;
+    *)
+      export AGENT_OS_NAME=notset
+      ;;
+  esac
+  case $(uname -m | tr '[:upper:]' '[:lower:]') in
+    aarch64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    arm64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    amd64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    x86_64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    *)
+      export AGENT_OS_ARCH=notset
+      ;;
+  esac
+}
+
+# Wrapper function for executing mage
+mage() {
+    go version
+    if ! [ -x "$(type -p mage | sed 's/mage is //g')" ];
+    then
+        echo "installing mage ${SETUP_MAGE_VERSION}"
+        make mage
+    fi
+    pushd "$WORKSPACE"
+    command "mage" "$@"
+    ACTUAL_EXIT_CODE=$?
+    popd
+    return $ACTUAL_EXIT_CODE
+}
+
+# Wrapper function for executing go
+go(){
+    # Search for the go in the Path
+    if ! [ -x "$(type -p go | sed 's/go is //g')" ];
+    then
+        getOSOptions
+        echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" "
+        local _bin="${WORKSPACE}/bin"
+        mkdir -p "${_bin}"
+        retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${AGENT_OS_NAME}-${AGENT_OS_ARCH}"
+        chmod +x "${_bin}/gvm"
+        eval "$(command "${_bin}/gvm" "${GO_VERSION}" )"
+        export GOPATH=$(command go env GOPATH)
+        export PATH="${PATH}:${GOPATH}/bin"
+    fi
+    pushd "$WORKSPACE"
+    command go "$@"
+    ACTUAL_EXIT_CODE=$?
+    popd
+    return $ACTUAL_EXIT_CODE
+}
+
+google_cloud_auth() {
+    local keyFile=$1
+
+    gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null
+
+    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}
+
+retry() {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}

.buildkite/scripts/steps/merge.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov).
+# Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#  
+
+set -exuo pipefail
+
+COV_ARTIFACT="coverage.out"
+MERGED_COV_FILE="build/TEST-go-unit.cov"
+# Space separated list of paths to coverage files
+COV_PATHS=""
+
+go install github.com/wadey/gocovmerge@latest
+mkdir -p build
+
+for STEP_ID in "$@"; do
+  mkdir -p $STEP_ID 
+  buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID
+  COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT"
+done
+
+gocovmerge $COV_PATHS > $MERGED_COV_FILE
+echo "Merged coverage file: $MERGED_COV_FILE. See artifacts"