Skip to content

Commit

Permalink
[CI] Pull requests migration to buildkite (#3573) (#3720)
Browse files Browse the repository at this point in the history
* buildkite test on Ubuntu

* Fixed step ids

* fix image family

* Use aws from arm64

* try windows post-checkout hook

* Windows pre-command hook and unit tests ps

* Windows pre-command hook and unit tests ps

* Windows pre-command hook and unit tests ps

* Windows pre-command hook and unit tests ps

* Windows pre-command hook and unit tests ps

* Windows pre-command hook and unit tests ps

* Windows pre-command hook and unit tests ps

* Windows pre-command hook and unit tests ps

* set GOPATH

* set GOPATH

* set GOPATH

* set GOPATH

* set GOPATH

* Installed GCC

* Installed GCC

* Installed GCC

* Windows 2016

* Propagate exit code on windows

* Added GOTMPDIR

* Added GOTMPDIR

* Merge coverage reports step

* Merge coverage reports step

* Merge coverage reports step

* changed installation path

* changed installation path

* changed installation path

* changed installation path

* changed installation path

* changed installation path

* test

* test

* merge.sh

* Added coverage.out to simplify artifacts download

* Fixed artifacts copy

* buildite orka integration

* fix os name

* Add err message to upgrade test

* Fix autocrlf for windows

* Craig's patch

* Complete test pipeline

* Complete test pipeline

* Remove macos GHA workflow

* Fix Processing test results step

* fix k8s jenkins step

* Removed m1 stage

* Enabled race detector for unit tests

* Enables race detector for windows

(cherry picked from commit bf78cc1)

Co-authored-by: Pavel Zorin <[email protected]>
  • Loading branch information
mergify[bot] and pazone authored Nov 7, 2023
1 parent 6008507 commit 0cb9f6b
Show file tree
Hide file tree
Showing 10 changed files with 256 additions and 154 deletions.
60 changes: 60 additions & 0 deletions .buildkite/hooks/post-checkout.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Set error handling
$ErrorActionPreference = "Stop"

# Define a function to checkout and merge
function Checkout-Merge {
param (
[string]$targetBranch,
[string]$prCommit,
[string]$mergeBranch
)

if (-not $targetBranch) {
Write-Host "No pull request target branch"
exit 1
}

git fetch -v origin $targetBranch
git checkout FETCH_HEAD
Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)"

# Create a temporary branch to merge the PR with the target branch
git checkout -b $mergeBranch
Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)"

# Set author identity so it can be used for git merge
git config user.name "github-merged-pr-post-checkout"
git config user.email "auto-merge@buildkite"

git merge --no-edit $prCommit

if ($LASTEXITCODE -ne 0) {
$mergeResult = $LASTEXITCODE
Write-Host "Merge failed: $mergeResult"
git merge --abort
exit $mergeResult
}
}

$pullRequest = $env:BUILDKITE_PULL_REQUEST

if ($pullRequest -eq "false") {
Write-Host "Not a pull request, skipping"
exit 0
}

$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
$prCommit = $env:BUILDKITE_COMMIT
$prId = $env:BUILDKITE_PULL_REQUEST
$mergeBranch = "pr_merge_$prId"

Checkout-Merge $targetBranch $prCommit $mergeBranch

Write-Host "Commit information"
git --no-pager log --format=%B -n 1

Write-Host "Fixing CRLF in git checkout --"
git config core.autocrlf true

# Ensure Buildkite groups are rendered
Write-Host ""
25 changes: 25 additions & 0 deletions .buildkite/hooks/pre-command.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Install gcc TODO: Move to the VM image
choco install mingw
Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
refreshenv

# Install gvm and go
# TODO: Move GVM download to the base VM image
$env:GvmVersion = "0.5.2"
[Net.ServicePointManager]::SecurityProtocol = "tls12"
$env:GoVersion = Get-Content -Path .go-version
Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
gvm --format=powershell $env:GoVersion | Invoke-Expression
go version

$GOPATH = $(go env GOPATH)
$env:Path = "$GOPATH\bin;" + $env:Path
[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)

# Install tools
go install github.com/magefile/mage
go install github.com/elastic/go-licenser
go install golang.org/x/tools/cmd/goimports
go install github.com/jstemmer/go-junit-report
go install gotest.tools/gotestsum
117 changes: 111 additions & 6 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,134 @@ env:
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
DOCKER_REGISTRY: "docker.elastic.co"
steps:
- label: "Unit tests"
key: "unit-tests"
- label: "Unit tests - Ubuntu 22.04"
key: "unit-tests-2204"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-ubuntu-2204"
retry:
manual:
allowed: true

- label: "Unit tests - Ubuntu 22.04 ARM64"
key: "unit-tests-2204-arm64"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "aws"
imagePrefix: "core-ubuntu-2204-aarch64"
diskSizeGb: 200
instanceType: "m6g.4xlarge"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2022"
key: "unit-tests-win2022"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2022"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2016"
key: "unit-tests-win2016"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2016"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13 ARM"
key: "unit-tests-macos-13-arm"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-arm
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13"
key: "unit-tests-macos-13"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-x64
retry:
manual:
allowed: true

- label: "Merge coverage reports"
key: "merge-coverage"
env:
BUILDKITE_REPO: ""
command: "
.buildkite/scripts/steps/merge.sh
unit-tests-2204
unit-tests-2204-arm64
unit-tests-win2016
unit-tests-win2022
unit-tests-macos-13
unit-tests-macos-13-arm
"
artifact_paths:
- "build/TEST-**"
agents:
image: "golang:1.20.10"
depends_on:
- unit-tests-2204
- unit-tests-2204-arm64
- unit-tests-win2022
- unit-tests-win2016
- unit-tests-macos-13
- unit-tests-macos-13-arm
allow_dependency_failure: true

- label: ":sonarqube: Continuous Code Inspection"
env:
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
agents:
image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
command:
- "buildkite-agent artifact download build/TEST-go-unit.cov ."
- "buildkite-agent artifact download build/TEST-go-unit.out.json ."
- "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
- "/scan-source-code.sh"
depends_on:
- "unit-tests"
- "merge-coverage"
retry:
manual:
allowed: true
Expand All @@ -48,4 +153,4 @@ steps:
provider: "gcp"
plugins:
- junit-annotate#v2.4.1:
artifacts: build/TEST-go-*.xml
artifacts: build/TEST-go-integration*.xml
2 changes: 1 addition & 1 deletion .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ getOSOptions() {
export AGENT_OS_NAME=linux
;;
darwin*)
export AGENT_OS_NAME=osx
export AGENT_OS_NAME=darwin
;;
msys*)
export AGENT_OS_NAME=windows
Expand Down
23 changes: 23 additions & 0 deletions .buildkite/scripts/steps/merge.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash

# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov).
# Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#

set -exuo pipefail

COV_ARTIFACT="coverage.out"
MERGED_COV_FILE="build/TEST-go-unit.cov"
# Space separated list of paths to coverage files
COV_PATHS=""

go install github.com/wadey/gocovmerge@latest
mkdir -p build

for STEP_ID in "$@"; do
mkdir -p $STEP_ID
buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID
COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT"
done

gocovmerge $COV_PATHS > $MERGED_COV_FILE
echo "Merged coverage file: $MERGED_COV_FILE. See artifacts"
28 changes: 28 additions & 0 deletions .buildkite/scripts/steps/unit-tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
$ErrorActionPreference = "Stop"

Write-Host "-- Fixing CRLF in git checkout --"
git config core.autocrlf input
git rm --quiet --cached -r .
git reset --quiet --hard

$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"

Write-Host "--- Build"
mage build

if ($LASTEXITCODE -ne 0) {
exit 1
}

Write-Host "--- Unit tests"
$env:TEST_COVERAGE = $true
$env:RACE_DETECTOR = $true
mage unitTest
# Copy coverage file to build directory so it can be downloaded as an artifact
cp .\build\TEST-go-unit.cov coverage.out

if ($LASTEXITCODE -ne 0) {
exit 1
}


4 changes: 3 additions & 1 deletion .buildkite/scripts/steps/unit-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,6 @@ set -euxo pipefail
source .buildkite/scripts/common.sh

echo "--- Unit tests"
TEST_COVERAGE=true mage unitTest
RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
# Copy coverage file to build directory so it can be downloaded as an artifact
cp build/TEST-go-unit.cov coverage.out
Loading

0 comments on commit 0cb9f6b

Please sign in to comment.