From 2d4b6ff9b756b74cd0187281c646509cc0e3f81d Mon Sep 17 00:00:00 2001 From: Justin Alvarez Date: Thu, 17 Oct 2024 23:23:54 +0000 Subject: [PATCH] build: add linux release automation and release automation validation Signed-off-by: Justin Alvarez --- .github/workflows/build-and-test-msi.yaml | 33 +-- .github/workflows/build-and-test-pkg.yaml | 23 +- .github/workflows/ci-docs.yaml | 14 + .github/workflows/ci-release.yaml | 268 ++++++++++++++++++ .../get-version-and-tag-for-ref.yaml | 55 ++++ .github/workflows/release-linux.yaml | 118 ++++++++ Makefile | 10 +- 7 files changed, 480 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/ci-release.yaml create mode 100644 .github/workflows/get-version-and-tag-for-ref.yaml create mode 100644 .github/workflows/release-linux.yaml diff --git a/.github/workflows/build-and-test-msi.yaml b/.github/workflows/build-and-test-msi.yaml index 53fd2a97d..f9d828552 100644 --- a/.github/workflows/build-and-test-msi.yaml +++ b/.github/workflows/build-and-test-msi.yaml @@ -5,13 +5,19 @@ on: workflow_dispatch: inputs: ref_name: + description: "the ref (tag/branch) to use to extract tag/version" required: true type: string workflow_call: inputs: ref_name: + description: "the ref (tag/branch) to use to extract tag/version" required: true type: string + version: + description: "override for version, will be used instead of ref if set, used for testing" + required: false + type: string schedule: - cron: '0 9 * * *' env: @@ -26,29 +32,10 @@ permissions: jobs: get-tag-name: name: Get tag name - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.check-tag.outputs.tag }} - version: ${{ steps.check-tag.outputs.version }} - steps: - - name: Check tag from workflow input and github ref - id: check-tag - run: | - if [ -n "${{ inputs.ref_name }}" ]; then - tag=${{ inputs.ref_name }} - else - tag=${{ github.ref_name }} - fi - echo "tag=$tag" >> ${GITHUB_OUTPUT} - - version=${tag#v} - if [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Version matches format: $version" - else - echo "Version $version doesn't match format. Using test version: 0.0.1" - version="0.0.1" - fi - echo "version=$version" >> ${GITHUB_OUTPUT} + uses: ./.github/workflows/get-version-and-tag-for-ref.yaml + with: + ref_name: ${{ inputs.ref_name }} + version: ${{ inputs.version }} windows-msi-build: needs: get-tag-name diff --git a/.github/workflows/build-and-test-pkg.yaml b/.github/workflows/build-and-test-pkg.yaml index 258b76b4d..62a0a38e6 100644 --- a/.github/workflows/build-and-test-pkg.yaml +++ b/.github/workflows/build-and-test-pkg.yaml @@ -5,13 +5,19 @@ on: workflow_dispatch: inputs: ref_name: + description: "the ref (tag/branch) to use to extract tag/version" required: true type: string workflow_call: inputs: ref_name: + description: "the ref (tag/branch) to use to extract tag/version" required: true type: string + version: + description: "override for version, will be used instead of ref if set, used for testing" + required: false + type: string schedule: - cron: '0 9 * * *' env: @@ -20,19 +26,10 @@ env: jobs: get-tag-name: name: Get tag name - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.check-tag.outputs.tag }} - steps: - - name: Check tag from workflow input and github ref - id: check-tag - run: | - if [ -n "${{ inputs.ref_name }}" ]; then - tag=${{ inputs.ref_name }} - else - tag=${{ github.ref_name }} - fi - echo "tag=$tag" >> ${GITHUB_OUTPUT} + uses: ./.github/workflows/get-version-and-tag-for-ref.yaml + with: + ref_name: ${{ inputs.ref_name }} + version: ${{ inputs.version }} macos-aarch64-pkg-build: needs: get-tag-name diff --git a/.github/workflows/ci-docs.yaml b/.github/workflows/ci-docs.yaml index b5ba265dc..8a9f8fd59 100644 --- a/.github/workflows/ci-docs.yaml +++ b/.github/workflows/ci-docs.yaml @@ -24,6 +24,13 @@ on: - '!.github/workflows/e2e-macos.yaml' - '!.github/workflows/e2e-windows.yaml' - '!.github/workflows/e2e-linux.yaml' + - '!.github/workflows/release-automation.yaml' + - '!.github/workflows/release-linux.yaml' + - '!.github/workflows/upload-build-to-S3.yaml' + - '!.github/workflows/build-and-test-msi.yaml' + - '!.github/workflows/build-and-test-pkg.yaml' + - '!.github/workflows/ci-release.yaml' + - '!CHANGELOG.md' pull_request: branches: - main @@ -39,6 +46,13 @@ on: - '!.github/workflows/e2e-macos.yaml' - '!.github/workflows/e2e-windows.yaml' - '!.github/workflows/e2e-linux.yaml' + - '!.github/workflows/release-automation.yaml' + - '!.github/workflows/release-linux.yaml' + - '!.github/workflows/upload-build-to-S3.yaml' + - '!.github/workflows/build-and-test-msi.yaml' + - '!.github/workflows/build-and-test-pkg.yaml' + - '!.github/workflows/ci-release.yaml' + - '!CHANGELOG.md' jobs: git-secrets: diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml new file mode 100644 index 000000000..5553e7a38 --- /dev/null +++ b/.github/workflows/ci-release.yaml @@ -0,0 +1,268 @@ +name: CI +on: + push: + branches: + - main + paths: + - '.github/workflows/release-automation.yaml' + - '.github/workflows/release-linux.yaml' + - '.github/workflows/upload-build-to-S3.yaml' + - '.github/workflows/build-and-test-msi.yaml' + - '.github/workflows/build-and-test-pkg.yaml' + - 'deps/finch-core' + - 'CHANGELOG.md' + pull_request: + branches: + - main + paths: + - '.github/workflows/release-automation.yaml' + - '.github/workflows/release-linux.yaml' + - '.github/workflows/upload-build-to-S3.yaml' + - '.github/workflows/build-and-test-msi.yaml' + - '.github/workflows/build-and-test-pkg.yaml' + - 'deps/finch-core' + - 'CHANGELOG.md' + workflow_dispatch: +permissions: + id-token: write + contents: write + +env: + DEBUG: ${{ secrets.ACTIONS_STEP_DEBUG }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + git-secrets: + runs-on: ubuntu-latest + steps: + - name: Pull latest awslabs/git-secrets repo + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + repository: awslabs/git-secrets + ref: 1.3.0 + fetch-tags: true + path: git-secrets + - name: Install git secrets from source + run: sudo make install + working-directory: git-secrets + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Scan repository for git secrets + run: | + git secrets --register-aws + git secrets --scan-history + + gen-code-no-diff: + strategy: + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + cache: true + - run: make gen-code + - run: git diff --exit-code + unit-tests: + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Configure git CRLF settings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + # Since this repository is not meant to be used as a library, + # we don't need to test the latest 2 major releases like Go does: https://go.dev/doc/devel/release#policy. + go-version-file: go.mod + cache: true + - run: make test-unit + # It's recommended to run golangci-lint in a job separate from other jobs (go test, etc) because different jobs run in parallel. + go-linter: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + cache: false # caching can result in tar errors that files already exist + - name: set GOOS env to windows + run: | + echo "GOOS=windows" >> $GITHUB_ENV + - name: golangci-lint - windows + uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + with: + # Pin the version in case all the builds start to fail at the same time. + # There may not be an automatic way (e.g., dependabot) to update a specific parameter of a GitHub Action, + # so we will just update it manually whenever it makes sense (e.g., a feature that we want is added). + version: v1.56.1 + args: --fix=false --timeout=5m + - name: set GOOS env to darwin + run: | + echo "GOOS=darwin" >> $GITHUB_ENV + - name: golangci-lint - darwin + uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + with: + # Pin the version in case all the builds start to fail at the same time. + # There may not be an automatic way (e.g., dependabot) to update a specific parameter of a GitHub Action, + # so we will just update it manually whenever it makes sense (e.g., a feature that we want is added). + version: v1.56.1 + args: --fix=false --timeout=5m --skip-dirs="(^|/)deps($|/)" + shellcheck: + name: ShellCheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0 + with: + version: v0.9.0 + continue-on-error: true + go-mod-tidy-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + cache: true + # TODO: Use `go mod tidy --check` after https://github.com/golang/go/issues/27005 is fixed. + - run: go mod tidy + - run: git diff --exit-code + check-licenses: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + cache: true + - run: make check-licenses + macos-e2e-tests: + strategy: + matrix: + version: ['13', '14'] + test-command: ['test-e2e-vm-serial', 'test-e2e-container'] + arch: ['X64', 'arm64'] + runner-type: ['test'] + uses: ./.github/workflows/e2e-docs.yaml + windows-e2e-tests: + strategy: + matrix: + test-command: ['test-e2e-vm-serial', 'test-e2e-container'] + arch: ['amd64'] + runner-type: ['test'] + uses: ./.github/workflows/e2e-docs.yaml + linux-e2e-tests: + strategy: + matrix: + os: ['amazonlinux'] + arch: ['X64', 'arm64'] + version: ['2023', '2'] + runner-type: ['test'] + uses: ./.github/workflows/e2e-docs.yaml + + mdlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: avto-dev/markdown-lint@04d43ee9191307b50935a753da3b775ab695eceb # v1.5.0 + with: + args: '**/*.md' + # CHANGELOG.md is only updated by release-please bot. + ignore: 'CHANGELOG.md' + + get-intermediate-version: + name: Create intermediate version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get-intermediate-version.version }} + steps: + - name: Creates an intermediate version + id: calculate-version + run: | + version=$(git describe --match 'v[0-9]*' --dirty='.modified' --always --tags) + version="${version}-no-release" + echo "version=$version" >> ${GITHUB_OUTPUT} + + release-linux: + needs: get-intermediate-version + uses: ./.github/workflows/release-linux.yaml + secrets: inherit + with: + ref_name: "" + version: ${{ needs.get-intermediate-version.outputs.version }} + + build-and-test-finch-msi: + needs: get-intermediate-version + uses: ./.github/workflows/build-and-test-msi.yaml + secrets: inherit + with: + ref_name: "" + version: ${{ needs.get-intermediate-version.outputs.version }} + + macos-aarch64-pkg-build: + needs: get-intermediate-version + uses: ./.github/workflows/build-pkg.yaml + secrets: inherit + with: + os: macos + arch: arm64 + output_arch: aarch64 + version: 14 + tag: ${{ needs.get-intermediate-version.outputs.version }} + + macos-x86-64-pkg-build: + needs: get-intermediate-version + uses: ./.github/workflows/build-pkg.yaml + secrets: inherit + with: + os: macos + arch: amd64 + output_arch: x86_64 + version: 14 + tag: ${{ needs.get-intermediate-version.outputs.version }} + + macos-aarch64-pkg-test: + strategy: + fail-fast: false + matrix: + version: [13, 14] + needs: + - get-intermediate-version + - macos-aarch64-pkg-build + uses: ./.github/workflows/test-pkg.yaml + secrets: inherit + with: + os: macos + arch: arm64 + output_arch: aarch64 + version: ${{ matrix.version }} + tag: ${{ needs.get-intermediate-version.outputs.version } + + macos-x86-64-pkg-test: + strategy: + fail-fast: false + matrix: + version: [13, 14] + needs: + - get-intermediate-version + - macos-x86-64-pkg-build + uses: ./.github/workflows/test-pkg.yaml + secrets: inherit + with: + os: macos + arch: amd64 + output_arch: x86_64 + version: ${{ matrix.version }} + tag: ${{ needs.get-intermediate-version.outputs.version } diff --git a/.github/workflows/get-version-and-tag-for-ref.yaml b/.github/workflows/get-version-and-tag-for-ref.yaml new file mode 100644 index 000000000..0c6148b27 --- /dev/null +++ b/.github/workflows/get-version-and-tag-for-ref.yaml @@ -0,0 +1,55 @@ +name: get tag and version from ref + +on: + workflow_dispatch: + inputs: + ref_name: + description: "the ref (tag/branch) to use to extract tag/version" + required: true + type: string + workflow_call: + inputs: + ref_name: + description: "the ref (tag/branch) to use to extract tag/version" + required: true + type: string + version: + description: "override for version, will be used instead of ref if set, used for testing" + required: false + type: string + outputs: + tag: + description: "The first output string" + value: ${{ jobs.get-version-and-tag-for-ref.outputs.tag }} + version: + description: "The second output string" + value: ${{ jobs.get-version-and-tag-for-ref.outputs.version }} + +jobs: + get-version-and-tag-for-ref: + name: Get tag name + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.calculate-tag.outputs.tag }} + version: ${{ steps.calculate-tag.outputs.version }} + steps: + - name: Calculate a version and tag for a given ref + id: calculate-tag + run: | + if [ -n "${{ inputs.ref_name }}" ]; then + tag=${{ inputs.ref_name }} + else + tag=${{ github.ref_name }} + fi + echo "tag=$tag" >> ${GITHUB_OUTPUT} + + version=${tag#v} + if [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version matches format: $version" + elif [ -n "${{ inputs.ref_name }}" ]; then + version=${{ inputs.version }} + else + echo "Version $version doesn't match format. Using test version: 0.0.1" + version="0.0.1" + fi + echo "version=$version" >> ${GITHUB_OUTPUT} diff --git a/.github/workflows/release-linux.yaml b/.github/workflows/release-linux.yaml new file mode 100644 index 000000000..6816fd867 --- /dev/null +++ b/.github/workflows/release-linux.yaml @@ -0,0 +1,118 @@ +name: Release Linux + +on: + workflow_dispatch: + inputs: + ref_name: + description: "the ref (tag/branch) to use to extract tag/version" + required: true + type: string + workflow_call: + inputs: + ref_name: + description: "the ref (tag/branch) to use to extract tag/version" + required: true + type: string + version: + description: "override for version, will be used instead of ref if set, used for testing" + required: false + type: string + schedule: + - cron: '0 9 * * *' + +env: + GO_VERSION: "1.22.7" +permissions: + contents: write + deployments: write +jobs: + get-tag-name: + name: Get tag name + uses: ./.github/workflows/get-version-and-tag-for-ref.yaml + with: + ref_name: ${{ inputs.ref_name }} + version: ${{ inputs.version }} + generate-artifacts: + needs: get-latest-tag + runs-on: ubuntu-latest + env: + # Set during setup. + RELEASE_TAG: ${{ needs.get-latest-tag.outputs.tag }} + STATIC_BINARY_NAME: "" + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-tags: true + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + - name: "Echo RELEASE_TAG ENV" + run: echo ${{ env.RELEASE_TAG }} + - name: Build + run: | + RELEASE_VERSION="${{ needs.get-tag-name.outputs.version }}" + make download-licenses + # static amd64 + export STATIC_AMD64_BINARY_NAME="finch-${RELEASE_VERSION}-linux-amd64-static.tar.gz" + GOARCH=amd64 STATIC=1 make + pushd _output/ + touch "${STATIC_AMD64_BINARY_NAME}" + tar --exclude "*.tar.gz" --exclude "*.tar.gz.sha256sum" -cvzf "${STATIC_AMD64_BINARY_NAME}" . + echo "${STATIC_AMD64_BINARY_NAME}" >> $GITHUB_ENV + popd + rm -rf ./_output/bin/ + # static arm64 + export STATIC_ARM64_BINARY_NAME="finch-${RELEASE_VERSION}-linux-arm64-static.tar.gz" + GOARCH=arm64 STATIC=1 make + pushd _output/ + touch "${STATIC_ARM64_BINARY_NAME}" + tar --exclude "*.tar.gz" --exclude "*.tar.gz.sha256sum" -cvzf "${STATIC_ARM64_BINARY_NAME}" . + echo "${STATIC_ARM64_BINARY_NAME}" >> $GITHUB_ENV + popd + + pushd _output/ + sha256sum "${STATIC_AMD64_BINARY_NAME}" > "${STATIC_AMD64_BINARY_NAME}".sha256sum + sha256sum "${STATIC_ARM64_BINARY_NAME}" > "${STATIC_ARM64_BINARY_NAME}".sha256sum + popd + + rm -rf ./_output/bin/ + - name: Verify Release versions + run: | + mkdir -p ./_output/${{ env.RELEASE_TAG }}/static/amd64 + tar -xzf ./_output/${{ env.STATIC_AMD64_BINARY_NAME }} -C ./_output/static/amd64 + STATIC_AMD64_BINARY_VERSION=$(./output/${{ env.RELEASE_TAG }}/static/amd64/finch --version | grep -oP 'v\d+\.\d+\.\d+') + export RELEASE_TAG=${{ env.RELEASE_TAG }} + if [ "$STATIC_AMD64_BINARY_VERSION" != "$RELEASE_TAG" ]; then + echo "Version mismatch" + exit 1 + fi + - uses: actions/upload-artifact@v4 + with: + name: artifacts + path: release/ + if-no-files-found: error + outputs: + static_amd64_binary_name: ${{ env.STATIC_AMD64_BINARY_NAME }} + static_arm64_binary_name: ${{ env.STATIC_ARM64_BINARY_NAME }} + validate-artifacts: + needs: generate-artifacts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: artifacts + path: release/ + - run: bash scripts/verify-release-artifacts.sh ${{ needs.generate-artifacts.outputs.release_tag }} + upload-artifacts: + needs: + - generate-artifacts + runs-on: ubuntu-latest + steps: + - name: upload to S3 + run: | + aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_amd64_binary_name }} s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/ + aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_amd64_binary_name }}.sha256sum s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/ + aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_arm64_binary_name }} s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/ + aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_arm64_binary_name }}.sha256sum s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/ diff --git a/Makefile b/Makefile index c5adf413f..0411ff9e5 100644 --- a/Makefile +++ b/Makefile @@ -121,10 +121,8 @@ ifeq ($(GOOS),windows) finch: finch-windows finch-all else ifeq ($(GOOS),darwin) finch: finch-macos -else ifeq ($(NATIVE_BUILD),true) -finch: finch-native else -finch: finch-all +finch: finch-linux endif finch-windows: @@ -137,9 +135,11 @@ finch-macos: finch-unix finch-unix: finch-all -finch-native: GO_BUILD_TAGS += native -finch-native: finch-all +finch-linux: finch-all +ifneq ($(STATIC),) +finch-all: export CGO_ENABLED=0 +endif finch-all: $(GO) build -ldflags $(LDFLAGS) -tags "$(GO_BUILD_TAGS)" -o $(OUTDIR)/bin/$(BINARYNAME) $(PACKAGE)/cmd/finch