build: add linux release automation and release automation validation

.github/workflows/build-and-test-msi.yaml

@@ -5,13 +5,19 @@ on:
   workflow_dispatch:
     inputs:
       ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
   workflow_call:
     inputs:
       ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
+      version:
+        description: "override for version, will be used instead of ref if set, used for testing"
+        required: false
+        type: string
   schedule:
     - cron: '0 9 * * *'
 env:
@@ -26,29 +32,10 @@ permissions:
 jobs:
   get-tag-name:
     name: Get tag name
-    runs-on: ubuntu-latest
-    outputs:
-      tag: ${{ steps.check-tag.outputs.tag }}
-      version: ${{ steps.check-tag.outputs.version }}
-    steps:
-      - name: Check tag from workflow input and github ref
-        id: check-tag
-        run: |
-          if [ -n "${{ inputs.ref_name }}" ]; then
-            tag=${{ inputs.ref_name }}
-          else
-            tag=${{ github.ref_name }}
-          fi
-          echo "tag=$tag" >> ${GITHUB_OUTPUT}
-
-          version=${tag#v}
-          if [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "Version matches format: $version"
-          else
-            echo "Version $version doesn't match format. Using test version: 0.0.1"
-            version="0.0.1"
-          fi
-          echo "version=$version" >> ${GITHUB_OUTPUT}
+    uses: ./.github/workflows/get-version-and-tag-for-ref.yaml
+    with:
+      ref_name: ${{ inputs.ref_name }}
+      version: ${{ inputs.version }}
 
   windows-msi-build:
     needs: get-tag-name

.github/workflows/build-and-test-pkg.yaml

@@ -5,13 +5,19 @@ on:
   workflow_dispatch:
     inputs:
       ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
   workflow_call:
     inputs:
       ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
+      version:
+        description: "override for version, will be used instead of ref if set, used for testing"
+        required: false
+        type: string
   schedule:
     - cron: '0 9 * * *'
 env:
@@ -20,19 +26,10 @@ env:
 jobs:
   get-tag-name:
     name: Get tag name
-    runs-on: ubuntu-latest
-    outputs:
-      tag: ${{ steps.check-tag.outputs.tag }}
-    steps:
-      - name: Check tag from workflow input and github ref
-        id: check-tag
-        run: |
-          if [ -n "${{ inputs.ref_name }}" ]; then
-            tag=${{ inputs.ref_name }}
-          else
-            tag=${{ github.ref_name }}
-          fi
-          echo "tag=$tag" >> ${GITHUB_OUTPUT}
+    uses: ./.github/workflows/get-version-and-tag-for-ref.yaml
+    with:
+      ref_name: ${{ inputs.ref_name }}
+      version: ${{ inputs.version }}
 
   macos-aarch64-pkg-build:
     needs: get-tag-name

.github/workflows/ci-docs.yaml

@@ -24,6 +24,15 @@ on:
       - '!.github/workflows/e2e-macos.yaml'
       - '!.github/workflows/e2e-windows.yaml'
       - '!.github/workflows/e2e-linux.yaml'
+      - '!.github/workflows/release-automation.yaml'
+      - '!.github/workflows/release-linux.yaml'
+      - '!.github/workflows/upload-build-to-S3.yaml'
+      - '!.github/workflows/build-and-test-msi.yaml'
+      - '!.github/workflows/build-and-test-pkg.yaml'
+      - '!.github/workflows/ci-release.yaml'
+      - '!.github/workflows/get-version-and-tag-for-ref.yaml'
+      - '!.github/workflows/ci-docs.yaml'
+      - '!CHANGELOG.md'
   pull_request:
     branches:
       - main
@@ -39,6 +48,15 @@ on:
       - '!.github/workflows/e2e-macos.yaml'
       - '!.github/workflows/e2e-windows.yaml'
       - '!.github/workflows/e2e-linux.yaml'
+      - '!.github/workflows/release-automation.yaml'
+      - '!.github/workflows/release-linux.yaml'
+      - '!.github/workflows/upload-build-to-S3.yaml'
+      - '!.github/workflows/build-and-test-msi.yaml'
+      - '!.github/workflows/build-and-test-pkg.yaml'
+      - '!.github/workflows/ci-release.yaml'
+      - '!.github/workflows/get-version-and-tag-for-ref.yaml'
+      - '!.github/workflows/ci-docs.yaml'
+      - '!CHANGELOG.md'
 
 jobs:
   git-secrets:

.github/workflows/ci-release.yaml

@@ -0,0 +1,225 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/release-automation.yaml'
+      - '.github/workflows/release-linux.yaml'
+      - '.github/workflows/upload-build-to-S3.yaml'
+      - '.github/workflows/build-and-test-msi.yaml'
+      - '.github/workflows/build-and-test-pkg.yaml'
+      - '.github/workflows/get-version-and-tag-for-ref.yaml'
+      - 'deps/finch-core'
+      - 'CHANGELOG.md'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/release-automation.yaml'
+      - '.github/workflows/release-linux.yaml'
+      - '.github/workflows/upload-build-to-S3.yaml'
+      - '.github/workflows/build-and-test-msi.yaml'
+      - '.github/workflows/build-and-test-pkg.yaml'
+      - '.github/workflows/get-version-and-tag-for-ref.yaml'
+      - 'deps/finch-core'
+      - 'CHANGELOG.md'
+  workflow_dispatch:
+permissions:
+  id-token: write
+  contents: write
+
+env:
+  DEBUG: ${{ secrets.ACTIONS_STEP_DEBUG }}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  git-secrets:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pull latest awslabs/git-secrets repo
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          repository: awslabs/git-secrets
+          ref: 1.3.0
+          fetch-tags: true
+          path: git-secrets
+      - name: Install git secrets from source
+        run: sudo make install
+        working-directory: git-secrets
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Scan repository for git secrets
+        run: |
+          git secrets --register-aws
+          git secrets --scan-history
+
+  gen-code-no-diff:
+    strategy:
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+          cache: true
+      - run: make gen-code
+      - run: git diff --exit-code
+  unit-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Configure git CRLF settings
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          # Since this repository is not meant to be used as a library,
+          # we don't need to test the latest 2 major releases like Go does: https://go.dev/doc/devel/release#policy.
+          go-version-file: go.mod
+          cache: true
+      - run: make test-unit
+  # It's recommended to run golangci-lint in a job separate from other jobs (go test, etc) because different jobs run in parallel.
+  go-linter:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+          cache: false # caching can result in tar errors that files already exist
+      - name: set GOOS env to windows
+        run: |
+          echo "GOOS=windows" >> $GITHUB_ENV
+      - name: golangci-lint - windows
+        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
+        with:
+          # Pin the version in case all the builds start to fail at the same time.
+          # There may not be an automatic way (e.g., dependabot) to update a specific parameter of a GitHub Action,
+          # so we will just update it manually whenever it makes sense (e.g., a feature that we want is added).
+          version: v1.56.1
+          args: --fix=false --timeout=5m
+      - name: set GOOS env to darwin
+        run: |
+          echo "GOOS=darwin" >> $GITHUB_ENV
+      - name: golangci-lint - darwin
+        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
+        with:
+          # Pin the version in case all the builds start to fail at the same time.
+          # There may not be an automatic way (e.g., dependabot) to update a specific parameter of a GitHub Action,
+          # so we will just update it manually whenever it makes sense (e.g., a feature that we want is added).
+          version: v1.56.1
+          args: --fix=false --timeout=5m --skip-dirs="(^|/)deps($|/)"
+  shellcheck:
+    name: ShellCheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0
+        with:
+          version: v0.9.0
+        continue-on-error: true
+  go-mod-tidy-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+          cache: true
+      # TODO: Use `go mod tidy --check` after https://github.com/golang/go/issues/27005 is fixed.
+      - run: go mod tidy
+      - run: git diff --exit-code
+  check-licenses:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+          cache: true
+      - run: make check-licenses  
+  macos-e2e-tests:
+    strategy:
+      matrix:
+        version: ['13', '14']
+        test-command: ['test-e2e-vm-serial', 'test-e2e-container']
+        arch: ['X64', 'arm64']
+        runner-type: ['test']
+    uses: ./.github/workflows/e2e-docs.yaml
+  windows-e2e-tests:
+    strategy:
+      matrix:
+        test-command: ['test-e2e-vm-serial', 'test-e2e-container']
+        arch: ['amd64']
+        runner-type: ['test']
+    uses: ./.github/workflows/e2e-docs.yaml
+  linux-e2e-tests:
+    strategy:
+      matrix:
+        os: ['amazonlinux']
+        arch: ['X64', 'arm64']
+        version: ['2023', '2']
+        runner-type: ['test']
+    uses: ./.github/workflows/e2e-docs.yaml
+
+  mdlint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - uses: avto-dev/markdown-lint@04d43ee9191307b50935a753da3b775ab695eceb # v1.5.0
+        with:
+          args: '**/*.md'
+          # CHANGELOG.md is only updated by release-please bot.
+          ignore: 'CHANGELOG.md'
+
+  get-intermediate-version:
+      name: Get intermediate version
+      runs-on: ubuntu-latest
+      outputs:
+        version: ${{ steps.calculate-version.outputs.version }}
+      steps:
+        - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        - name: Gets an intermediate version string
+          id: calculate-version
+          run: |
+            version=$(git describe --match 'v[0-9]*' --dirty='.modified' --always --tags)
+            echo "${version}"
+            version="${version}-no-release"
+            echo "${version}"
+            echo "version=$version" >> ${GITHUB_OUTPUT}
+
+  release-linux:
+    needs: get-intermediate-version
+    uses: ./.github/workflows/release-linux.yaml
+    secrets: inherit
+    with:
+      ref_name: ""
+      version: ${{ needs.get-intermediate-version.outputs.version }}
+
+  build-and-test-finch-msi:
+    needs: get-intermediate-version
+    uses: ./.github/workflows/build-and-test-msi.yaml
+    secrets: inherit
+    with:
+      ref_name: ""
+      version: ${{ needs.get-intermediate-version.outputs.version }}
+
+  macos-aarch64-pkg-build:
+    needs: get-intermediate-version
+    uses: ./.github/workflows/build-and-test-pkg.yaml
+    secrets: inherit
+    with:
+      ref_name: ""
+      version: ${{ needs.get-intermediate-version.outputs.version }}