From 92ae023409a31000338176e1f7484a2e21ac6729 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 27 Jun 2024 10:52:39 +1000 Subject: [PATCH] Removed old CI/CD and replaced with simplified version (#10) --- .github/workflows/cd.yml | 51 +------ .github/workflows/ci.yml | 259 ++++---------------------------- .github/workflows/comment.yml | 101 ------------- .github/workflows/pr-closed.yml | 39 ----- 4 files changed, 33 insertions(+), 417 deletions(-) delete mode 100644 .github/workflows/comment.yml delete mode 100644 .github/workflows/pr-closed.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 6e8f129..a2ccbfc 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -7,53 +7,12 @@ on: paths: - config/** - spack.yaml -env: - SPACK_YAML_MODEL_YQ: .spack.specs[0] jobs: - generate-tag: - name: Generate Tag Name - # Get the tag name from the `spack.yaml` that was merged into main, which - # is of the form `access-om2@git.`. - runs-on: ubuntu-latest - outputs: - name: ${{ steps.tag.outputs.name }} - steps: - - uses: actions/checkout@v4 - - - name: Generate Tag - id: tag - # Get the tag name from the access-om2 spec in the `spack.yaml`. - run: | - access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) - echo "name=${access_om2_package/*@git./}" >> $GITHUB_OUTPUT - - push-tag: - name: Tag Deployment - needs: - - generate-tag - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - name: Push Tag - # NOTE: Regarding the config user.name/user.email, see https://github.com/actions/checkout/pull/1184 - run: | - git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} - git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} - git tag ${{ needs.generate-tag.outputs.name }} --force - git push --tags --force - - deploy-release: - name: Deploy Release - needs: - - generate-tag - - push-tag - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main + cd: + name: CD + uses: access-nri/build-cd/.github/workflows/cd.yml@main with: - ref: ${{ github.ref_name }} - version: ${{ needs.generate-tag.outputs.name }} - secrets: inherit + model: ${{ vars.NAME }} permissions: contents: write + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13e368c..33f6828 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,250 +1,47 @@ name: CI on: pull_request: + types: + - opened + - reopened + - synchronize + - closed branches: - main - backport/*.* paths: - config/** - spack.yaml -env: - SPACK_YAML_MODEL_YQ: .spack.specs[0] - MODEL_NAME: access-om2-bgc + issue_comment: + types: + - created + - edited jobs: - # There are a lot of interconnected jobs here. Here is a dependency diagram: - # validate-json ──> check-json ─────────┬─────────────────────────┐ - # ├> notifier ├> prerelease-deploy - # check-spack-yaml ──> deploy-versions ─┼> update-prerelease-tag ─┤ - # └─────────────────────────┘ - - # --------------------- - # | JSON-RELATED JOBS | - # --------------------- - validate-json: - name: Validate JSON - uses: access-nri/actions/.github/workflows/validate-json.yml@main + pr-ci: + name: CI + if: github.event_name == 'pull_request' && github.event.action != 'closed' + uses: access-nri/build-cd/.github/workflows/ci.yml@main with: - src: "config" - - check-json: - name: Check JSON Fields - needs: - - validate-json - runs-on: ubuntu-latest - outputs: - spack-packages-version: ${{ steps.versions.outputs.packages }} - spack-config-version: ${{ steps.versions.outputs.config }} - steps: - - uses: actions/checkout@v4 - - # The next two steps checkout the spack-{packages,config} repos to confirm that the versions in - # versions.json exist in the repositories. - - name: Setup - id: versions - run: | - echo "packages=$(jq --compact-output --raw-output '."spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT - echo "config=$(jq --compact-output --raw-output '."spack-config"' ./config/versions.json)" >> $GITHUB_OUTPUT - - - name: Spack Packages - id: spack-packages - continue-on-error: true - uses: actions/checkout@v4 - with: - repository: access-nri/spack-packages - ref: ${{ steps.versions.outputs.packages }} - path: packages - - - name: Spack Config - id: spack-config - continue-on-error: true - uses: actions/checkout@v4 - with: - repository: access-nri/spack-config - ref: ${{ steps.versions.outputs.config }} - path: config - - - name: Failure Notifier - if: contains(steps.*.outcome, 'failure') - run: | - if [[ "${{ steps.spack-packages.outcome }}" == "failure" ]]; then - echo "::error::spack-packages at the specified ref (${{ steps.versions.outputs.packages }}) doesn't exist." - fi - if [[ "${{ steps.spack-config.outcome }}" == "failure" ]]; then - echo "::error::spack-config at the specified ref (${{ steps.versions.outputs.config }}) doesn't exist." - fi - exit 1 - - # --------------------------- - # | SPACK.YAML-RELATED JOBS | - # --------------------------- - check-spack-yaml: - name: Check spack.yaml - runs-on: ubuntu-latest + model: ${{ vars.NAME }} permissions: pull-requests: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Check Model Version Modified - id: version - run: | - git checkout ${{ github.base_ref }} - base_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) - - git checkout ${{ github.head_ref }} - current_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) - echo "current=${current_version}" >> $GITHUB_OUTPUT - - if [[ "${base_version}" == "${current_version}" ]]; then - echo "::warning::The version string hasn't been modified in this PR, but needs to be before merging." - exit 1 - fi - - - name: Same Model Version Failure Notifier - if: failure() && steps.version.outcome == 'failure' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BODY: | - The model version in the `spack.yaml` has not been updated. - Either update it manually, or comment the following to have it updated and committed automatically: - * `!bump major` for feature releases - * `!bump minor` for bugfixes - run: | - gh pr checkout ${{ github.event.pull_request.number }} - gh pr comment --body '${{ env.BODY }}' - - - name: Projection Version Matches - # this step checks that the versions of the packages themselves match with the - # names of the modules. For example, mom5@git.2023.12.12 matches with the - # modulefile mom5/2023.12.12 (specifically, the version strings match) - run: | - FAILED='false' - DEPS=$(yq ".spack.modules.default.tcl.include | join(\" \")" spack.yaml) - - # for each of the packages (access-om2-bgc, mom5, cice5...) - for DEP in $DEPS; do - DEP_VER='' - if [[ "$DEP" == "${{ env.MODEL_NAME }}" ]]; then - DEP_STR=$(yq ".spack.specs[]" spack.yaml) - else - DEP_STR=$(yq ".spack.packages.\"$DEP\".require" spack.yaml) - fi - DEP_VER=${DEP_STR/*@git./} - - MODULE_STR=$(yq ".spack.modules.default.tcl.projections.\"$DEP\"" spack.yaml) - MODULE_VER=${MODULE_STR/*\//} # Keep the '2023.12.12' after the '/' in '{name}/2024.12.12' - - if [[ "$DEP_VER" != "$MODULE_VER" ]]; then - echo "::error::Version of dependency and projection of $DEP do not match ($DEP_VER != $MODULE_VER)" - FAILED='true' - else - echo "Version of dependency and projection of $DEP match ($DEP_VER)" - fi - done - if [[ "$FAILED" == "true" ]]; then - exit 1 - fi - - get-versions: - name: Get Version and Build Number - needs: - - check-spack-yaml - runs-on: ubuntu-latest - outputs: - release: ${{ steps.get-release.outputs.version }} - prerelease: ${{ steps.get-prerelease.outputs.version }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 - - - name: Generate Release Version - id: get-release - # The step generates a general version number from the spack.yaml, looking the - # same as a regular release build. - # Ex. 'access-om2-bgc@git.2024.01.1' -> '2024.01.1' - run: | - access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) - echo "version=${access_om2_package/*@git./}" >> $GITHUB_OUTPUT - - - name: Generate Prerelease Version - id: get-prerelease - # This step generates the version number for prereleases, - # which looks like: `pr-`. - # Ex. Pull Request #12 with 2 commits on branch -> `pr12-2`. - run: | - number_of_commits=$(git rev-list --count ${{ github.event.pull_request.base.sha }}..HEAD) - echo "version=pr${{ github.event.pull_request.number }}-${number_of_commits}" >> $GITHUB_OUTPUT - - update-prerelease-tag: - name: Update Prerelease Tag ${{ needs.get-versions.outputs.release }} - needs: - - get-versions - runs-on: ubuntu-latest - permissions: contents: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Push - # NOTE: Regarding the config user.name/user.email, see https://github.com/actions/checkout/pull/1184 - run: | - git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} - git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} - git tag ${{ needs.get-versions.outputs.release }} --force - git push --tags --force - - # ----------------------------- - # | PRERELEASE DEPLOYMENT JOB | - # ----------------------------- - prerelease-deploy: - name: Deploy to Prerelease - # This will create a `spack` environment with the name `access-om2-bgc-pr-`. - # For example, `access-om2-bgc-pr12-3` for the deployment based on the third commit on the 12th PR. - needs: - - get-versions # so we can give an appropriate version to the prerelease build - - update-prerelease-tag # implies all the spack.yaml-related checks have passed - - check-json # implies all the json-related checks have passed - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main - with: - type: prerelease - ref: ${{ github.head_ref }} - version: ${{ needs.get-versions.outputs.prerelease }} secrets: inherit - notifier: - name: Notifier - runs-on: ubuntu-latest - needs: - - check-json - - get-versions + pr-comment: + name: Comment + if: github.event_name == 'issue_comment' + uses: access-nri/build-cd/.github/workflows/ci-comment.yml@main + with: + model: ${{ vars.NAME }} permissions: pull-requests: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 - - - name: Comment - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - BODY: | - This `${{ github.repository }}` model will be deployed with the following versions: - * `${{ needs.get-versions.outputs.release }}` as a Release (when merged). - * `${{ needs.get-versions.outputs.prerelease }}` as a Prerelease (during this PR). This can be accessed on `Gadi` via `spack` at `/g/data/vk83/prerelease/apps/spack/0.20/spack` once deployed. - - It will be deployed using: - * `access-nri/spack-packages` version [`${{ needs.check-json.outputs.spack-packages-version }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.check-json.outputs.spack-packages-version }}) - * `access-nri/spack-config` version [`${{ needs.check-json.outputs.spack-config-version }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.check-json.outputs.spack-config-version }}) + contents: write - If this is not what was expected, commit changes to `config/versions.json`. - run: | - gh pr checkout ${{ github.event.pull_request.number }} - gh pr comment --body '${{ env.BODY }}' + pr-closed: + name: Closed + if: github.event_name == 'pull_request' && github.event.action == 'closed' + uses: access-nri/build-cd/.github/workflows/ci-closed.yml@main + with: + model: ${{ vars.NAME }} + secrets: inherit diff --git a/.github/workflows/comment.yml b/.github/workflows/comment.yml deleted file mode 100644 index 572ddc7..0000000 --- a/.github/workflows/comment.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Comment Command -on: - issue_comment: - types: - - created - - edited -env: - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - SPACK_YAML_MODEL_YQ: .spack.specs[0] - SPACK_YAML_MODEL_PROJECTION_YQ: .spack.modules.default.tcl.projections.access-om2 -jobs: - bump-version: - name: Bump spack.yaml - if: github.event.issue.pull_request && startsWith(github.event.comment.body, '!bump') - runs-on: ubuntu-latest - permissions: - pull-requests: write - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }} - - - name: Setup - id: setup - # outputs: - # original-version: The version contained within the spack.yaml - # version: The version that will be bumped (could be latest tag instead of original-version) - # bump: The bump type (major, minor or current as specified in the bump-version action) - run: | - # Get the version of access-om2 from the spack.yaml in the PR the comment was written in - gh pr checkout ${{ github.event.issue.number }} - access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) - original_version=${access_om2_package/*@git./} - echo "original-version=${original_version}" >> $GITHUB_OUTPUT - - # Validate the comment - if [[ "${{ contains(github.event.comment.body, 'major') }}" == "true" ]]; then - # Compare the current date (year-month) with the latest git tag (year-month) - # to determine the next valid tag. We do this because especially feature-rich - # months might increment the date part beyond the current date. - - d="$(date +%Y-%m)-01" - d_s=$(date --date "$d" +%s) - - latest_tag=$(git describe --tags --abbrev=0 | tr '.' '-') - tag_date=${latest_tag%-*}-01 - tag_date_s=$(date --date "$tag_date" +%s) - - echo "Comparing current date ${d} with ${tag_date} (tag looks like ${latest_tag})" - - if (( d_s <= tag_date_s )); then - echo "version=${tag_date}" >> $GITHUB_OUTPUT - echo "bump=major" >> $GITHUB_OUTPUT - else - echo "version=${original_version}" >> $GITHUB_OUTPUT - echo "bump=current" >> $GITHUB_OUTPUT - fi - elif [[ "${{ contains(github.event.comment.body, 'minor')}}" == "true" ]]; then - echo "version=${original_version}" >> $GITHUB_OUTPUT - echo "bump=minor" >> $GITHUB_OUTPUT - else - echo "::warning::Usage: `!bump [major|minor]`, got `${{ github.event.comment.body }}`" - exit 1 - fi - - - name: Bump Version - id: bump - uses: access-nri/actions/.github/actions/bump-version@main - with: - version: ${{ steps.setup.outputs.version }} - versioning-scheme: calver-minor - bump-type: ${{ steps.setup.outputs.bump }} - - - name: Update, Commit and Push the Bump - run: | - git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} - git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} - - yq -i '${{ env.SPACK_YAML_MODEL_YQ }} = "access-om2@git.${{ steps.bump.outputs.after }}"' spack.yaml - yq -i '${{ env.SPACK_YAML_MODEL_PROJECTION_YQ }} = "{name}/${{ steps.bump.outputs.after }}"' spack.yaml - git add spack.yaml - git commit -m "spack.yaml: Updated access-om2 package version from ${{ steps.setup.outputs.original-version }} to ${{ steps.bump.outputs.after }}" - git push - - - name: Success Notifier - env: - BODY: | - :white_check_mark: Version bumped from `${{ steps.setup.outputs.original-version }}` to `${{ steps.bump.outputs.after }}` :white_check_mark: - run: | - gh pr comment --body '${{ env.BODY }}' - - - name: Failure Notifier - if: failure() - env: - BODY: | - :x: Failed to bump version or commit changes, see ${{ env.RUN_URL }} :x: - run: gh pr comment --body '${{ env.BODY }}' diff --git a/.github/workflows/pr-closed.yml b/.github/workflows/pr-closed.yml deleted file mode 100644 index 6ea3c20..0000000 --- a/.github/workflows/pr-closed.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: PR Closed Cleanup -# Remove prereleases that were part of a closed PR, so we save space -# on our deployment targets. If needed, one can still get the -# spack.yaml as part of the closed PR and revive it themselves. -on: - pull_request: - types: - - closed - branches: - - main - - backport/*.* - paths: - - config/** - - spack.yaml -env: - SPACK_YAML_MODEL_YQ: .spack.specs[0] -jobs: - setup: - name: Setup - runs-on: ubuntu-latest - outputs: - version-pattern: ${{ steps.version.outputs.pattern }} - steps: - - name: Version Pattern - id: version - # For example, `access-om2-bgc-pr12-*` - run: | - repo_name_lower=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:]) - echo "pattern=${repo_name_lower}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT - - - undeploy-prereleases: - name: Undeploy Prereleases - needs: - - setup - uses: access-nri/build-cd/.github/workflows/undeploy-1-setup.yml@main - with: - version-pattern: ${{ needs.setup.outputs.version-pattern }} - secrets: inherit