-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #34 from ACCESS-NRI/32-prerelease
Generic `Prerelease` Functionality
- Loading branch information
Showing
8 changed files
with
233 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,6 +3,10 @@ concurrency: ${{ inputs.deployment-environment }} | |
| on: | ||
| workflow_call: | ||
| inputs: | ||
| type: | ||
| type: string | ||
| required: true | ||
| description: The type of deployment - either 'release' or 'prerelease' | ||
| model: | ||
| type: string | ||
| required: true | ||
|
|
@@ -72,11 +76,14 @@ jobs: | |
| # Release | ||
| - name: Get Release Metadata | ||
| if: inputs.type == 'release' | ||
| run: | | ||
| rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ | ||
| '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.*' \ | ||
| ./${{ inputs.env-name }} | ||
| - name: Create Release | ||
| if: inputs.type == 'release' | ||
| uses: softprops/[email protected] | ||
| with: | ||
| tag_name: ${{ inputs.version }} | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| name: Undeployment Setup | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| version-pattern: | ||
| type: string | ||
| required: true | ||
| description: A wildcard-supported string for version(s) of the model being removed | ||
| jobs: | ||
| get-spack-env: | ||
| name: Get Spack Environment | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| model: ${{ steps.get-model.outputs.model }} | ||
| env-name: ${{ steps.get-env-name.outputs.env-name }} | ||
| steps: | ||
| - name: Get Model | ||
| id: get-model | ||
| # for the cases where the repo name is in uppercase but the package name is lowercase (eg. access-nri/MOM5) | ||
| run: echo "model=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:])" >> $GITHUB_OUTPUT | ||
| - name: Set Spack Env Name String | ||
| id: get-env-name | ||
| # replace occurences of '.' with '_' in environment name as spack doesn't support '.'. Ex: 'access-om2-v1.0.0' -> 'access-om2-v1_0_0'. | ||
| run: echo "env-name=$(echo '${{ steps.get-model.outputs.model }}-${{ inputs.version-pattern }}' | tr '.' '_')" >> $GITHUB_OUTPUT | ||
|
|
||
| get-prerelease-deployment-env: | ||
| name: Get Prerelease Deployment Environment | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }} | ||
| steps: | ||
| - name: Checkout Config | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: access-nri/build-cd | ||
|
|
||
| - name: Get Environments | ||
| id: get-deployment-environment | ||
| run: echo "deployment-environments=$(jq --compact-output '.prerelease-environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT | ||
|
|
||
|
|
||
| undeployment: | ||
| name: Remove Deployment | ||
| needs: | ||
| - get-spack-env | ||
| - get-prerelease-deployment-env | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| deployment-environment: ${{ fromJson(needs.get-prerelease-deployment-env.outputs.deployment-environments) }} | ||
| uses: access-nri/build-cd/.github/workflows/undeploy-2-start.yml@main | ||
| with: | ||
| deployment-environment: ${{ matrix.deployment-environment }} | ||
| env-name: ${{ needs.get-spack-env.outputs.env-name }} | ||
| secrets: inherit |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| name: Undeployment Start | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| env-name: | ||
| type: string | ||
| required: true | ||
| description: The spack-env-compliant model name to remove | ||
| deployment-environment: | ||
| type: string | ||
| required: true | ||
| description: The GitHub deployment environment name | ||
| jobs: | ||
| undeploy-from-environment: | ||
| name: Undeploy ${{ inputs.env-name }} from ${{ inputs.deployment-environment }} | ||
| runs-on: ubuntu-latest | ||
| environment: ${{ inputs.deployment-environment }} | ||
| steps: | ||
| - name: Setup SSH | ||
| id: ssh | ||
| uses: access-nri/actions/.github/actions/setup-ssh@main | ||
| with: | ||
| private-key: ${{ secrets.SSH_KEY }} | ||
| hosts: | | ||
| ${{ secrets.HOST }} | ||
| ${{ secrets.HOST_DATA }} | ||
| - name: Undeploy | ||
| # ssh into deployment environment, create and activate the env, remove all the unneeded environments | ||
| id: undeploy | ||
| run: | | ||
| ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' | ||
| . ${{ vars.SPACK_CONFIG_LOCATION }}/spack-enable.bash | ||
| envs=$(find ${{ vars.SPACK_LOCATION }}/var/spack/environments -type d -name '${{ inputs.env-name }}' -printf '%f ') | ||
| for env in $envs; do | ||
| spack env activate $env | ||
| spack uninstall --remove --dependents --yes-to-all --all | ||
| spack env deactivate $env | ||
| spack env rm $env --yes-to-all | ||
| done | ||
| spack gc --yes-to-all | ||
| EOT | ||
| - name: Undeploy Status Notifier | ||
| if: always() | ||
| run: | | ||
| if [[ "${{ steps.undeploy.outcome }}" == "success" ]]; then | ||
| echo "::notice::Deployment ${{ inputs.env-name }} was successfully removed from ${{ inputs.deployment-environment }}, found at ${{ vars.SPACK_LOCATION }}" | ||
| else | ||
| echo "::error::Deployment ${{ inputs.env-name }} couldn't be removed from ${{ inputs.deployment-environment}}, found at ${{ vars.SPACK_LOCATION }}. Please check manually." | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,41 +1,102 @@ | ||
| # build-cd | ||
|
|
||
| This repository houses reusable workflows for the building and deployment of ACCESS-NRI models to different environments. | ||
| This repository houses reusable workflows for the building and deployment of ACCESS-NRI models to different environments. | ||
|
|
||
| ## Overview | ||
|
|
||
| This repository is broken down into two folders: `config` and `.github/workflows`. | ||
| This repository is broken down into two folders: `config` and `.github/workflows`. | ||
|
|
||
| `config` contains information on the deployment environments that the models deploy to, which is currently just the name of the deployment target. This is used by the aforementioned deployment workflows to gather secrets and configuration details from the associated GitHub Environment. | ||
| `config` contains information on the deployment environments that the models deploy to, which is currently just the name of the deployment target. This is used by the aforementioned deployment workflows to gather secrets and configuration details from the associated GitHub Environment. | ||
|
|
||
| `.github/workflows` houses validation and reusable deployment workflows that are called by ACCESS-NRI model repositories. Currently, only [ACCESS-NRI/ACCESS-OM2](https://github.com/ACCESS-NRI/access-om2) is supported. | ||
|
|
||
| Below is a brief summary of the two pipelines, `deploy-*` and `validate-json`. | ||
| Below is a brief summary of the three pipelines, `deploy-*`, `undeploy-*` and `validate-json`. | ||
|
|
||
| ### deploy-* | ||
| ### `deploy-*` | ||
|
|
||
| This pipeline is responsible for the gathering of configuration information, building and deployment of whatever model repository calls it. It is split into two workflows, as noted below. | ||
| #### Inputs | ||
|
|
||
| #### deploy-1-setup.yml | ||
| | Name | Type | Description | Required | Default | Example | | ||
| | ---- | ---- | ----------- | -------- | ------- | ------- | | ||
| | `type` | string | The type of deployment - either 'release' or 'prerelease' | true | `release` | `prerelease` | | ||
| | `version` | string | The version of the model being deployed | true | N/A | `2024.01.1` | | ||
|
|
||
| This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized deployments based on the list of environments. | ||
| #### Explanation | ||
|
|
||
| #### deploy-2-start.yml | ||
| This pipeline is responsible for the gathering of configuration information, building and deployment of whatever model repository calls it. It is split into two workflows. | ||
|
|
||
| Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository. | ||
| ##### `deploy-1-setup.yml` | ||
|
|
||
| ### validate-json | ||
| This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized deployments based on the list of environments. | ||
|
|
||
| This workflow is used to validate the `config` folders `*.json` files based on their associated `*.schema.json`. This is used for PR checks on the `build-cd` repo itself. | ||
| ##### `deploy-2-start.yml` | ||
|
|
||
| ## Usage | ||
| Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository, if it is not a `prerelease` deployment. | ||
|
|
||
| For supported `spack`-installable ACCESS-NRI models, simply call the `deploy-1-setup.yml` reusable workflow from the given repositories workflow file, like so: | ||
| #### Usage | ||
|
|
||
| For supported `spack`-installable ACCESS-NRI models, simply call the `deploy-1-setup.yml` reusable workflow from the given repositories workflow file, as shown below. Don't forget to add required inputs! | ||
|
|
||
| ```yml | ||
| deploy: | ||
| uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main | ||
| with: | ||
| version: 1.2.3 | ||
| secrets: inherit | ||
| permissions: | ||
| contents: write | ||
| ``` | ||
| ### `undeploy-*` | ||
|
|
||
| For given `spack` environments, we can also remove deployments. | ||
|
|
||
| > [!NOTE] | ||
| > This workflow is not one that should be used directly. It is used within the automated deployment pipeline to remove prerelease builds of models. | ||
|
|
||
| #### Inputs | ||
|
|
||
| | Name | Type | Description | Required | Default | Example | | ||
| | ---- | ---- | ----------- | -------- | ------- | ------- | | ||
| | `version-pattern` | string | A wildcard-supported string for version(s) of the model being removed | true | N/A | `2024.01.1-pre*` | | ||
|
|
||
| #### Explanation | ||
|
|
||
| This pipeline is responsible for removing deployed models from a deployment environment. Particular use-cases include removing `prerelease` builds from a deployment environment once a PR is merged. | ||
|
|
||
| ##### `undeploy-1-setup.yml` | ||
|
|
||
| This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized jobs removing the given `spack` environments based on the list of deployment environments. | ||
|
|
||
| ##### `undeploy-2-start.yml` | ||
|
|
||
| Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository, if it is not a `prerelease` deployment. | ||
|
|
||
| #### Usage | ||
|
|
||
| ```yml | ||
| remove-prereleases: | ||
| uses: access-nri/build-cd/.github/workflows/undeploy-2-setup.yml@main | ||
| with: | ||
| version-pattern: ${{ inputs.model }}-pre* | ||
| secrets: inherit | ||
| ``` | ||
|
|
||
| This will remove every `spack` environment from the deployment target that matches `<model>-pre*`. | ||
|
|
||
| ### `validate-json.yml` | ||
|
|
||
| This workflow is used to validate the `config` folders `*.json` files based on their associated `*.schema.json`. This is used for PR checks on the `build-cd` repo itself. | ||
|
|
||
| ### `create-deployment-spack.yml` | ||
|
|
||
| This workflow_dispatch-triggered workflow is used to create a version of `spack` on `Gadi`. | ||
|
|
||
| #### Inputs | ||
|
|
||
| | Name | Type | Description | Required | Default | Example | | ||
| | ---- | ---- | ----------- | -------- | ------- | ------- | | ||
| | `spack-version` | string | A version of `spack` | true | N/A | `0.20` | | ||
| | `spack-packages-version` | string | A version of ACCESS-NRI/spack-packages to be bundled with the install of `spack` | true | `main` | `2023.11.12` | | ||
| | `spack-config-version` | string | A version of ACCESS-NRI/spack-config to be bundled with the install of `spack` | true | `main` | `2024.01.01` | | ||
| | `deployment-location` | true | A path in the deployment environment where Spack should be created. For example, if it is `opt/spack`, spack will be installed under `opt/spack/<spack-version>/` | true | N/A | `/opt/spack` | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters