diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index a3a418d..b645886 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -47,12 +47,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ role is assumed (in both senses of the verb). The here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input. The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -138,12 +143,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ role is assumed (in both senses of the verb). The here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input. The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -210,6 +220,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write jobs: @@ -255,7 +266,35 @@ jobs: echo "$secrets" | jq 'del(.github_token)' | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + - name: Configure credentials for runtime access to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + - name: Report identity of runtime access to AWS + run: aws sts get-caller-identity + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -263,6 +302,9 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }} role-duration-seconds: 43200 # seconds, or 12 hours + - if: inputs.runtime == 'aws-batch' + name: Report identity of GitHub Actions job access to AWS Batch + run: aws sts get-caller-identity - name: Setup runtime ${{ inputs.runtime }} uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli with: @@ -271,6 +313,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x @@ -331,6 +374,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -388,6 +432,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -446,6 +491,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -504,6 +550,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -585,6 +632,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index dac5e9c..2e7b031 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -66,13 +66,26 @@ on: All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime - via the `--env` option. If AWS credentials were acquired by the - GitHub Action job via role assumption, the following environment - variables are also available to be passed: + via the `--env` option. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + It is assumed that the pathogen repo build requires AWS credentials for + read/write access to S3 buckets. These may come directly from secrets + or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this + workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ + role is assumed (in both senses of the verb). The here is + always the _calling workflow's_ repository name (without owner), + regardless of the "repo" input. The repository must already be + configured in nextstrain/infra for the role to exist; see + documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -162,6 +175,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write @@ -218,8 +232,40 @@ jobs: | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" - - &setup-aws-credentials + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + + - name: Configure credentials for runtime access to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + + - name: Report identity of runtime access to AWS + run: aws sts get-caller-identity + + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML + - &setup-aws-batch-credentials if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -228,6 +274,10 @@ jobs: role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }} role-duration-seconds: 43200 # seconds, or 12 hours + - if: inputs.runtime == 'aws-batch' + name: Report identity of GitHub Actions job access to AWS Batch + run: aws sts get-caller-identity + - &setup-runtime name: Setup runtime ${{ inputs.runtime }} uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli @@ -238,6 +288,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x @@ -294,7 +345,7 @@ jobs: steps: # Uses needs.workflow-context.outputs - *checkout-workflow-support - - *setup-aws-credentials + - *setup-aws-batch-credentials - *setup-runtime - id: attach @@ -379,7 +430,7 @@ jobs: steps: # Uses needs.workflow-context.outputs - *checkout-workflow-support - - *setup-aws-credentials + - *setup-aws-batch-credentials - *setup-runtime - id: cancel diff --git a/README.md b/README.md index 94c25a0..261e48e 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,7 @@ Executable scripts that are used in our workflows. - [interpolate-env](bin/interpolate-env) - [json-to-envvars](bin/json-to-envvars) +- [write-envdir](bin/write-envdir) - [yaml-to-envvars](bin/yaml-to-envvars) ## Workflow text templates diff --git a/bin/write-envdir b/bin/write-envdir new file mode 100755 index 0000000..a190e91 --- /dev/null +++ b/bin/write-envdir @@ -0,0 +1,22 @@ +#!/bin/bash +# usage: write-envdir [[var1 [var2 [var3 […]]]] +# +# Writes the current value for each environment variable name given into the +# directory , one file per variable. Creates if it doesn't +# already exist. +# +# This was copied from the ncov-ingest repo: +# +# +set -eou pipefail + +dir="${1:?no envdir path}" +shift + +mkdir -pv "$dir" +cd "$dir" + +for name in "$@"; do + echo "${!name}" > "$name" + echo "Wrote $dir/$name" +done