From 7430cefd3f411eb7fdc94af4b129ea35096cb2fc Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Thu, 9 May 2024 17:08:23 -0700 Subject: [PATCH 1/5] Add bin/write-envdir The script was copied from the ncov-ingest repo Adding in preparation for configuring separate AWS credentials for the build runtime in pathogen-repo-build. --- README.md | 1 + bin/write-envdir | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100755 bin/write-envdir diff --git a/README.md b/README.md index 94c25a0..261e48e 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,7 @@ Executable scripts that are used in our workflows. - [interpolate-env](bin/interpolate-env) - [json-to-envvars](bin/json-to-envvars) +- [write-envdir](bin/write-envdir) - [yaml-to-envvars](bin/yaml-to-envvars) ## Workflow text templates diff --git a/bin/write-envdir b/bin/write-envdir new file mode 100755 index 0000000..a190e91 --- /dev/null +++ b/bin/write-envdir @@ -0,0 +1,22 @@ +#!/bin/bash +# usage: write-envdir [[var1 [var2 [var3 […]]]] +# +# Writes the current value for each environment variable name given into the +# directory , one file per variable. Creates if it doesn't +# already exist. +# +# This was copied from the ncov-ingest repo: +# +# +set -eou pipefail + +dir="${1:?no envdir path}" +shift + +mkdir -pv "$dir" +cd "$dir" + +for name in "$@"; do + echo "${!name}" > "$name" + echo "Wrote $dir/$name" +done From 515637f6e93cbec62ff25524eb531e60a2c89d24 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Thu, 9 May 2024 16:20:29 -0700 Subject: [PATCH 2/5] pathogen-repo-build: rename `setup-aws-credentials` anchor Renamed to `setup-aws-batch-credentials` in preparation for adding another step that sets up AWS credentials for the runtime. The new step for setting up runtime AWS credentials probably won't need an anchor as it should only run once before the build starts. I just wanted to be specific with the existing anchor name to prevent any potential confusion. --- .github/workflows/pathogen-repo-build.yaml.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index dac5e9c..315ffe8 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -218,7 +218,7 @@ jobs: | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" - - &setup-aws-credentials + - &setup-aws-batch-credentials if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 with: @@ -294,7 +294,7 @@ jobs: steps: # Uses needs.workflow-context.outputs - *checkout-workflow-support - - *setup-aws-credentials + - *setup-aws-batch-credentials - *setup-runtime - id: attach @@ -379,7 +379,7 @@ jobs: steps: # Uses needs.workflow-context.outputs - *checkout-workflow-support - - *setup-aws-credentials + - *setup-aws-batch-credentials - *setup-runtime - id: cancel From 68c5c72f50fe30304a3d2284382ee9e8d32690c3 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 10 May 2024 12:37:09 -0700 Subject: [PATCH 3/5] pathogen-repo-build: support assuming AWS role for runtime permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assumes repo-specific roles, `GitHubActionsRoleNextstrainRepo@zika` for example, which are managed by the Terraform configuration in the nextstrain/infra repo. The repo here is always the _calling_ repository, regardless of if the "repo" input was provided to use workflows from another place. The runtime credentials are then saved in an envdir that is passed to the build command via `NEXTSTRAIN_RUNTIME_ENVDIRS`. Namespaced `NEXTSTRAIN_RUNTIME_ENVDIR` with `./git/nextstrain` as suggested by @tsibley in review¹ ¹ Related-to: Co-authored-by: Thomas Sibley --- .github/workflows/pathogen-repo-build.yaml | 52 +++++++++++++++--- .github/workflows/pathogen-repo-build.yaml.in | 54 ++++++++++++++++--- 2 files changed, 92 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index a3a418d..883dc15 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -47,12 +47,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ role is assumed (in both senses of the verb). The here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input. The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -138,12 +143,17 @@ on: If your build runs longer than the 6 hour limit for a single GitHub Action job, then use the aws-batch runtime and the `--detach` flag. Subsequent chained jobs will be automatically used to wait on the remote build for up to 24 hours total. - All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. If AWS credentials were acquired by the GitHub Action job via role assumption, the following environment variables are also available to be passed: + All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime via the `--env` option. + + It is assumed that the pathogen repo build requires AWS credentials for read/write access to S3 buckets. These may come directly from secrets or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + They must be defined in the repo's Actions secrets and passed to this workflow with `secrets: inherit`. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ role is assumed (in both senses of the verb). The here is always the _calling workflow's_ repository name (without owner), regardless of the "repo" input. The repository must already be configured in nextstrain/infra for the role to exist; see documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -210,6 +220,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write jobs: @@ -255,6 +266,30 @@ jobs: echo "$secrets" | jq 'del(.github_token)' | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 with: @@ -271,6 +306,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 315ffe8..8becae3 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -66,13 +66,26 @@ on: All environment variables provided via the env input and all secrets provided via `secrets: inherit` can be passed to the build runtime - via the `--env` option. If AWS credentials were acquired by the - GitHub Action job via role assumption, the following environment - variables are also available to be passed: + via the `--env` option. - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN + It is assumed that the pathogen repo build requires AWS credentials for + read/write access to S3 buckets. These may come directly from secrets + or indirectly from assuming a role via GitHub Actions' OIDC provider. + + The following secrets are used if present: + + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + + They must be defined in the repo's Actions secrets and passed to this + workflow with `secrets: inherit`. + + If no secrets are present, the GitHubActionsRoleNextstrainRepo@ + role is assumed (in both senses of the verb). The here is + always the _calling workflow's_ repository name (without owner), + regardless of the "repo" input. The repository must already be + configured in nextstrain/infra for the role to exist; see + documentation there for how to add the repository if necessary. type: string default: nextstrain build . required: false @@ -162,6 +175,7 @@ on: env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log + NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d permissions: id-token: write @@ -218,6 +232,33 @@ jobs: | "$NEXTSTRAIN_GITHUB_DIR"/bin/json-to-envvars | tee -a "$GITHUB_ENV" + - id: role + name: Set repo-specific role to (potentially) assume for runtime access to AWS + env: + REPO_FULL_NAME: ${{ github.repository }} + run: | + echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} + role-duration-seconds: 43200 # seconds, or 12 hours + + - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} + run: | + "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ + AWS_ACCESS_KEY_ID \ + AWS_SECRET_ACCESS_KEY \ + AWS_SESSION_TOKEN + + # This will overwrite the runtime AWS credential envvars configured above + # so if the build is using the aws-batch runtime, the Nextstrain CLI will + # have access to the AWS Batch session credentials + # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so + # outdenting comments to not repeat it with expanded YAML - &setup-aws-batch-credentials if: inputs.runtime == 'aws-batch' uses: aws-actions/configure-aws-credentials@v4 @@ -238,6 +279,7 @@ jobs: - name: Run build via ${{ inputs.runtime }} env: NEXTSTRAIN_BUILD_COMMAND: ${{ inputs.run }} + NEXTSTRAIN_RUNTIME_ENVDIRS: ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | # shellcheck disable=SC2154 set -x From 20f8226501a387bf25593353bd94238f8676e5e6 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Mon, 20 May 2024 16:47:18 -0700 Subject: [PATCH 4/5] pathogen-repo-build: Run `aws sts get-caller-identity` after acquiring credentials Very helpful for troubleshooting when looking at logs. Note that when using the AWS Batch runtime the credential identity will only be emitted for the initial job submission, not subsequent wait-N jobs. This seems fine. --- .github/workflows/pathogen-repo-build.yaml | 3 +++ .github/workflows/pathogen-repo-build.yaml.in | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index 883dc15..188eda7 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -279,6 +279,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} role-duration-seconds: 43200 # seconds, or 12 hours + - run: aws sts get-caller-identity - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ @@ -298,6 +299,8 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }} role-duration-seconds: 43200 # seconds, or 12 hours + - if: inputs.runtime == 'aws-batch' + run: aws sts get-caller-identity - name: Setup runtime ${{ inputs.runtime }} uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli with: diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 8becae3..1601000 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -246,6 +246,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} role-duration-seconds: 43200 # seconds, or 12 hours + - run: aws sts get-caller-identity - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | @@ -268,6 +269,8 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }} role-duration-seconds: 43200 # seconds, or 12 hours + - if: inputs.runtime == 'aws-batch' + run: aws sts get-caller-identity - &setup-runtime name: Setup runtime ${{ inputs.runtime }} From 3a2d597a020068825f9d463b8cc66c33933f8a92 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Mon, 20 May 2024 16:56:00 -0700 Subject: [PATCH 5/5] pathogen-repo-build: Rename the two aws-actions/configure-aws-credentials steps So they are more easily differentiated/recognized in the job logs. I found myself easily mixing them up when trying to find the right one. --- .github/workflows/pathogen-repo-build.yaml | 13 +++++++++++-- .github/workflows/pathogen-repo-build.yaml.in | 10 ++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index 188eda7..b645886 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -272,14 +272,16 @@ jobs: REPO_FULL_NAME: ${{ github.repository }} run: | echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" - - uses: aws-actions/configure-aws-credentials@v4 + - name: Configure credentials for runtime access to AWS + uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} role-duration-seconds: 43200 # seconds, or 12 hours - - run: aws sts get-caller-identity + - name: Report identity of runtime access to AWS + run: aws sts get-caller-identity - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ @@ -292,6 +294,7 @@ jobs: # Comment only applies to this first use of the `&setup-aws-batch-credentials`, so # outdenting comments to not repeat it with expanded YAML - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -300,6 +303,7 @@ jobs: role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }} role-duration-seconds: 43200 # seconds, or 12 hours - if: inputs.runtime == 'aws-batch' + name: Report identity of GitHub Actions job access to AWS Batch run: aws sts get-caller-identity - name: Setup runtime ${{ inputs.runtime }} uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli @@ -370,6 +374,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -427,6 +432,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -485,6 +491,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -543,6 +550,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -624,6 +632,7 @@ jobs: ref: ${{ needs.workflow-context.outputs.sha }} path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} - if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 1601000..2e7b031 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -239,14 +239,17 @@ jobs: run: | echo "arn=arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainRepo@${REPO_FULL_NAME#*/}" | tee -a "$GITHUB_OUTPUT" - - uses: aws-actions/configure-aws-credentials@v4 + - name: Configure credentials for runtime access to AWS + uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && steps.role.outputs.arn || '' }} role-duration-seconds: 43200 # seconds, or 12 hours - - run: aws sts get-caller-identity + + - name: Report identity of runtime access to AWS + run: aws sts get-caller-identity - name: Save runtime AWS credentials to ${{ env.NEXTSTRAIN_RUNTIME_ENVDIR }} run: | @@ -262,6 +265,7 @@ jobs: # outdenting comments to not repeat it with expanded YAML - &setup-aws-batch-credentials if: inputs.runtime == 'aws-batch' + name: Configure credentials for GitHub Actions job access to AWS Batch uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -269,7 +273,9 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} role-to-assume: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'arn:aws:iam::827581582529:role/GitHubActionsRoleNextstrainBatchJobs' || '' }} role-duration-seconds: 43200 # seconds, or 12 hours + - if: inputs.runtime == 'aws-batch' + name: Report identity of GitHub Actions job access to AWS Batch run: aws sts get-caller-identity - &setup-runtime