Merge pull request #101 from nextstrain/pathogen-repo-build-grab-bag

`pathogen-repo-build` improvements
nextstrain · Jun 21, 2024 · 2e5550a · 2e5550a
2 parents 18e3f79 + e5d5d4f
commit 2e5550a
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 1 deletion.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -49,7 +49,7 @@ jobs:
       id-token: write
     strategy:
       matrix:
-        runtime: [docker, conda, aws-batch]
+        runtime: [docker, conda]
     uses: ./.github/workflows/pathogen-repo-build.yaml
     with:
       repo: nextstrain/zika-tutorial
@@ -60,6 +60,37 @@ jobs:
         BOGUS_ENVVAR: BOGUS_ENVVAR
     secrets: inherit
 
+  # This job has to be separate from the matrix above because we want
+  # to check the aws-batch-job-id output from the workflow but matrix outputs
+  # currently will overwrite each other.
+  # See <https://github.com/orgs/community/discussions/26639>
+  test-pathogen-repo-build-aws-batch:
+    permissions:
+      id-token: write
+    uses: ./.github/workflows/pathogen-repo-build.yaml
+    with:
+      repo: nextstrain/zika-tutorial
+      runtime: aws-batch
+      run: nextstrain build --env BOGUS_ENVVAR .
+      artifact-name: build-output-aws-batch
+      env: |
+        BOGUS_ENVVAR: BOGUS_ENVVAR
+    secrets: inherit
+
+  test-pathogen-repo-build-aws-batch-id-output:
+    needs: [test-pathogen-repo-build-aws-batch]
+    permissions:
+      id-token: write
+    uses: ./.github/workflows/pathogen-repo-build.yaml
+    with:
+      repo: nextstrain/zika-tutorial
+      runtime: aws-batch
+      run: nextstrain build --attach "$AWS_BATCH_JOB_ID" .
+      artifact-name: build-output-aws-batch-reattached
+      env: |
+        AWS_BATCH_JOB_ID: ${{ needs.test-pathogen-repo-build-aws-batch.outputs.aws-batch-job-id }}
+    secrets: inherit
+
   lint:
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml
@@ -115,6 +115,16 @@ on:
           This is not supported for builds on AWS Batch because the workflow detaches from the build. Please use the `nextstrain build` command locally to reattach to AWS Batch builds to download outputs.
         type: string
         required: false
+    outputs:
+      aws-batch-job-id:
+        description: >-
+          AWS Batch job id of the build if it was run with the `aws-batch` runtime. This can be used to re-attach to the AWS Batch job with:
+
+
+            nextstrain build --aws-batch --attach <aws-batch-job-id>
+
+          This can be useful to download results from the build and use them in subsequent jobs.
+        value: ${{ jobs.run-build.outputs.AWS_BATCH_JOB_ID }}
   workflow_dispatch:
     inputs:
       runtime:
@@ -287,6 +297,7 @@ jobs:
           "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \
             AWS_ACCESS_KEY_ID \
             AWS_SECRET_ACCESS_KEY \
+            AWS_DEFAULT_REGION \
             ${AWS_SESSION_TOKEN:+AWS_SESSION_TOKEN}
         # This will overwrite the runtime AWS credential envvars configured above
         # so if the build is using the aws-batch runtime, the Nextstrain CLI will

diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in
@@ -162,6 +162,19 @@ on:
         type: string
         required: false
 
+    outputs:
+      aws-batch-job-id:
+        description: >-
+          AWS Batch job id of the build if it was run with the `aws-batch` runtime.
+          This can be used to re-attach to the AWS Batch job with:
+
+            nextstrain build --aws-batch --attach <aws-batch-job-id>
+
+          This can be useful to download results from the build and use them
+          in subsequent jobs.
+
+        value: ${{ jobs.run-build.outputs.AWS_BATCH_JOB_ID }}
+
   workflow_dispatch:
     inputs:
       <<: *inputs
@@ -256,6 +269,7 @@ jobs:
           "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \
             AWS_ACCESS_KEY_ID \
             AWS_SECRET_ACCESS_KEY \
+            AWS_DEFAULT_REGION \
             ${AWS_SESSION_TOKEN:+AWS_SESSION_TOKEN}
 
     # This will overwrite the runtime AWS credential envvars configured above