Propagate environment #1358 (#1392)

Co-authored-by: Ryan Ling <[email protected]> Co-authored-by: samchungy <[email protected]>
seek-oss · Jan 23, 2024 · 3a99839 · 3a99839
1 parent 454710c
commit 3a99839
Show file tree

Hide file tree

Showing 13 changed files with 102 additions and 71 deletions.
diff --git a/.changeset/new-crabs-buy.md b/.changeset/new-crabs-buy.md
@@ -0,0 +1,53 @@
+---
+'skuba': patch
+---
+
+template: Use `propagate-environment` for Docker Compose Buildkite plugin
+
+This simplifies the Docker Compose environment variable configuration required for Buildkite and GitHub integrations.
+
+In your `docker-compose.yml`:
+
+```diff
+services:
+  app:
+-   environment:
+-     # Enable Buildkite + GitHub integrations.
+-     - BUILDKITE
+-     - BUILDKITE_AGENT_ACCESS_TOKEN
+-     - BUILDKITE_BRANCH
+-     - BUILDKITE_BUILD_NUMBER
+-     - BUILDKITE_JOB_ID
+-     - BUILDKITE_PIPELINE_DEFAULT_BRANCH
+-     - BUILDKITE_STEP_ID
+-     - GITHUB_API_TOKEN
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
+    init: true
+    volumes:
+      - ./:/workdir
+      # Mount agent for Buildkite annotations.
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+      # Mount cached dependencies.
+      - /workdir/node_modules
+```
+
+In your `.buildkite/pipeline.yml`:
+
+```diff
+steps:
+  - commands:
+      - pnpm run lint
+      - pnpm run test
+    env:
+      # At SEEK, this instructs the build agent to populate the GITHUB_API_TOKEN environment variable for this step.
+      GET_GITHUB_TOKEN: 'please'
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.16.0:
++         environment:
++           - GITHUB_API_TOKEN
++         propagate-environment: true
+          run: app
+```
diff --git a/docs/deep-dives/github.md b/docs/deep-dives/github.md
@@ -42,25 +42,38 @@ steps:
             - /workdir/node_modules
 ```
 
-With Docker Compose,
-declare the environment variables and volume mounts in your [Compose file]:
+With Docker Compose, declare the volume mounts in your [Compose file]:
 
 ```yaml
 services:
   app:
-    environment:
-      # Enable GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - GITHUB_API_TOKEN
     volumes:
       - ./:/workdir
       # Mount cached dependencies.
       - /workdir/node_modules
 ```
 
+and the `environment` and `propagate-environment` options in the [Docker Compose Buildkite plugin]:
+
+```yaml
+steps:
+  - commands:
+      - pnpm run lint
+      - pnpm run test
+    env:
+      # At SEEK, this instructs the build agent to populate the GITHUB_API_TOKEN environment variable for this step.
+      GET_GITHUB_TOKEN: 'please'
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.16.0:
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
+          run: app
+```
+
 If you're running in GitHub Actions,
 your workflow will automatically have access to the following environment variables to achieve the same effect:
 
@@ -121,6 +134,7 @@ jobs:
 [actions/checkout]: https://github.com/actions/checkout
 [compose file]: https://docs.docker.com/compose/compose-file
 [docker buildkite plugin]: https://github.com/buildkite-plugins/docker-buildkite-plugin
+[docker compose buildkite plugin]: https://github.com/buildkite-plugins/docker-compose-buildkite-plugin
 [github checks api]: https://docs.github.com/en/rest/reference/checks/
 [github.createcheckrun]: ../development-api/github.md#createcheckrun
 [will not trigger workflows]: https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
diff --git a/scripts/test-template.sh b/scripts/test-template.sh
@@ -48,10 +48,12 @@ EOF
 
 mv "${directory}" "../${directory}"
 
+skuba_dir=$(pwd)
+
 cd "../${directory}" || exit 1
 
-echo "--- pnpm add --save-dev ../skuba/${skuba_tar}"
-pnpm add --save-dev "../skuba/${skuba_tar}"
+echo "--- pnpm add --save-dev ${skuba_dir}/${skuba_tar}"
+pnpm add --save-dev "${skuba_dir}/${skuba_tar}"
 
 echo "--- skuba version ${template}"
 pnpm exec skuba version

diff --git a/template/express-rest-api/.buildkite/pipeline.yml b/template/express-rest-api/.buildkite/pipeline.yml
@@ -56,6 +56,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - label: 📦 Build & Package

diff --git a/template/express-rest-api/docker-compose.yml b/template/express-rest-api/docker-compose.yml
@@ -2,16 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes:

diff --git a/template/greeter/.buildkite/pipeline.yml b/template/greeter/.buildkite/pipeline.yml
@@ -35,4 +35,7 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
diff --git a/template/greeter/docker-compose.yml b/template/greeter/docker-compose.yml
@@ -2,16 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes:

diff --git a/template/koa-rest-api/.buildkite/pipeline.yml b/template/koa-rest-api/.buildkite/pipeline.yml
@@ -56,6 +56,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - label: 📦 Build & Package

diff --git a/template/koa-rest-api/docker-compose.yml b/template/koa-rest-api/docker-compose.yml
@@ -2,16 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes:

diff --git a/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml b/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml
@@ -33,12 +33,18 @@ configs:
         - docker-compose#v4.16.0:
             dependencies: false
             run: app
+            environment:
+              - GITHUB_API_TOKEN
+            propagate-environment: true
       retry:
         manual:
           # Only use this if you need to roll back a deployment ASAP.
           # Always follow up with a proper revert or fix in Git history.
           permit_on_passed: true
 
+env:
+  VERSION: ${BUILDKITE_COMMIT:0:7}.${BUILDKITE_BUILD_NUMBER}
+
 steps:
   - label: 🧪 Test, Lint & Build
     commands:
@@ -56,6 +62,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - agents:

diff --git a/template/lambda-sqs-worker-cdk/docker-compose.yml b/template/lambda-sqs-worker-cdk/docker-compose.yml
@@ -2,21 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
-      # Tag AWS resources with the commit hash.
-      - BUILDKITE_COMMIT
-      # Pass through application configuration.
-      - ENVIRONMENT
-      - VERSION
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes:

diff --git a/template/lambda-sqs-worker/.buildkite/pipeline.yml b/template/lambda-sqs-worker/.buildkite/pipeline.yml
@@ -36,6 +36,7 @@ configs:
         - docker-compose#v4.16.0:
             dependencies: false
             run: app
+            propagate-environment: true
       retry:
         manual:
           # Only use this if you need to roll back a deployment ASAP.
@@ -65,6 +66,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - agents:

diff --git a/template/lambda-sqs-worker/docker-compose.yml b/template/lambda-sqs-worker/docker-compose.yml
@@ -2,21 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
-      # Tag AWS resources with the commit hash.
-      - BUILDKITE_COMMIT
-      # Pass through application configuration.
-      - ENVIRONMENT
-      - VERSION
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes: