From 3a9983967bf6744c0eb95432f783b6e4fa26c290 Mon Sep 17 00:00:00 2001 From: Arnav Jain <93686265+ajain58@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:16:26 +1100 Subject: [PATCH] Propagate environment #1358 (#1392) Co-authored-by: Ryan Ling Co-authored-by: samchungy --- .changeset/new-crabs-buy.md | 53 +++++++++++++++++++ docs/deep-dives/github.md | 32 +++++++---- scripts/test-template.sh | 6 ++- .../express-rest-api/.buildkite/pipeline.yml | 3 ++ template/express-rest-api/docker-compose.yml | 10 ---- template/greeter/.buildkite/pipeline.yml | 3 ++ template/greeter/docker-compose.yml | 10 ---- template/koa-rest-api/.buildkite/pipeline.yml | 3 ++ template/koa-rest-api/docker-compose.yml | 10 ---- .../.buildkite/pipeline.yml | 9 ++++ .../lambda-sqs-worker-cdk/docker-compose.yml | 15 ------ .../lambda-sqs-worker/.buildkite/pipeline.yml | 4 ++ template/lambda-sqs-worker/docker-compose.yml | 15 ------ 13 files changed, 102 insertions(+), 71 deletions(-) create mode 100644 .changeset/new-crabs-buy.md diff --git a/.changeset/new-crabs-buy.md b/.changeset/new-crabs-buy.md new file mode 100644 index 000000000..35097efc8 --- /dev/null +++ b/.changeset/new-crabs-buy.md @@ -0,0 +1,53 @@ +--- +'skuba': patch +--- + +template: Use `propagate-environment` for Docker Compose Buildkite plugin + +This simplifies the Docker Compose environment variable configuration required for Buildkite and GitHub integrations. + +In your `docker-compose.yml`: + +```diff +services: + app: +- environment: +- # Enable Buildkite + GitHub integrations. +- - BUILDKITE +- - BUILDKITE_AGENT_ACCESS_TOKEN +- - BUILDKITE_BRANCH +- - BUILDKITE_BUILD_NUMBER +- - BUILDKITE_JOB_ID +- - BUILDKITE_PIPELINE_DEFAULT_BRANCH +- - BUILDKITE_STEP_ID +- - GITHUB_API_TOKEN + image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''} + init: true + volumes: + - ./:/workdir + # Mount agent for Buildkite annotations. + - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent + # Mount cached dependencies. + - /workdir/node_modules +``` + +In your `.buildkite/pipeline.yml`: + +```diff +steps: + - commands: + - pnpm run lint + - pnpm run test + env: + # At SEEK, this instructs the build agent to populate the GITHUB_API_TOKEN environment variable for this step. + GET_GITHUB_TOKEN: 'please' + plugins: + - *aws-sm + - *private-npm + - *docker-ecr-cache + - docker-compose#v4.16.0: ++ environment: ++ - GITHUB_API_TOKEN ++ propagate-environment: true + run: app +``` diff --git a/docs/deep-dives/github.md b/docs/deep-dives/github.md index cb7265549..55b6a4512 100644 --- a/docs/deep-dives/github.md +++ b/docs/deep-dives/github.md @@ -42,25 +42,38 @@ steps: - /workdir/node_modules ``` -With Docker Compose, -declare the environment variables and volume mounts in your [Compose file]: +With Docker Compose, declare the volume mounts in your [Compose file]: ```yaml services: app: - environment: - # Enable GitHub integrations. - - BUILDKITE - - BUILDKITE_BRANCH - - BUILDKITE_BUILD_NUMBER - - BUILDKITE_PIPELINE_DEFAULT_BRANCH - - GITHUB_API_TOKEN volumes: - ./:/workdir # Mount cached dependencies. - /workdir/node_modules ``` +and the `environment` and `propagate-environment` options in the [Docker Compose Buildkite plugin]: + +```yaml +steps: + - commands: + - pnpm run lint + - pnpm run test + env: + # At SEEK, this instructs the build agent to populate the GITHUB_API_TOKEN environment variable for this step. + GET_GITHUB_TOKEN: 'please' + plugins: + - *aws-sm + - *private-npm + - *docker-ecr-cache + - docker-compose#v4.16.0: + environment: + - GITHUB_API_TOKEN + propagate-environment: true + run: app +``` + If you're running in GitHub Actions, your workflow will automatically have access to the following environment variables to achieve the same effect: @@ -121,6 +134,7 @@ jobs: [actions/checkout]: https://github.com/actions/checkout [compose file]: https://docs.docker.com/compose/compose-file [docker buildkite plugin]: https://github.com/buildkite-plugins/docker-buildkite-plugin +[docker compose buildkite plugin]: https://github.com/buildkite-plugins/docker-compose-buildkite-plugin [github checks api]: https://docs.github.com/en/rest/reference/checks/ [github.createcheckrun]: ../development-api/github.md#createcheckrun [will not trigger workflows]: https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow diff --git a/scripts/test-template.sh b/scripts/test-template.sh index 22b1b596a..e0ae9a7e2 100755 --- a/scripts/test-template.sh +++ b/scripts/test-template.sh @@ -48,10 +48,12 @@ EOF mv "${directory}" "../${directory}" +skuba_dir=$(pwd) + cd "../${directory}" || exit 1 -echo "--- pnpm add --save-dev ../skuba/${skuba_tar}" -pnpm add --save-dev "../skuba/${skuba_tar}" +echo "--- pnpm add --save-dev ${skuba_dir}/${skuba_tar}" +pnpm add --save-dev "${skuba_dir}/${skuba_tar}" echo "--- skuba version ${template}" pnpm exec skuba version diff --git a/template/express-rest-api/.buildkite/pipeline.yml b/template/express-rest-api/.buildkite/pipeline.yml index f91a365f3..3d1358983 100644 --- a/template/express-rest-api/.buildkite/pipeline.yml +++ b/template/express-rest-api/.buildkite/pipeline.yml @@ -56,6 +56,9 @@ steps: - *docker-ecr-cache - docker-compose#v4.16.0: run: app + environment: + - GITHUB_API_TOKEN + propagate-environment: true timeout_in_minutes: 10 - label: ๐Ÿ“ฆ Build & Package diff --git a/template/express-rest-api/docker-compose.yml b/template/express-rest-api/docker-compose.yml index 0903e2f04..be1b82d32 100644 --- a/template/express-rest-api/docker-compose.yml +++ b/template/express-rest-api/docker-compose.yml @@ -2,16 +2,6 @@ version: '3.7' services: app: - environment: - # Enable Buildkite + GitHub integrations. - - BUILDKITE - - BUILDKITE_AGENT_ACCESS_TOKEN - - BUILDKITE_BRANCH - - BUILDKITE_BUILD_NUMBER - - BUILDKITE_JOB_ID - - BUILDKITE_PIPELINE_DEFAULT_BRANCH - - BUILDKITE_STEP_ID - - GITHUB_API_TOKEN image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''} init: true volumes: diff --git a/template/greeter/.buildkite/pipeline.yml b/template/greeter/.buildkite/pipeline.yml index f6fd232c1..9f33961ef 100644 --- a/template/greeter/.buildkite/pipeline.yml +++ b/template/greeter/.buildkite/pipeline.yml @@ -35,4 +35,7 @@ steps: - *docker-ecr-cache - docker-compose#v4.16.0: run: app + environment: + - GITHUB_API_TOKEN + propagate-environment: true timeout_in_minutes: 10 diff --git a/template/greeter/docker-compose.yml b/template/greeter/docker-compose.yml index 0903e2f04..be1b82d32 100644 --- a/template/greeter/docker-compose.yml +++ b/template/greeter/docker-compose.yml @@ -2,16 +2,6 @@ version: '3.7' services: app: - environment: - # Enable Buildkite + GitHub integrations. - - BUILDKITE - - BUILDKITE_AGENT_ACCESS_TOKEN - - BUILDKITE_BRANCH - - BUILDKITE_BUILD_NUMBER - - BUILDKITE_JOB_ID - - BUILDKITE_PIPELINE_DEFAULT_BRANCH - - BUILDKITE_STEP_ID - - GITHUB_API_TOKEN image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''} init: true volumes: diff --git a/template/koa-rest-api/.buildkite/pipeline.yml b/template/koa-rest-api/.buildkite/pipeline.yml index f91a365f3..3d1358983 100644 --- a/template/koa-rest-api/.buildkite/pipeline.yml +++ b/template/koa-rest-api/.buildkite/pipeline.yml @@ -56,6 +56,9 @@ steps: - *docker-ecr-cache - docker-compose#v4.16.0: run: app + environment: + - GITHUB_API_TOKEN + propagate-environment: true timeout_in_minutes: 10 - label: ๐Ÿ“ฆ Build & Package diff --git a/template/koa-rest-api/docker-compose.yml b/template/koa-rest-api/docker-compose.yml index 0903e2f04..be1b82d32 100644 --- a/template/koa-rest-api/docker-compose.yml +++ b/template/koa-rest-api/docker-compose.yml @@ -2,16 +2,6 @@ version: '3.7' services: app: - environment: - # Enable Buildkite + GitHub integrations. - - BUILDKITE - - BUILDKITE_AGENT_ACCESS_TOKEN - - BUILDKITE_BRANCH - - BUILDKITE_BUILD_NUMBER - - BUILDKITE_JOB_ID - - BUILDKITE_PIPELINE_DEFAULT_BRANCH - - BUILDKITE_STEP_ID - - GITHUB_API_TOKEN image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''} init: true volumes: diff --git a/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml b/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml index dc78586b2..157438526 100644 --- a/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml +++ b/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml @@ -33,12 +33,18 @@ configs: - docker-compose#v4.16.0: dependencies: false run: app + environment: + - GITHUB_API_TOKEN + propagate-environment: true retry: manual: # Only use this if you need to roll back a deployment ASAP. # Always follow up with a proper revert or fix in Git history. permit_on_passed: true +env: + VERSION: ${BUILDKITE_COMMIT:0:7}.${BUILDKITE_BUILD_NUMBER} + steps: - label: ๐Ÿงช Test, Lint & Build commands: @@ -56,6 +62,9 @@ steps: - *docker-ecr-cache - docker-compose#v4.16.0: run: app + environment: + - GITHUB_API_TOKEN + propagate-environment: true timeout_in_minutes: 10 - agents: diff --git a/template/lambda-sqs-worker-cdk/docker-compose.yml b/template/lambda-sqs-worker-cdk/docker-compose.yml index 175670fff..5b8f68b9a 100644 --- a/template/lambda-sqs-worker-cdk/docker-compose.yml +++ b/template/lambda-sqs-worker-cdk/docker-compose.yml @@ -2,21 +2,6 @@ version: '3.7' services: app: - environment: - # Enable Buildkite + GitHub integrations. - - BUILDKITE - - BUILDKITE_AGENT_ACCESS_TOKEN - - BUILDKITE_BRANCH - - BUILDKITE_BUILD_NUMBER - - BUILDKITE_JOB_ID - - BUILDKITE_PIPELINE_DEFAULT_BRANCH - - BUILDKITE_STEP_ID - - GITHUB_API_TOKEN - # Tag AWS resources with the commit hash. - - BUILDKITE_COMMIT - # Pass through application configuration. - - ENVIRONMENT - - VERSION image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''} init: true volumes: diff --git a/template/lambda-sqs-worker/.buildkite/pipeline.yml b/template/lambda-sqs-worker/.buildkite/pipeline.yml index b764a24fc..245dee10b 100644 --- a/template/lambda-sqs-worker/.buildkite/pipeline.yml +++ b/template/lambda-sqs-worker/.buildkite/pipeline.yml @@ -36,6 +36,7 @@ configs: - docker-compose#v4.16.0: dependencies: false run: app + propagate-environment: true retry: manual: # Only use this if you need to roll back a deployment ASAP. @@ -65,6 +66,9 @@ steps: - *docker-ecr-cache - docker-compose#v4.16.0: run: app + environment: + - GITHUB_API_TOKEN + propagate-environment: true timeout_in_minutes: 10 - agents: diff --git a/template/lambda-sqs-worker/docker-compose.yml b/template/lambda-sqs-worker/docker-compose.yml index 6441a52fb..be1b82d32 100644 --- a/template/lambda-sqs-worker/docker-compose.yml +++ b/template/lambda-sqs-worker/docker-compose.yml @@ -2,21 +2,6 @@ version: '3.7' services: app: - environment: - # Enable Buildkite + GitHub integrations. - - BUILDKITE - - BUILDKITE_AGENT_ACCESS_TOKEN - - BUILDKITE_BRANCH - - BUILDKITE_BUILD_NUMBER - - BUILDKITE_JOB_ID - - BUILDKITE_PIPELINE_DEFAULT_BRANCH - - BUILDKITE_STEP_ID - - GITHUB_API_TOKEN - # Tag AWS resources with the commit hash. - - BUILDKITE_COMMIT - # Pass through application configuration. - - ENVIRONMENT - - VERSION image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''} init: true volumes: