From 78b43c3aba45ecf7984c736e3c9d71248b44b90f Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Thu, 19 Sep 2024 09:34:08 +0200 Subject: [PATCH] chore(ci) : Deploy to staging through a PR label It is annoying to be competing with each other when pushing code, so I feel like it's going to help if we decide to deploy through adding a non-mandatory label than "all the time unless draft". Also, drafts mean something different IMHO. They should be used as "please do not review yet, this is a work in progress, open for discussion". This commit also splits the jobs in two: - one job will be responsible of building and deploying images to staging - another one will deploy to prod, the 'release' branch only. Reusable actions have been used to ensure DRYness. --- .github/workflows/_build_images.yml | 78 +++++++++++++ .github/workflows/_terraform_deploy.yml | 62 ++++++++++ .github/workflows/build_deploy.yml | 149 ------------------------ .github/workflows/deploy_prod.yml | 15 +++ .github/workflows/deploy_staging.yml | 19 +++ 5 files changed, 174 insertions(+), 149 deletions(-) create mode 100644 .github/workflows/_build_images.yml create mode 100644 .github/workflows/_terraform_deploy.yml delete mode 100644 .github/workflows/build_deploy.yml create mode 100644 .github/workflows/deploy_prod.yml create mode 100644 .github/workflows/deploy_staging.yml diff --git a/.github/workflows/_build_images.yml b/.github/workflows/_build_images.yml new file mode 100644 index 000000000..192fe2ffe --- /dev/null +++ b/.github/workflows/_build_images.yml @@ -0,0 +1,78 @@ +name: Build docker images + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + service: ["api", "datawarehouse", "pipeline"] + + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-${{ matrix.service }} + + permissions: + contents: read + packages: write + + defaults: + run: + working-directory: ${{ matrix.service }} + + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=sha,format=long,prefix= + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and export to Docker + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.service }} + load: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max + labels: ${{ steps.meta.outputs.labels }} + + - name: Run tests + if: matrix.service == 'api' + env: + API_ENV: test + run: | + docker compose run --entrypoint pytest api -p no:cacheprovider -vv + + - name: Run tests + if: matrix.service == 'pipeline' + run: | + echo #TODO + + - name: Push image to GitHub registry + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.service }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/_terraform_deploy.yml b/.github/workflows/_terraform_deploy.yml new file mode 100644 index 000000000..bf471e782 --- /dev/null +++ b/.github/workflows/_terraform_deploy.yml @@ -0,0 +1,62 @@ +name: Build docker images + +on: + workflow_call: + inputs: + environment: + type: string + description: 'Github environment to use' + required: true + +jobs: + deploy: + runs-on: ubuntu-latest + + environment: ${{ inputs.environment }}" + + env: + ENV: ${{ vars.ENVIRONMENT }} + AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TF_VARS_BASE64: ${{ secrets.TF_VARS_BASE64 }} + TF_VAR_stack_version: ${{ github.sha }} + + defaults: + run: + working-directory: deployment + + steps: + - uses: actions/checkout@v4 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.8.1" + + - name: mask tf variables + run: | + echo "${TF_VARS_BASE64}" \ + | base64 --decode \ + | jq 'to_entries | map(.value // empty) | .[]' \ + | xargs -I{} echo '::add-mask::{}' + + - name: tf init + run: | + terraform init \ + -backend-config "bucket=data-inclusion-tf-states" \ + -backend-config "key=${ENV}" + + - name: tf validate + run: | + terraform validate + + - name: tf plan + run: | + trap "rm -f terraform.tfvars.json" EXIT + echo "${TF_VARS_BASE64}" | base64 --decode > terraform.tfvars.json + terraform plan -input=false + + - name: tf apply + run: | + trap "rm -f terraform.tfvars.json" EXIT + echo "${TF_VARS_BASE64}" | base64 --decode > terraform.tfvars.json + terraform apply -input=false -auto-approve diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml deleted file mode 100644 index ac2484400..000000000 --- a/.github/workflows/build_deploy.yml +++ /dev/null @@ -1,149 +0,0 @@ -# References -# https://docs.docker.com/build/ci/github-actions/ -# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images - -name: build_deploy - -on: - push: - branches: [main] - pull_request: - branches: [main] - # default types + ready_for_review - types: [opened, synchronize, repopened, ready_for_review] - -jobs: - build: - runs-on: ubuntu-latest - - if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.draft) }} - - strategy: - matrix: - service: ["api", "datawarehouse", "pipeline"] - - env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}-${{ matrix.service }} - - permissions: - contents: read - packages: write - - defaults: - run: - working-directory: ${{ matrix.service }} - - steps: - - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=ref,event=pr - type=sha,format=long,prefix= - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and export to Docker - uses: docker/build-push-action@v5 - with: - context: ./${{ matrix.service }} - load: true - tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max - labels: ${{ steps.meta.outputs.labels }} - - - name: Run tests - if: matrix.service == 'api' - env: - API_ENV: test - run: | - docker compose run --entrypoint pytest api -p no:cacheprovider -vv - - - name: Run tests - if: matrix.service == 'pipeline' - run: | - echo #TODO - - - name: Push image to GitHub registry - uses: docker/build-push-action@v5 - with: - context: ./${{ matrix.service }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - deploy: - needs: build - runs-on: ubuntu-latest - - strategy: - matrix: - environment: [staging, prod] - - # prevent deployment failure in an environment to interrupt other deployments - fail-fast: true - - environment: ${{ matrix.environment }} - - env: - ENV: ${{ vars.ENVIRONMENT }} - AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TF_VARS_BASE64: ${{ secrets.TF_VARS_BASE64 }} - TF_VAR_stack_version: ${{ github.sha }} - - defaults: - run: - working-directory: deployment - - steps: - - uses: actions/checkout@v4 - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: "1.8.1" - - - name: mask tf variables - run: | - echo "${TF_VARS_BASE64}" \ - | base64 --decode \ - | jq 'to_entries | map(.value // empty) | .[]' \ - | xargs -I{} echo '::add-mask::{}' - - - name: tf init - run: | - terraform init \ - -backend-config "bucket=data-inclusion-tf-states" \ - -backend-config "key=${ENV}" - - - name: tf validate - run: | - terraform validate - - - name: tf plan - run: | - trap "rm -f terraform.tfvars.json" EXIT - echo "${TF_VARS_BASE64}" | base64 --decode > terraform.tfvars.json - terraform plan -input=false - - - name: tf apply - run: | - trap "rm -f terraform.tfvars.json" EXIT - echo "${TF_VARS_BASE64}" | base64 --decode > terraform.tfvars.json - terraform apply -input=false -auto-approve diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml new file mode 100644 index 000000000..45d251bda --- /dev/null +++ b/.github/workflows/deploy_prod.yml @@ -0,0 +1,15 @@ +name: deploy_prod + +on: + push: + branches: [release] + +jobs: + build: + uses: ./.github/workflows/_build_images.yml + + deploy_prod: + needs: build + uses: ./.github/workflows/_terraform_deploy.yml + with: + environment: prod diff --git a/.github/workflows/deploy_staging.yml b/.github/workflows/deploy_staging.yml new file mode 100644 index 000000000..ba0f0f5c3 --- /dev/null +++ b/.github/workflows/deploy_staging.yml @@ -0,0 +1,19 @@ +name: build_deploy_staging + +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review, labeled] + +jobs: + build: + uses: ./.github/workflows/_build_images.yml + + deploy_staging: + needs: build + if: contains(github.event.pull_request.labels.*.name, 'deploy-to-staging') + uses: ./.github/workflows/_terraform_deploy.yml + with: + environment: staging