diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 4e2abc8f86..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,161 +0,0 @@ ---- -common-steps: - - &rebaseontarget - run: - name: Rebase on top of GitHub target branch - command: ./devops/scripts/rebase-ci.sh - - - &installenchant - run: - name: Install enchant - command: sudo apt-get update && sudo apt-get install -y enchant - - - &createcachedir - run: - name: Ensure cache dir exists and permissions are good - command: | - sudo mkdir -p /caches && sudo chown circleci: -R /caches - - - &restorecache - restore_cache: - key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }} - paths: - - /caches/layers.tar.gz - - - &loadimagelayers - run: - name: Load image layer cache - command: | - set +o pipefail - docker load -i /caches/layers.tar || true - - - &dockerimagebuild - run: - name: Build Docker images - command: | - set +o pipefail - docker images - fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') - cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" ./bin/dev-shell true - - - &saveimagelayers - run: - name: Save Docker image layer cache - command: | - docker images - docker save -o /caches/layers.tar securedrop-test-focal-py3:latest - - - &savecache - save_cache: - key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }} - paths: - - /caches/layers.tar - -version: 2.1 -orbs: - slack: circleci/slack@4.4.4 - -slack-fail-post-step: &slack-fail-post-step - post-steps: - - run: - name: Slack - Skip from Forks - command: | - if [ -z "$SLACK_ACCESS_TOKEN" ]; then - circleci-agent step halt - fi - - slack/notify: - channel: C01EY9C1X45 - event: fail - template: basic_fail_1 - -jobs: - lint: - machine: - image: ubuntu-2004:current - enabled: true - environment: - DOCKER_API_VERSION: 1.24 - BASE_OS: focal - steps: - - checkout - - *rebaseontarget - - *createcachedir - - *restorecache - - *loadimagelayers - - *dockerimagebuild - - *saveimagelayers - - *savecache - - *installenchant - - - run: - name: Run all linters - command: | - fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') - DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell \ - bash -c "/opt/venvs/securedrop-app-code/bin/pip3 install --require-hashes -r requirements/python3/develop-requirements.txt && make -C .. lint" - - - run: - name: Check that translation machine objects are reproducible - command: | - fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') - DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell \ - bash -c "/opt/venvs/securedrop-app-code/bin/pip3 install --require-hashes -r requirements/python3/develop-requirements.txt && make -C .. verify-mo" - - staging-test-with-rebase: - machine: - image: ubuntu-2004:current - enabled: true - - working_directory: ~/sd - steps: - - checkout - - *rebaseontarget - - *installenchant - - - run: - name: Run Staging tests on GCE - command: | - make ci-go - no_output_timeout: 35m - - - run: - name: Ensure environment torn down - # Always report true, since env should will destroyed already - # if all tests passed. - command: make ci-teardown || true - when: always - - - store_test_results: - path: ~/sd/junit - - - store_artifacts: - path: ~/sd/junit - -workflows: - version: 2 - securedrop_ci: - jobs: - - lint: - context: - - circleci-slack - <<: *slack-fail-post-step - - staging-test-with-rebase: - filters: - branches: - only: /(stg-|release\/).*/ - requires: - - lint - context: - - circleci-slack - <<: *slack-fail-post-step - - nightly: - triggers: - - schedule: - cron: "0 4 * * *" - filters: - branches: - only: - - develop - jobs: - - staging-test-with-rebase diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml new file mode 100644 index 0000000000..94084f8d91 --- /dev/null +++ b/.github/workflows/staging.yml @@ -0,0 +1,23 @@ +name: Staging +on: + push: + branches: + - 'stg-*' + - 'release/*' + schedule: + - cron: '0 3 * * *' + +jobs: + staging: + runs-on: ubuntu-latest + env: + GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }} + steps: + - uses: actions/checkout@v4 + - name: Run staging tests on GCE + run: | + make ci-go + - name: Teardown (if needed) + if: always() + run: | + make ci-teardown || true diff --git a/devops/gce-nested/ci-env.sh b/devops/gce-nested/ci-env.sh index 82309162e0..a2d7af355a 100644 --- a/devops/gce-nested/ci-env.sh +++ b/devops/gce-nested/ci-env.sh @@ -4,8 +4,8 @@ # If these scripts are run on developer workstations, the CI env # vars populated by CircleCI won't be present; make a sane default. -if [ -z "${CIRCLE_BUILD_NUM:-}" ]; then - export CIRCLE_BUILD_NUM="${USER}" +if [ -z "${GITHUB_RUN_ID:-}" ]; then + export GITHUB_RUN_ID="${USER}" fi # Set common vars we'll need throughout the CI scripts. @@ -13,7 +13,7 @@ TOPLEVEL="$(git rev-parse --show-toplevel)" export TOPLEVEL GCE_CREDS_FILE="${TOPLEVEL}/.gce.creds" export GCE_CREDS_FILE -export BUILD_NUM="${CIRCLE_BUILD_NUM}" +export BUILD_NUM="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" export PROJECT_ID="securedrop-ci" export JOB_NAME="sd-ci-nested" export GCLOUD_MACHINE_TYPE="c2-standard-8" diff --git a/devops/gce-nested/gce-start.sh b/devops/gce-nested/gce-start.sh index 9fa2b7469a..5f78c365b8 100755 --- a/devops/gce-nested/gce-start.sh +++ b/devops/gce-nested/gce-start.sh @@ -46,6 +46,7 @@ function create_sd_ci_gce_instance() { --metadata "ssh-keys=${SSH_USER_NAME}:$(cat $SSH_PUBKEY)" # Give box a few more seconds for SSH to become available + echo "Sleeping for 20s to wait for SSH to become available" sleep 20 fi } diff --git a/devops/gce-nested/gce-stop.sh b/devops/gce-nested/gce-stop.sh index e46053f0b8..4366fed21d 100755 --- a/devops/gce-nested/gce-stop.sh +++ b/devops/gce-nested/gce-stop.sh @@ -11,4 +11,4 @@ TOPLEVEL="$(git rev-parse --show-toplevel)" . "${TOPLEVEL}/devops/gce-nested/ci-env.sh" # Destroy remote instance -gcloud_call --quiet compute instances delete "${JOB_NAME}-${BUILD_NUM}" +gcloud_call compute instances delete "${JOB_NAME}-${BUILD_NUM}" diff --git a/devops/scripts/rebase-ci.sh b/devops/scripts/rebase-ci.sh deleted file mode 100755 index 78d024cd82..0000000000 --- a/devops/scripts/rebase-ci.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2086 -# -# To be utilized in CircleCI -# Determine target branch from PR number and rebase against that - -set -e -set -u -set -o pipefail - - -# Only run this logic on PRs in CI, not on regular branch checks. -if [[ -n "${CIRCLE_PULL_REQUEST:-}" ]] -then - ORGANIZATION=${CIRCLE_PROJECT_USERNAME} - REPOSITORY=${CIRCLE_PROJECT_REPONAME} - GITHUB_PR_URL="https://api.github.com/repos/${ORGANIZATION}/${REPOSITORY}/pulls/${CIRCLE_PULL_REQUEST##*/}" - # basic auth credentials for private repositories - if [[ -n "${GITHUB_AUTH:-}" ]]; then - CURL_ARGS="-u ${GITHUB_AUTH}" - else - CURL_ARGS="" - fi - - # Git will yell at you when trying to rebase without author/email configured - git config --global user.email "ci@freedom.press" - git config --global user.name "CI User" - - # Ensure presence of upstream remote - git ls-remote --exit-code --quiet upstream 2>/dev/null || git remote add upstream https://github.com/${ORGANIZATION}/${REPOSITORY}.git - - # Determine target branch via API - target_branch="$(curl ${CURL_ARGS} -s ${GITHUB_PR_URL} | python -c 'import sys, json; print(json.load(sys.stdin)["base"]["ref"])')" - - # Fetch and rebase onto the latest in develop - git fetch upstream "${target_branch}" - git rebase "upstream/${target_branch}" - - # Print out the current head for debugging potential CI issues - git rev-parse HEAD -fi