[Dev] Build 398 of branch main by @rjsparks #398
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Release | |
| run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} | |
| on: | |
| push: | |
| branches: [release] | |
| workflow_dispatch: | |
| inputs: | |
| summary: | |
| description: 'Release Summary' | |
| required: false | |
| type: string | |
| default: '' | |
| sandbox: | |
| description: 'Deploy to Sandbox' | |
| default: true | |
| required: true | |
| type: boolean | |
| legacySandbox: | |
| description: 'Deploy to Legacy Sandbox' | |
| default: false | |
| required: false | |
| type: boolean | |
| skiptests: | |
| description: 'Skip Tests' | |
| default: false | |
| required: true | |
| type: boolean | |
| ignoreLowerCoverage: | |
| description: 'Ignore Lower Coverage' | |
| default: false | |
| required: true | |
| type: boolean | |
| updateCoverage: | |
| description: 'Update Baseline Coverage' | |
| default: false | |
| required: true | |
| type: boolean | |
| jobs: | |
| # ----------------------------------------------------------------- | |
| # PREPARE | |
| # ----------------------------------------------------------------- | |
| prepare: | |
| name: Prepare Release | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_deploy: ${{ steps.buildvars.outputs.should_deploy }} | |
| pkg_version: ${{ steps.buildvars.outputs.pkg_version }} | |
| from_tag: ${{ steps.semver.outputs.nextStrict }} | |
| to_tag: ${{ steps.semver.outputs.current }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get Next Version | |
| if: ${{ github.ref_name == 'release' }} | |
| id: semver | |
| uses: ietf-tools/semver-action@v1 | |
| with: | |
| token: ${{ github.token }} | |
| branch: release | |
| skipInvalidTags: true | |
| - name: Set Next Version Env Var | |
| if: ${{ github.ref_name == 'release' }} | |
| run: | | |
| echo "NEXT_VERSION=$nextStrict" >> $GITHUB_ENV | |
| - name: Create Draft Release | |
| uses: ncipollo/[email protected] | |
| if: ${{ github.ref_name == 'release' }} | |
| with: | |
| prerelease: true | |
| draft: false | |
| commit: ${{ github.sha }} | |
| tag: ${{ env.NEXT_VERSION }} | |
| name: ${{ env.NEXT_VERSION }} | |
| body: '*pending*' | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set Build Variables | |
| id: buildvars | |
| run: | | |
| if [[ $NEXT_VERSION ]]; then | |
| echo "Using AUTO SEMVER mode: $NEXT_VERSION" | |
| echo "should_deploy=true" >> $GITHUB_OUTPUT | |
| echo "pkg_version=$NEXT_VERSION" >> $GITHUB_OUTPUT | |
| echo "::notice::Release $NEXT_VERSION created using branch $GITHUB_REF_NAME" | |
| else | |
| echo "Using TEST mode: 11.0.0-dev.$GITHUB_RUN_NUMBER" | |
| echo "should_deploy=false" >> $GITHUB_OUTPUT | |
| echo "pkg_version=11.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT | |
| echo "::notice::Non-production build 11.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" | |
| fi | |
| # ----------------------------------------------------------------- | |
| # TESTS | |
| # ----------------------------------------------------------------- | |
| tests-python: | |
| name: Run Tests (Python) | |
| if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} | |
| needs: [prepare] | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/ietf-tools/datatracker-app-base:latest | |
| services: | |
| db: | |
| image: ghcr.io/ietf-tools/datatracker-db:latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Prepare for tests | |
| run: | | |
| chmod +x ./dev/tests/prepare.sh | |
| sh ./dev/tests/prepare.sh | |
| - name: Ensure DB is ready | |
| run: | | |
| /usr/local/bin/wait-for db:5432 -- echo "DB ready" | |
| - name: Run all tests | |
| shell: bash | |
| run: | | |
| echo "Running checks..." | |
| ./ietf/manage.py check | |
| ./ietf/manage.py migrate --fake-initial | |
| echo "Validating migrations..." | |
| if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then | |
| echo "Model changes without migrations found." | |
| exit 1 | |
| fi | |
| echo "Running tests..." | |
| if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then | |
| echo "Lower coverage failures will be ignored." | |
| ./ietf/manage.py test --settings=settings_postgrestest --ignore-lower-coverage | |
| else | |
| ./ietf/manage.py test --settings=settings_postgrestest | |
| fi | |
| coverage xml | |
| - name: Upload Coverage Results to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| files: coverage.xml | |
| - name: Convert Coverage Results | |
| if: ${{ always() }} | |
| run: | | |
| mv latest-coverage.json coverage.json | |
| - name: Upload Coverage Results as Build Artifact | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| name: coverage | |
| path: coverage.json | |
| tests-playwright: | |
| name: Run Tests (Playwright) | |
| if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} | |
| needs: [prepare] | |
| runs-on: macos-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| project: [chromium, firefox] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - name: Run all tests | |
| run: | | |
| echo "Installing dependencies..." | |
| yarn | |
| echo "Installing Playwright..." | |
| cd playwright | |
| mkdir test-results | |
| npm ci | |
| npx playwright install --with-deps ${{ matrix.project }} | |
| echo "Running tests..." | |
| npx playwright test --project=${{ matrix.project }} | |
| - name: Upload Report | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| with: | |
| name: playwright-results-${{ matrix.project }} | |
| path: playwright/test-results/ | |
| if-no-files-found: ignore | |
| tests-playwright-legacy: | |
| name: Run Tests (Playwright Legacy) | |
| if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} | |
| needs: [prepare] | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/ietf-tools/datatracker-app-base:latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| project: [chromium, firefox] | |
| services: | |
| db: | |
| image: ghcr.io/ietf-tools/datatracker-db:latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Prepare for tests | |
| run: | | |
| chmod +x ./dev/tests/prepare.sh | |
| sh ./dev/tests/prepare.sh | |
| - name: Ensure DB is ready | |
| run: | | |
| /usr/local/bin/wait-for db:5432 -- echo "DB ready" | |
| - name: Start Datatracker | |
| run: | | |
| echo "Running checks..." | |
| ./ietf/manage.py check | |
| echo "Starting datatracker..." | |
| ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local & | |
| echo "Waiting for datatracker to be ready..." | |
| /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready" | |
| - name: Run all tests | |
| env: | |
| # Required to get firefox to run as root: | |
| HOME: "" | |
| run: | | |
| echo "Installing dependencies..." | |
| yarn | |
| echo "Installing Playwright..." | |
| cd playwright | |
| mkdir test-results | |
| npm ci | |
| npx playwright install --with-deps ${{ matrix.project }} | |
| echo "Running tests..." | |
| npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js | |
| - name: Upload Report | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| with: | |
| name: playwright-legacy-results-${{ matrix.project }} | |
| path: playwright/test-results/ | |
| if-no-files-found: ignore | |
| # ----------------------------------------------------------------- | |
| # RELEASE | |
| # ----------------------------------------------------------------- | |
| release: | |
| name: Make Release | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: [tests-python, tests-playwright, tests-playwright-legacy, prepare] | |
| runs-on: ubuntu-latest | |
| env: | |
| SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} | |
| PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} | |
| FROM_TAG: ${{needs.prepare.outputs.from_tag}} | |
| TO_TAG: ${{needs.prepare.outputs.to_tag}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: '16' | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Download a Coverage Results | |
| if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: coverage | |
| - name: Make Release Build | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| echo "PKG_VERSION: $PKG_VERSION" | |
| echo "GITHUB_SHA: $GITHUB_SHA" | |
| echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" | |
| echo "Running build script..." | |
| chmod +x ./dev/deploy/build.sh | |
| sh ./dev/deploy/build.sh | |
| echo "Setting version $PKG_VERSION..." | |
| sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py | |
| sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py | |
| sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py | |
| - name: Set Production Flags | |
| if: ${{ env.SHOULD_DEPLOY == 'true' }} | |
| run: | | |
| echo "Setting production flags in settings.py..." | |
| sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py | |
| - name: Make Release Tarball | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| echo "Build release tarball..." | |
| mkdir -p /home/runner/work/release | |
| tar -czf /home/runner/work/release/release.tar.gz -X dev/deploy/exclude-patterns.txt . | |
| - name: Update CHANGELOG | |
| id: changelog | |
| uses: Requarks/changelog-action@v1 | |
| if: ${{ env.SHOULD_DEPLOY == 'true' }} | |
| with: | |
| token: ${{ github.token }} | |
| fromTag: ${{ env.FROM_TAG }} | |
| toTag: ${{ env.TO_TAG }} | |
| writeToFile: false | |
| - name: Prepare Coverage Action | |
| if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} | |
| working-directory: ./dev/coverage-action | |
| run: npm install | |
| - name: Process Coverage Stats + Chart | |
| id: covprocess | |
| uses: ./dev/coverage-action/ | |
| if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} | |
| with: | |
| token: ${{ github.token }} | |
| tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} | |
| repoCommon: common | |
| version: ${{needs.prepare.outputs.pkg_version}} | |
| changelog: ${{ steps.changelog.outputs.changes }} | |
| summary: ${{ github.event.inputs.summary }} | |
| coverageResultsPath: coverage.json | |
| histCoveragePath: historical-coverage.json | |
| - name: Create Release | |
| uses: ncipollo/[email protected] | |
| if: ${{ env.SHOULD_DEPLOY == 'true' }} | |
| with: | |
| allowUpdates: true | |
| makeLatest: true | |
| draft: false | |
| tag: ${{ env.PKG_VERSION }} | |
| name: ${{ env.PKG_VERSION }} | |
| body: ${{ steps.covprocess.outputs.changelog }} | |
| artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update Baseline Coverage | |
| uses: ncipollo/[email protected] | |
| if: ${{ github.event.inputs.updateCoverage == 'true' }} | |
| with: | |
| allowUpdates: true | |
| tag: baseline | |
| omitBodyDuringUpdate: true | |
| omitNameDuringUpdate: true | |
| omitPrereleaseDuringUpdate: true | |
| replacesArtifacts: true | |
| artifacts: "coverage.json" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload Build Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: release-${{ env.PKG_VERSION }} | |
| path: /home/runner/work/release/release.tar.gz | |
| # ----------------------------------------------------------------- | |
| # NOTIFY | |
| # ----------------------------------------------------------------- | |
| notify: | |
| name: Notify | |
| if: ${{ always() }} | |
| needs: [prepare, tests-python, tests-playwright, tests-playwright-legacy, release] | |
| runs-on: ubuntu-latest | |
| env: | |
| PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} | |
| steps: | |
| - name: Notify on Slack (Success) | |
| if: ${{ !contains(join(needs.*.result, ','), 'failure') }} | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} | |
| payload: | | |
| { | |
| "text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>", | |
| "attachments": [ | |
| { | |
| "color": "28a745", | |
| "fields": [ | |
| { | |
| "title": "Status", | |
| "short": true, | |
| "value": "Completed" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} | |
| - name: Notify on Slack (Failure) | |
| if: ${{ contains(join(needs.*.result, ','), 'failure') }} | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} | |
| payload: | | |
| { | |
| "text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>", | |
| "attachments": [ | |
| { | |
| "color": "a82929", | |
| "fields": [ | |
| { | |
| "title": "Status", | |
| "short": true, | |
| "value": "Failed" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} | |
| # ----------------------------------------------------------------- | |
| # SANDBOX | |
| # ----------------------------------------------------------------- | |
| sandbox: | |
| name: Deploy to Sandbox | |
| if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }} | |
| needs: [prepare, release] | |
| runs-on: [self-hosted, dev-server] | |
| environment: | |
| name: sandbox | |
| env: | |
| PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download a Release Artifact | |
| uses: actions/[email protected] | |
| with: | |
| name: release-${{ env.PKG_VERSION }} | |
| - name: Deploy to containers | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| echo "Reset production flags in settings.py..." | |
| sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py | |
| echo "Install Deploy to Container CLI dependencies..." | |
| cd dev/deploy-to-container | |
| npm ci | |
| cd ../.. | |
| echo "Start Deploy..." | |
| node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} | |
| - name: Cleanup old docker resources | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| docker image prune -a -f | |
| legacySandbox: | |
| name: Deploy to Legacy Sandbox | |
| if: ${{ !failure() && !cancelled() && github.event.inputs.legacySandbox == 'true' }} | |
| needs: [prepare, release] | |
| runs-on: [self-hosted, legacy-sandbox-server] | |
| environment: | |
| name: legacy-sandbox | |
| url: "https://sandbox.ietf.org" | |
| env: | |
| PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} | |
| steps: | |
| - name: Download a Release Artifact | |
| uses: actions/[email protected] | |
| with: | |
| name: release-${{ env.PKG_VERSION }} | |
| path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} | |
| - name: Extract Release | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} | |
| run: | | |
| echo "Extracting release tarball..." | |
| tar xzf release.tar.gz | |
| echo "Deleting release tarball..." | |
| rm -rf release.tar.gz | |
| - name: Setup Environment | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} | |
| run: | | |
| echo "Copying settings from previous deploy..." | |
| cp ../web/ietf/settings_local.py ietf/ | |
| rsync -a ../web/test/ test/ | |
| echo "Installing Python dependencies..." | |
| python3.9 -mvenv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| pip freeze > frozen-requirements.txt | |
| echo "Collecting static..." | |
| ietf/manage.py collectstatic | |
| echo "Running checks..." | |
| ietf/manage.py check | |
| - name: Update Docker Containers | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| working-directory: /a/docker/datatracker | |
| run: | | |
| echo "Pulling latest docker images..." | |
| docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback | |
| docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback | |
| docker-compose pull | |
| # echo "Shutting down containers..." | |
| # docker-compose down -t 300 | |