add workflow: fetch_sra_bams_for_genbank_accession #1469
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "viral-pipelines CI" | |
| on: | |
| push: | |
| pull_request: | |
| branches: | |
| - master | |
| release: | |
| types: | |
| - created | |
| env: | |
| PYTHONIOENCODING: UTF8 | |
| DOCKER_REGISTRY: "quay.io" | |
| DOCKER_REPO_PROD: "quay.io/broadinstitute/viral-pipelines" | |
| DOCKER_REPO_DEV: "quay.io/broadinstitute/viral-pipelines-dev" | |
| MODULE_VERSIONS: "requirements-modules.txt" | |
| # TravisCI variables described here: | |
| # https://docs.travis-ci.com/user/environment-variables/ | |
| # GitHub Actions environment variables and context described here: | |
| # https://docs.github.com/en/actions/reference/environment-variables | |
| # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#env-context | |
| GITHUB_ACTIONS_COMMIT: ${{ github.sha }} | |
| GITHUB_ACTIONS_BUILD_DIR: ${{ github.workspace }} | |
| GITHUB_ACTIONS_BRANCH: ${{ github.ref }} | |
| GITHUB_ACTIONS_PULL_REQUEST: ${{ github.event.number }} | |
| GITHUB_ACTIONS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} | |
| GITHUB_ACTIONS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }} | |
| GITHUB_ACTIONS_BASE_REF: ${{ github.base_ref }} | |
| jobs: | |
| validate_wdl_miniwdl: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| # fetch git tags (tagged releases) because | |
| # actions/checkout@v2 does either a full checkout or a shallow checkout without tags | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -e -x | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| # if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build | |
| if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| # if event_name=="release", this is a tagged build | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| else | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| - name: List environment | |
| run: | | |
| env | |
| - name: install python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| - name: install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install shellcheck | |
| - name: install miniwdl | |
| run: | | |
| pip3 -q install miniwdl | |
| - name: run miniwdl checks | |
| shell: bash | |
| run: | | |
| set -e | |
| github_actions_ci/check-wdl-runtimes.sh | |
| miniwdl check pipes/WDL/workflows/*.wdl | |
| validate_wdl_womtool: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| # fetch git tags (tagged releases) because | |
| # actions/checkout@v2 does either a full checkout or a shallow checkout without tags | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -e -x | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| # if event_name=="release", this is a tagged build | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| else | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| - name: install java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '11' | |
| - name: install cromwell+womtool+dxWDL | |
| shell: bash | |
| run: | | |
| github_actions_ci/install-wdl.sh | |
| - name: run womtool checks | |
| shell: bash | |
| run: | | |
| set -e | |
| github_actions_ci/check-wdl-runtimes.sh | |
| github_actions_ci/validate-wdl-womtool.sh | |
| ## note: this test_docs job does not actually produce the output on readthedocs | |
| ## readthedocs does its own build trigger. this job exists simply to | |
| ## attempt a build and alert us of documentation build failures because otherwise we would never know. | |
| test_docs: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| # fetch git tags (tagged releases) because | |
| # actions/checkout@v2 does either a full checkout or a shallow checkout without tags | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -e -x | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| # if event_name=="release", this is a tagged build | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| else | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| - name: install python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| - name: install docs dependencies | |
| shell: bash | |
| run: | | |
| github_actions_ci/install-pip-docs.sh | |
| - name: test docs build | |
| shell: bash | |
| run: | | |
| github_actions_ci/build-docs.sh | |
| test_cromwell: | |
| needs: validate_wdl_womtool | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| # fetch git tags (tagged releases) because | |
| # actions/checkout@v2 does either a full checkout or a shallow checkout without tags | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -ex | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| # if event_name=="release", this is a tagged build | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| else | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| - name: install java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '11' | |
| - name: install cromwell+womtool+dxWDL | |
| shell: bash | |
| run: | | |
| github_actions_ci/install-wdl.sh | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: test with cromwell | |
| shell: bash | |
| run: | | |
| set -e | |
| github_actions_ci/relative-wdl-paths.sh | |
| github_actions_ci/tests-cromwell.sh | |
| test_miniwdl: | |
| needs: validate_wdl_miniwdl | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| # fetch git tags (tagged releases) because | |
| # actions/checkout@v2 does either a full checkout or a shallow checkout without tags | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -ex | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| # if event_name=="release", this is a tagged build | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| else | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| - name: install python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| - name: install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install jq | |
| - name: install miniwdl | |
| run: | | |
| pip3 install miniwdl docker[tls] six | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: test with miniwdl | |
| shell: bash | |
| run: | | |
| github_actions_ci/tests-miniwdl.sh | |
| deploy_dnanexus: | |
| #if: ${{ github.event.ref == 'dnanexus' }} | |
| if: github.event_name == 'release' || github.ref == 'refs/heads/master' | |
| needs: | |
| - validate_wdl_womtool | |
| - validate_wdl_miniwdl | |
| runs-on: ubuntu-20.04 | |
| env: | |
| # DX_API_TOKEN (for DNAnexus builds) -- GitHub Actions token #1, expires 2025-01-17 | |
| # stored here: https://github.com/broadinstitute/viral-pipelines/settings/secrets/actions | |
| DX_API_TOKEN: ${{ secrets.DX_API_TOKEN }} | |
| DX_PROJECT: project-F8PQ6380xf5bK0Qk0YPjB17P | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| # fetch git tags (tagged releases) because | |
| # actions/checkout@v2 does either a full checkout or a shallow checkout without tags | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -e -x | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| # if event_name=="release", this is a tagged build | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| else | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| - name: list GH event info | |
| run: | | |
| echo "${{ github.event_name }}" | |
| echo "${{ github.event.action }}" | |
| echo "${{ github.event.pull_request.merged }}" | |
| - name: install python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| - name: install java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '11' | |
| - name: install miniwdl | |
| run: | | |
| pip3 install pyyaml dxpy | |
| - name: install dx toolkit | |
| shell: bash | |
| run: | | |
| github_actions_ci/install-wdl.sh | |
| - name: | |
| shell: bash | |
| run: | | |
| github_actions_ci/relative-wdl-paths.sh | |
| github_actions_ci/build-dx.sh | |
| github_actions_ci/tests-dx.sh |