TorchBench Nightly Docker Build #491
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: TorchBench nightly docker build | |
| on: | |
| schedule: | |
| # Push the nightly docker daily at 3 PM UTC | |
| - cron: '0 15 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| nightly_date: | |
| description: "PyTorch nightly version" | |
| required: false | |
| env: | |
| WITH_PUSH: "true" | |
| CONDA_ENV: "torchbench" | |
| DOCKER_IMAGE: "ghcr.io/pytorch/torchbench:latest" | |
| SETUP_SCRIPT: "/workspace/setup_instance.sh" | |
| HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} | |
| jobs: | |
| build-push-docker: | |
| if: ${{ github.repository_owner == 'pytorch' }} | |
| runs-on: [self-hosted, linux.4xlarge] | |
| environment: docker-s3-upload | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| path: benchmark | |
| - name: Login to GitHub Container Registry | |
| if: ${{ env.WITH_PUSH == 'true' }} | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: pytorch | |
| password: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} | |
| - name: Build TorchBench nightly docker | |
| run: | | |
| export NIGHTLY_DATE="${{ github.event.inputs.nightly_date }}" | |
| if [ -z "${NIGHTLY_DATE}" ]; then | |
| export TODAY=$(date +'%Y%m%d') | |
| export DOCKER_TAG=dev${TODAY} | |
| else | |
| export DOCKER_TAG=dev${NIGHTLY_DATE} | |
| fi | |
| cd benchmark/docker | |
| full_ref="${{ github.ref }}" | |
| prefix="refs/heads/" | |
| branch_name=${full_ref#$prefix} | |
| docker build . --build-arg TORCHBENCH_BRANCH="${branch_name}" --build-arg FORCE_DATE="${NIGHTLY_DATE}" \ | |
| -f torchbench-nightly.dockerfile -t ghcr.io/pytorch/torchbench:${DOCKER_TAG} | |
| docker tag ghcr.io/pytorch/torchbench:${DOCKER_TAG} ghcr.io/pytorch/torchbench:latest | |
| - name: Push docker to remote | |
| if: ${{ env.WITH_PUSH == 'true' }} | |
| run: | | |
| export TODAY=$(date +'%Y%m%d') | |
| export DOCKER_TAG=dev${TODAY} | |
| docker push ghcr.io/pytorch/torchbench:${DOCKER_TAG} | |
| docker push ghcr.io/pytorch/torchbench:latest | |
| test-docker: | |
| if: ${{ github.repository_owner == 'pytorch' }} | |
| runs-on: [self-hosted, linux.g5.4xlarge.nvidia.gpu] | |
| needs: build-push-docker | |
| environment: docker-s3-upload | |
| steps: | |
| - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" | |
| uses: pytorch/test-infra/.github/actions/setup-ssh@main | |
| with: | |
| github-secret: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| path: benchmark | |
| - name: Pull docker image | |
| uses: pytorch/test-infra/.github/actions/pull-docker-image@main | |
| with: | |
| docker-image: ${{ env.DOCKER_IMAGE }} | |
| - name: Install NVIDIA Driver, docker runtime, set GPU_FLAG | |
| id: install-nvidia-driver | |
| uses: pytorch/test-infra/.github/actions/setup-nvidia@main | |
| - name: Install and Test TorchBench Docker | |
| run: | | |
| container_name=$(docker run \ | |
| -e CONDA_ENV="${CONDA_ENV}" \ | |
| -e SETUP_SCRIPT="${SETUP_SCRIPT}" \ | |
| -e HUGGING_FACE_HUB_TOKEN="${HUGGING_FACE_HUB_TOKEN}" \ | |
| --tty \ | |
| --detach \ | |
| --shm-size=32gb \ | |
| -v "${PWD}/benchmark:/benchmark" \ | |
| --gpus all \ | |
| -w / \ | |
| "${{ env.DOCKER_IMAGE }}" \ | |
| tail -f /dev/null | |
| ) | |
| echo "Container name: ${container_name}" | |
| docker exec -t -w "/" "${container_name}" bash -c "sudo chown -R runner /benchmark; sudo chgrp -R runner /benchmark" | |
| docker exec -t -w "/benchmark" "${container_name}" bash /benchmark/.ci/tritonbench/test.sh | |
| - name: Teardown Linux | |
| uses: pytorch/test-infra/.github/actions/teardown-linux@main | |
| if: always() | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} | |
| cancel-in-progress: true |