[kineto] populate src/dst rank for p2p (#130812) #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # @generated DO NOT EDIT MANUALLY | |
| # Template is at: .github/templates/windows_binary_build_workflow.yml.j2 | |
| # Generation script: .github/scripts/generate_ci_workflows.py | |
| name: windows-binary-libtorch-release | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| env: | |
| # Needed for conda builds | |
| ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" | |
| ANACONDA_USER: pytorch | |
| AWS_DEFAULT_REGION: us-east-1 | |
| BUILD_ENVIRONMENT: windows-binary-libtorch-release | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| SHA1: ${{ github.event.pull_request.head.sha || github.sha }} | |
| SKIP_ALL_TESTS: 1 | |
| concurrency: | |
| group: windows-binary-libtorch-release-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} | |
| cancel-in-progress: true | |
| jobs: | |
| libtorch-cpu-shared-with-deps-release-build: | |
| if: ${{ github.repository_owner == 'pytorch' }} | |
| runs-on: windows.4xlarge.nonephemeral | |
| timeout-minutes: 240 | |
| env: | |
| PYTORCH_ROOT: ${{ github.workspace }}/pytorch | |
| BUILDER_ROOT: ${{ github.workspace }}/builder | |
| PACKAGE_TYPE: libtorch | |
| # TODO: This is a legacy variable that we eventually want to get rid of in | |
| # favor of GPU_ARCH_VERSION | |
| DESIRED_CUDA: cpu | |
| GPU_ARCH_TYPE: cpu | |
| SKIP_ALL_TESTS: 1 | |
| LIBTORCH_CONFIG: release | |
| LIBTORCH_VARIANT: shared-with-deps | |
| # This is a dummy value for libtorch to work correctly with our batch scripts | |
| # without this value pip does not get installed for some reason | |
| DESIRED_PYTHON: "3.8" | |
| steps: | |
| - name: Display EC2 information | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| function get_ec2_metadata() { | |
| # Pulled from instance metadata endpoint for EC2 | |
| # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html | |
| category=$1 | |
| curl -fsSL "http://169.254.169.254/latest/meta-data/${category}" | |
| } | |
| echo "ami-id: $(get_ec2_metadata ami-id)" | |
| echo "instance-id: $(get_ec2_metadata instance-id)" | |
| echo "instance-type: $(get_ec2_metadata instance-type)" | |
| echo "system info $(uname -a)" | |
| - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" | |
| uses: pytorch/test-infra/.github/actions/setup-ssh@main | |
| continue-on-error: true | |
| with: | |
| github-secret: ${{ secrets.GITHUB_TOKEN }} | |
| # Needed for binary builds, see: https://github.com/pytorch/pytorch/issues/73339#issuecomment-1058981560 | |
| - name: Enable long paths on Windows | |
| shell: powershell | |
| run: | | |
| Set-ItemProperty -Path "HKLM:\\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 | |
| # Since it's just a defensive command, the workflow should continue even the command fails. This step can be | |
| # removed once Windows Defender is removed from the AMI | |
| - name: Disables Windows Defender scheduled and real-time scanning for files in directories used by PyTorch | |
| continue-on-error: true | |
| shell: powershell | |
| run: | | |
| Add-MpPreference -ExclusionPath $(Get-Location).tostring(),$Env:TEMP -ErrorAction Ignore | |
| # Let's both exclude the path and disable Windows Defender completely just to be sure | |
| # that it doesn't interfere | |
| Set-MpPreference -DisableRealtimeMonitoring $True -ErrorAction Ignore | |
| # NOTE: These environment variables are put here so that they can be applied on every job equally | |
| # They are also here because setting them at a workflow level doesn't give us access to the | |
| # runner.temp variable, which we need. | |
| - name: Populate binary env | |
| shell: bash | |
| run: | | |
| echo "BINARY_ENV_FILE=${RUNNER_TEMP}/env" >> "${GITHUB_ENV}" | |
| echo "PYTORCH_FINAL_PACKAGE_DIR=${RUNNER_TEMP}/artifacts" >> "${GITHUB_ENV}" | |
| echo "WIN_PACKAGE_WORK_DIR=${RUNNER_TEMP}" | |
| - name: Checkout PyTorch | |
| uses: malfet/checkout@silent-checkout | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| submodules: recursive | |
| path: pytorch | |
| quiet-checkout: true | |
| - name: Clean PyTorch checkout | |
| run: | | |
| # Remove any artifacts from the previous checkouts | |
| git clean -fxd | |
| working-directory: pytorch | |
| - name: Checkout pytorch/builder | |
| uses: malfet/checkout@silent-checkout | |
| with: | |
| ref: main | |
| submodules: recursive | |
| repository: pytorch/builder | |
| path: builder | |
| quiet-checkout: true | |
| - name: Clean pytorch/builder checkout | |
| run: | | |
| # Remove any artifacts from the previous checkouts | |
| git clean -fxd | |
| working-directory: builder | |
| - name: Populate binary env | |
| shell: bash | |
| run: | | |
| "${PYTORCH_ROOT}/.circleci/scripts/binary_populate_env.sh" | |
| - name: Build PyTorch binary | |
| shell: bash | |
| run: | | |
| "${PYTORCH_ROOT}/.circleci/scripts/binary_windows_build.sh" | |
| - uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: libtorch-cpu-shared-with-deps-release | |
| retention-days: 14 | |
| if-no-files-found: error | |
| path: "${{ env.PYTORCH_FINAL_PACKAGE_DIR }}" | |
| - name: Wait until all sessions have drained | |
| shell: powershell | |
| working-directory: pytorch | |
| if: always() | |
| timeout-minutes: 120 | |
| run: | | |
| .github\scripts\wait_for_ssh_to_drain.ps1 | |
| - name: Kill active ssh sessions if still around (Useful if workflow was cancelled) | |
| shell: powershell | |
| working-directory: pytorch | |
| if: always() | |
| run: | | |
| .github\scripts\kill_active_ssh_sessions.ps1 | |
| libtorch-cpu-shared-with-deps-release-test: # Testing | |
| if: ${{ github.repository_owner == 'pytorch' }} | |
| needs: libtorch-cpu-shared-with-deps-release-build | |
| runs-on: windows.4xlarge.nonephemeral | |
| timeout-minutes: 240 | |
| env: | |
| PYTORCH_ROOT: ${{ github.workspace }}/pytorch | |
| BUILDER_ROOT: ${{ github.workspace }}/builder | |
| PACKAGE_TYPE: libtorch | |
| # TODO: This is a legacy variable that we eventually want to get rid of in | |
| # favor of GPU_ARCH_VERSION | |
| DESIRED_CUDA: cpu | |
| GPU_ARCH_TYPE: cpu | |
| SKIP_ALL_TESTS: 1 | |
| LIBTORCH_CONFIG: release | |
| LIBTORCH_VARIANT: shared-with-deps | |
| # This is a dummy value for libtorch to work correctly with our batch scripts | |
| # without this value pip does not get installed for some reason | |
| DESIRED_PYTHON: "3.8" | |
| steps: | |
| - name: Display EC2 information | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| function get_ec2_metadata() { | |
| # Pulled from instance metadata endpoint for EC2 | |
| # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html | |
| category=$1 | |
| curl -fsSL "http://169.254.169.254/latest/meta-data/${category}" | |
| } | |
| echo "ami-id: $(get_ec2_metadata ami-id)" | |
| echo "instance-id: $(get_ec2_metadata instance-id)" | |
| echo "instance-type: $(get_ec2_metadata instance-type)" | |
| echo "system info $(uname -a)" | |
| - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" | |
| uses: pytorch/test-infra/.github/actions/setup-ssh@main | |
| continue-on-error: true | |
| with: | |
| github-secret: ${{ secrets.GITHUB_TOKEN }} | |
| # Needed for binary builds, see: https://github.com/pytorch/pytorch/issues/73339#issuecomment-1058981560 | |
| - name: Enable long paths on Windows | |
| shell: powershell | |
| run: | | |
| Set-ItemProperty -Path "HKLM:\\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 | |
| # Since it's just a defensive command, the workflow should continue even the command fails. This step can be | |
| # removed once Windows Defender is removed from the AMI | |
| - name: Disables Windows Defender scheduled and real-time scanning for files in directories used by PyTorch | |
| continue-on-error: true | |
| shell: powershell | |
| run: | | |
| Add-MpPreference -ExclusionPath $(Get-Location).tostring(),$Env:TEMP -ErrorAction Ignore | |
| # Let's both exclude the path and disable Windows Defender completely just to be sure | |
| # that it doesn't interfere | |
| Set-MpPreference -DisableRealtimeMonitoring $True -ErrorAction Ignore | |
| # NOTE: These environment variables are put here so that they can be applied on every job equally | |
| # They are also here because setting them at a workflow level doesn't give us access to the | |
| # runner.temp variable, which we need. | |
| - name: Populate binary env | |
| shell: bash | |
| run: | | |
| echo "BINARY_ENV_FILE=${RUNNER_TEMP}/env" >> "${GITHUB_ENV}" | |
| echo "PYTORCH_FINAL_PACKAGE_DIR=${RUNNER_TEMP}/artifacts" >> "${GITHUB_ENV}" | |
| echo "WIN_PACKAGE_WORK_DIR=${RUNNER_TEMP}" | |
| - uses: actions/download-artifact@v3 | |
| name: Download Build Artifacts | |
| with: | |
| name: libtorch-cpu-shared-with-deps-release | |
| path: "${{ env.PYTORCH_FINAL_PACKAGE_DIR }}" | |
| - name: Checkout PyTorch | |
| uses: malfet/checkout@silent-checkout | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| submodules: recursive | |
| path: pytorch | |
| quiet-checkout: true | |
| - name: Clean PyTorch checkout | |
| run: | | |
| # Remove any artifacts from the previous checkouts | |
| git clean -fxd | |
| working-directory: pytorch | |
| - name: Checkout pytorch/builder | |
| uses: malfet/checkout@silent-checkout | |
| with: | |
| ref: main | |
| submodules: recursive | |
| repository: pytorch/builder | |
| path: builder | |
| quiet-checkout: true | |
| - name: Clean pytorch/builder checkout | |
| run: | | |
| # Remove any artifacts from the previous checkouts | |
| git clean -fxd | |
| working-directory: builder | |
| - name: Populate binary env | |
| shell: bash | |
| run: | | |
| "${PYTORCH_ROOT}/.circleci/scripts/binary_populate_env.sh" | |
| - name: Test PyTorch binary | |
| shell: bash | |
| run: | | |
| "${PYTORCH_ROOT}/.circleci/scripts/binary_windows_test.sh" | |
| - name: Wait until all sessions have drained | |
| shell: powershell | |
| working-directory: pytorch | |
| if: always() | |
| timeout-minutes: 120 | |
| run: | | |
| .github\scripts\wait_for_ssh_to_drain.ps1 | |
| - name: Kill active ssh sessions if still around (Useful if workflow was cancelled) | |
| shell: powershell | |
| working-directory: pytorch | |
| if: always() | |
| run: | | |
| .github\scripts\kill_active_ssh_sessions.ps1 |