From a1322420cd0c70e2e171a631e7282bcb2c051cda Mon Sep 17 00:00:00 2001 From: Oliver Koenig Date: Mon, 22 Apr 2024 14:38:41 +0200 Subject: [PATCH] ci: Refactor `build_base` to action --- .github/actions/build_base/action.yaml | 157 ++++++++++++++++++++++ .github/workflows/_build_base.yaml | 178 ------------------------- .github/workflows/_ci.yaml | 25 +++- .github/workflows/_sandbox.yaml | 25 +++- 4 files changed, 195 insertions(+), 190 deletions(-) create mode 100644 .github/actions/build_base/action.yaml delete mode 100644 .github/workflows/_build_base.yaml diff --git a/.github/actions/build_base/action.yaml b/.github/actions/build_base/action.yaml new file mode 100644 index 000000000..da4f0ece8 --- /dev/null +++ b/.github/actions/build_base/action.yaml @@ -0,0 +1,157 @@ +name: ~build CUDA+Python base container +description: Builds the base container + +inputs: + ARCHITECTURE: + description: "CPU architecture to build the image for, e.g. amd64, arm64" + required: true + BASE_IMAGE: + description: "Base CUDA image, e.g. nvidia/cuda:X.Y.Z-devel-ubuntu22.04" + required: false + default: latest + BUILD_DATE: + description: "Build date in YYYY-MM-DD format" + required: false + default: "NOT SPECIFIED" + ARTIFACT_NAME: + description: "Name of the artifact zip file" + required: false + default: "artifact-base-build" + BADGE_FILENAME: + description: "Name of the endpoint JSON file for shields.io badge" + required: false + default: "badge-base-build" + GIT_USER_NAME: + description: "Username in GIT to perform git pull/push" + required: false + default: "JAX Toolbox" + GIT_USER_EMAIL: + description: "User email in GIT to perform git pull/push" + required: false + default: "jax@nvidia.com" + MANIFEST_ARTIFACT_NAME: + description: Artifact name in current run w/ manifest/patches. Leaving empty uses manifest/patches in current branch + default: "" + required: false + +outputs: + DOCKER_TAG: + description: "Tag of the image built" + value: ${{ steps.meta.outputs.tags }} + +runs: + using: composite + steps: + - name: Print environment variables + shell: bash + run: env + + - name: Check out the repository under ${GITHUB_WORKSPACE} + uses: actions/checkout@v4 + + - name: Delete checked-out manifest and patches + if: inputs.MANIFEST_ARTIFACT_NAME != '' + shell: bash + run: | + rm .github/container/manifest.yaml + rm -rf .github/container/patches + + - name: Replace checked-out manifest file/patches with bumped one + if: inputs.MANIFEST_ARTIFACT_NAME != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.MANIFEST_ARTIFACT_NAME }} + path: .github/container/ + + - name: Log the changes in manifest file and patch folder + working-directory: .github/container + shell: bash -x -e {0} + run: | + ls -lah + ls -lah patches + echo "Changes in manifest file and patch folder" + git diff + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ env.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + image=moby/buildkit:v0.12.1 + + - name: Set docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.UPLD_IMAGE }} + flavor: | + latest=false + tags: | + type=raw,value=${{ github.run_id }}-base-${{ inputs.ARCHITECTURE }} + labels: org.opencontainers.image.created=${{ inputs.BUILD_DATE }} + + - name: Build docker images + id: build + uses: docker/build-push-action@v5 + with: + context: .github/container + push: true + file: .github/container/Dockerfile.base + platforms: linux/${{ inputs.ARCHITECTURE }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_USER_NAME=${{ inputs.GIT_USER_NAME }} + GIT_USER_EMAIL=${{ inputs.GIT_USER_EMAIL }} + BUILD_DATE=${{ inputs.BUILD_DATE }} + ${{ inputs.BASE_IMAGE != 'latest' && format('BASE_IMAGE={0}', inputs.BASE_IMAGE) || '' }} + + - name: Generate sitrep + if: "!cancelled()" + shell: bash -x -e {0} + run: | + # bring in utility functions + source .github/workflows/scripts/to_json.sh + + badge_label='Base image ${{ inputs.ARCHITECTURE }} build' + tags="${{ steps.meta.outputs.tags }}" + digest="${{ steps.build.outputs.digest }}" + outcome="${{ steps.build.outcome }}" + + if [[ ${outcome} == "success" ]]; then + badge_message="pass" + badge_color=brightgreen + summary="Base image build on ${{ inputs.ARCHITECTURE }}: $badge_message" + else + badge_message="fail" + badge_color=red + summary="Base image build on ${{ inputs.ARCHITECTURE }}: $badge_message" + fi + + to_json \ + summary \ + badge_label tags digest outcome \ + > sitrep.json + + schemaVersion=1 \ + label="${badge_label}" \ + message="${badge_message}" \ + color="${badge_color}" \ + to_json schemaVersion label message color \ + > ${{ env.BADGE_FILENAME_FULL }} + + - name: Upload sitrep and badge + if: "!cancelled()" + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.ARTIFACT_NAME }}-${{ inputs.ARCHITECTURE }} + path: | + sitrep.json + ${{ env.BADGE_FILENAME_FULL }} diff --git a/.github/workflows/_build_base.yaml b/.github/workflows/_build_base.yaml deleted file mode 100644 index b575ec14b..000000000 --- a/.github/workflows/_build_base.yaml +++ /dev/null @@ -1,178 +0,0 @@ -name: ~build CUDA+Python base container - -on: - workflow_call: - inputs: - ARCHITECTURE: - type: string - description: 'CPU architecture to build the image for, e.g. amd64, arm64' - required: true - BASE_IMAGE: - type: string - description: 'Base CUDA image, e.g. nvidia/cuda:X.Y.Z-devel-ubuntu22.04' - required: false - default: latest - BUILD_DATE: - type: string - description: "Build date in YYYY-MM-DD format" - required: false - default: 'NOT SPECIFIED' - ARTIFACT_NAME: - type: string - description: 'Name of the artifact zip file' - required: false - default: 'artifact-base-build' - BADGE_FILENAME: - type: string - description: 'Name of the endpoint JSON file for shields.io badge' - required: false - default: 'badge-base-build' - GIT_USER_NAME: - type: string - description: 'Username in GIT to perform git pull/push' - required: false - default: 'JAX Toolbox' - GIT_USER_EMAIL: - type: string - description: 'User email in GIT to perform git pull/push' - required: false - default: 'jax@nvidia.com' - MANIFEST_ARTIFACT_NAME: - type: string - description: Artifact name in current run w/ manifest/patches. Leaving empty uses manifest/patches in current branch - default: '' - required: false - outputs: - DOCKER_TAG: - description: "Tag of the image built" - value: ${{ jobs.build-base.outputs.DOCKER_TAG }} - -env: - UPLD_IMAGE: ghcr.io/nvidia/jax-toolbox-internal - -permissions: - contents: read # to fetch code - actions: write # to cancel previous workflows - packages: write # to upload container - -jobs: - - build-base: - runs-on: [self-hosted, "${{ inputs.ARCHITECTURE }}", small] - env: - BADGE_FILENAME_FULL: ${{ inputs.BADGE_FILENAME }}-${{ inputs.ARCHITECTURE }}.json - outputs: - DOCKER_TAG: ${{ steps.meta.outputs.tags }} - steps: - - name: Print environment variables - run: env - - - name: Check out the repository under ${GITHUB_WORKSPACE} - uses: actions/checkout@v4 - - - name: Delete checked-out manifest and patches - if: inputs.MANIFEST_ARTIFACT_NAME != '' - run: | - rm .github/container/manifest.yaml - rm -rf .github/container/patches - - - name: Replace checked-out manifest file/patches with bumped one - if: inputs.MANIFEST_ARTIFACT_NAME != '' - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.MANIFEST_ARTIFACT_NAME }} - path: .github/container/ - - - name: Log the changes in manifest file and patch folder - working-directory: .github/container - shell: bash -x -e {0} - run: | - ls -lah - ls -lah patches - echo "Changes in manifest file and patch folder" - git diff - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver-opts: | - image=moby/buildkit:v0.12.1 - - - name: Set docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.UPLD_IMAGE }} - flavor: | - latest=false - tags: | - type=raw,value=${{ github.run_id }}-base-${{ inputs.ARCHITECTURE }} - labels: - org.opencontainers.image.created=${{ inputs.BUILD_DATE }} - - - name: Build docker images - id: build - uses: docker/build-push-action@v5 - with: - context: .github/container - push: true - file: .github/container/Dockerfile.base - platforms: linux/${{ inputs.ARCHITECTURE }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - GIT_USER_NAME=${{ inputs.GIT_USER_NAME }} - GIT_USER_EMAIL=${{ inputs.GIT_USER_EMAIL }} - BUILD_DATE=${{ inputs.BUILD_DATE }} - ${{ inputs.BASE_IMAGE != 'latest' && format('BASE_IMAGE={0}', inputs.BASE_IMAGE) || '' }} - - - name: Generate sitrep - if: "!cancelled()" - shell: bash -x -e {0} - run: | - # bring in utility functions - source .github/workflows/scripts/to_json.sh - - badge_label='Base image ${{ inputs.ARCHITECTURE }} build' - tags="${{ steps.meta.outputs.tags }}" - digest="${{ steps.build.outputs.digest }}" - outcome="${{ steps.build.outcome }}" - - if [[ ${outcome} == "success" ]]; then - badge_message="pass" - badge_color=brightgreen - summary="Base image build on ${{ inputs.ARCHITECTURE }}: $badge_message" - else - badge_message="fail" - badge_color=red - summary="Base image build on ${{ inputs.ARCHITECTURE }}: $badge_message" - fi - - to_json \ - summary \ - badge_label tags digest outcome \ - > sitrep.json - - schemaVersion=1 \ - label="${badge_label}" \ - message="${badge_message}" \ - color="${badge_color}" \ - to_json schemaVersion label message color \ - > ${{ env.BADGE_FILENAME_FULL }} - - - name: Upload sitrep and badge - if: "!cancelled()" - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.ARTIFACT_NAME }}-${{ inputs.ARCHITECTURE }} - path: | - sitrep.json - ${{ env.BADGE_FILENAME_FULL }} diff --git a/.github/workflows/_ci.yaml b/.github/workflows/_ci.yaml index 8d7962ece..01b06dcc3 100644 --- a/.github/workflows/_ci.yaml +++ b/.github/workflows/_ci.yaml @@ -28,12 +28,25 @@ permissions: jobs: build-base: - uses: ./.github/workflows/_build_base.yaml - with: - ARCHITECTURE: ${{ inputs.ARCHITECTURE }} - BUILD_DATE: ${{ inputs.BUILD_DATE }} - MANIFEST_ARTIFACT_NAME: ${{ inputs.MANIFEST_ARTIFACT_NAME }} - secrets: inherit + runs-on: [self-hosted, "${{ inputs.ARCHITECTURE }}", small] + env: + UPLD_IMAGE: ghcr.io/nvidia/jax-toolbox-internal + BADGE_FILENAME_FULL: ${{ inputs.BADGE_FILENAME }}-${{ inputs.ARCHITECTURE }}.json + outputs: + DOCKER_TAG: ${{ steps.main.outputs.DOCKER_TAG }} + steps: + - name: Check out the repository under ${GITHUB_WORKSPACE} + uses: actions/checkout@v4 + + - name: Main + id: main + uses: ./github/actions/build_base + with: + ARCHITECTURE: ${{ inputs.ARCHITECTURE }} + BUILD_DATE: ${{ inputs.BUILD_DATE }} + MANIFEST_ARTIFACT_NAME: ${{ inputs.MANIFEST_ARTIFACT_NAME }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build-jax: needs: build-base diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 78ccced41..43ebed9ee 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -104,13 +104,26 @@ jobs: .github/container/patches build-base: - uses: ./.github/workflows/_build_base.yaml + runs-on: [self-hosted, "amd64", small] needs: [metadata, bump-manifest] - with: - ARCHITECTURE: amd64 - BUILD_DATE: 20240418 - MANIFEST_ARTIFACT_NAME: ${{ needs.metadata.outputs.MANIFEST_ARTIFACT_NAME }} - secrets: inherit + env: + UPLD_IMAGE: ghcr.io/nvidia/jax-toolbox-internal + BADGE_FILENAME_FULL: ${{ inputs.BADGE_FILENAME }}-${{ inputs.ARCHITECTURE }}.json + outputs: + DOCKER_TAG: ${{ steps.main.outputs.DOCKER_TAG }} + steps: + - name: Check out the repository under ${GITHUB_WORKSPACE} + uses: actions/checkout@v4 + + - name: Main + id: main + uses: ./.github/actions/build_base + with: + ARCHITECTURE: amd64 + BUILD_DATE: 20240418 + MANIFEST_ARTIFACT_NAME: ${{ needs.metadata.outputs.MANIFEST_ARTIFACT_NAME }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build-jax: needs: build-base