From a08d9a90e54e91596ddb96188267a9eb0b98780f Mon Sep 17 00:00:00 2001 From: mwestphall Date: Tue, 24 Sep 2024 09:19:35 -0500 Subject: [PATCH 1/3] OSPOOL-131: Support ARM Refactor the build matrix to include platform as an argument. Add a second job to consolidate multi-platform artifacts under a single tag in the registry. See the following documentation for a reference implementation: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners --- .github/actions/push-digest-local/action.yaml | 100 ++++++++++ .github/workflows/build-container.yml | 173 ++++++++++++++---- 2 files changed, 237 insertions(+), 36 deletions(-) create mode 100644 .github/actions/push-digest-local/action.yaml diff --git a/.github/actions/push-digest-local/action.yaml b/.github/actions/push-digest-local/action.yaml new file mode 100644 index 0000000..f74ba9f --- /dev/null +++ b/.github/actions/push-digest-local/action.yaml @@ -0,0 +1,100 @@ +name: 'Push Container by Digest Action' +description: 'Pushes an image by digest to a given registry, then outputs its digest and tags as an artifact' + + + +inputs: + registry: + required: true + default: '' + username: + required: true + default: '' + password: + required: true + default: '' + osg_series: + required: true + default: '' + osg_repo: + required: true + default: '' + base_os: + required: true + default: '' + base_tag: + required: true + default: '' + platform: + required: false + default: 'linux/amd64' + timestamp: + required: false + default: '' + output_image: + required: false + default: '' + +runs: + using: "composite" + steps: + - uses: actions/checkout@v3 + + - id: slash-escape + shell: bash + run: | + platform=${{ inputs.platform }} + echo "platform=${platform//\//-}" >> ${GITHUB_OUTPUT} + + - name: Registry login + # if: >- + # github.ref == 'refs/heads/master' && + # github.event_name != 'pull_request' && + # github.repository_owner == 'opensciencegrid' + uses: docker/login-action@v2 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ inputs.password }} + + - id: upload-image + uses: opensciencegrid/build-container-action@420b64817b8a5d296a4201af6e955c4410107da5 + with: + registry_url: ${{ inputs.registry }} + osg_series: ${{ inputs.osg_series }} + osg_repo: ${{ inputs.osg_repo }} + base_os: ${{ inputs.base_os }} + platform: ${{ inputs.platform }} + push_by_digest: true + timestamp: ${{ inputs.timestamp }} + setup: false + output_image: ${{ inputs.output_image }} + + - name: Export digest + shell: bash + run: | + mkdir -p /tmp/${{ inputs.registry }}/digests + digest="${{ steps.upload-image.outputs.digest }}" + touch "/tmp/${{ inputs.registry }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ inputs.registry }}-${{ inputs.base_tag }}-${{ steps.slash-escape.outputs.platform }} + path: /tmp/${{ inputs.registry }}/digests/* + if-no-files-found: error + retention-days: 1 + + - name: Export tags + shell: bash + run: | + mkdir -p /tmp/${{ inputs.registry }}/tags/ + echo ${{ steps.upload-image.outputs.image-list }} > /tmp/${{ inputs.registry }}/tags/${{ inputs.base_tag }}-${{ steps.slash-escape.outputs.platform }} + + - name: Upload tags + uses: actions/upload-artifact@v4 + with: + name: tags-${{ inputs.registry }}-${{ inputs.base_tag }}-${{ steps.slash-escape.outputs.platform }} + path: /tmp/${{ inputs.registry }}/tags/* + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index cd64762..39b8ffa 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -11,12 +11,16 @@ on: jobs: build-images: runs-on: ubuntu-latest + # Continue to the push/tag step even if some build matrix combos fail + # Check that all arch artifacts are present in the push/tag step + continue-on-error: true strategy: fail-fast: false matrix: os: ['el9', 'cuda_11_8_0'] osg_series: ['23'] repo: ['development', 'testing', 'release'] + platform: ['linux/amd64','linux/arm64'] exclude: # cuda builds take a super long time; only do one of them - os: cuda_11_8_0 @@ -24,6 +28,8 @@ jobs: - os: cuda_11_8_0 repo: testing steps: + - uses: actions/checkout@v3 + - id: custom-image-name env: SERIES: ${{ matrix.osg_series }} @@ -31,21 +37,31 @@ jobs: OS: ${{ matrix.os }} run: | PREFIX="output_image=${GITHUB_REPOSITORY}:${SERIES}" + TIMESTAMP=$(date +%Y%m%d-%H%M) echo "${PREFIX}-${OS}-${REPO}" >> ${GITHUB_OUTPUT} + echo "base_tag=${SERIES}-${OS}-${REPO}" >> ${GITHUB_OUTPUT} + echo "timestamp=$TIMESTAMP" >> ${GITHUB_OUTPUT} - id: build-image - uses: opensciencegrid/build-container-action@v0.6.0 + name: Local image build + uses: opensciencegrid/build-container-action@420b64817b8a5d296a4201af6e955c4410107da5 with: + registry_url: hub.opensciencegrid.org osg_series: ${{ matrix.osg_series }} osg_repo: ${{ matrix.repo }} base_os: ${{ matrix.os }} + platform: ${{ matrix.platform }} output_image: ${{ steps.custom-image-name.outputs.output_image }} + timestamp: ${{ steps.custom-image-name.outputs.timestamp }} + - name: Prepare CVMFS + if: ${{ matrix.platform == 'linux/amd64' }} run: | sudo ./tests/setup_cvmfs.sh - name: Docker + CVMFS bindmount + if: ${{ matrix.platform == 'linux/amd64' }} id: test-docker-cvmfs env: CONTAINER_IMAGE: ${{ steps.build-image.outputs.timestamp-image }} @@ -54,7 +70,8 @@ jobs: bindmount \ "$CONTAINER_IMAGE" - - name: Docker + cvmfsexec + - name: Docker + cvmfsexec + if: ${{ matrix.platform == 'linux/amd64' }} id: test-docker-cvmfsexec env: CONTAINER_IMAGE: ${{ steps.build-image.outputs.timestamp-image }} @@ -64,6 +81,7 @@ jobs: "$CONTAINER_IMAGE" - name: Singularity + CVMFS bindmount + if: ${{ matrix.platform == 'linux/amd64' }} id: test-singularity-cvmfs env: CONTAINER_IMAGE: ${{ steps.build-image.outputs.timestamp-image }} @@ -77,55 +95,138 @@ jobs: "$CONTAINER_IMAGE" fi - - name: Harbor login + - id: upload-by-digest-harbor if: >- github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository_owner == 'opensciencegrid' - uses: docker/login-action@v2 + name: Upload By Digest to Harbor + uses: ./.github/actions/push-digest-local with: registry: hub.opensciencegrid.org username: ${{ secrets.OSG_HARBOR_ROBOT_USER }} password: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }} + osg_series: ${{ matrix.osg_series }} + osg_repo: ${{ matrix.repo }} + base_os: ${{ matrix.os }} + platform: ${{ matrix.platform }} + base_tag: ${{ steps.custom-image-name.outputs.base_tag }} + timestamp: ${{ steps.custom-image-name.outputs.timestamp }} + output_image: ${{ steps.custom-image-name.outputs.output_image }} - - name: Docker login + - id: upload-by-digest-dockerhub if: >- github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository_owner == 'opensciencegrid' - uses: docker/login-action@v2 + name: Upload By Digest to Docker Hub + uses: ./.github/actions/push-digest-local with: registry: docker.io username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + osg_series: ${{ matrix.osg_series }} + osg_repo: ${{ matrix.repo }} + base_os: ${{ matrix.os }} + platform: ${{ matrix.platform }} + base_tag: ${{ steps.custom-image-name.outputs.base_tag }} + timestamp: ${{ steps.custom-image-name.outputs.timestamp }} + output_image: ${{ steps.custom-image-name.outputs.output_image }} - - name: Push to OSG Harbor - if: >- - github.ref == 'refs/heads/master' && - github.event_name != 'pull_request' && - github.repository_owner == 'opensciencegrid' - env: - IMAGE_LIST: ${{ steps.build-image.outputs.image-list}} - OSG_SERIES: ${{ matrix.osg_series }} - run: | - case $OSG_SERIES in - '23' ) DEFAULT_OS=el9 ;; - * ) exit ;; - esac - for registry in hub.opensciencegrid.org docker.io; do - IFS=, - for image in ${IMAGE_LIST}; do - fqin=${registry}/${image} - docker tag ${image} ${fqin} - docker push ${fqin} - - # Also tag the image for the default OS as the OS-less tag - # (i.e. 23-el9-release -> 23-release) - image2=${image/-${DEFAULT_OS}-/-} # bash syntax for search-and-replace - if [[ $image2 != $image ]]; then - fqin2=${registry}/${image2} - docker tag ${image} ${fqin2} - docker push ${fqin2} - fi - done - done + merge-manifests: + runs-on: ubuntu-latest + if: >- + github.ref == 'refs/heads/master' && + github.event_name != 'pull_request' && + github.repository_owner == 'opensciencegrid' + needs: + - build-images + strategy: + fail-fast: false + matrix: + os: ['el9', 'cuda_11_8_0'] + osg_series: ['23'] + repo: ['development', 'testing', 'release'] + registry: [ + { + url: hub.opensciencegrid.org, + username: OSG_HARBOR_ROBOT_USER, + password: OSG_HARBOR_ROBOT_PASSWORD + }, + { + url: docker.io, + username: DOCKER_USERNAME, + password: DOCKER_PASSWORD + } + ] + exclude: + # cuda builds take a super long time; only do one of them + - os: cuda_11_8_0 + repo: development + - os: cuda_11_8_0 + repo: testing + steps: + - id: base-tag + env: + SERIES: ${{ matrix.osg_series }} + REPO: ${{ matrix.repo }} + OS: ${{ matrix.os }} + run: | + echo "base_tag=${SERIES}-${OS}-${REPO}" >> ${GITHUB_OUTPUT} + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/${{ matrix.registry.url }}/digests + pattern: digests-${{ matrix.registry.url }}-${{ steps.base-tag.outputs.base_tag }}-* + merge-multiple: true + + - name: Download tags + uses: actions/download-artifact@v4 + with: + path: /tmp/${{ matrix.registry.url }}/tags + pattern: tags-${{ matrix.registry.url }}-${{ steps.base-tag.outputs.base_tag }}-* + merge-multiple: true + + - name: Check Artifact Count + env: + EXPECTED: 2 + working-directory: /tmp/${{ matrix.registry.url }} + run: | + for dir in tags digests; do + artifact_count=$(ls $dir -1q | wc -l) + if [[ $artifact_count != $EXPECTED ]]; then + echo "Expected $EXPECTED artifacts in $dir; got $artifact_count" + exit 1 + fi + done + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Registry login + uses: docker/login-action@v2 + with: + registry: ${{ matrix.registry.url }} + username: ${{ secrets[matrix.registry.username] }} + password: ${{ secrets[matrix.registry.password] }} + + - name: Merge Artifacts + working-directory: /tmp/${{ matrix.registry.url }} + env: + DEFAULT_OS: el9 + run: | + BASE_IMG=${{ matrix.registry.url }}/opensciencegrid/osgvo-docker-pilot + DIGESTS=$(for digest in $(ls digests/); do echo $BASE_IMG@sha256:$digest; done) + TAGS=$(cat tags/*-amd64) + for tag in ${TAGS//,/ }; do + docker buildx imagetools create --tag $tag $DIGESTS; + + # Also tag the image for the default OS as the OS-less tag + # (i.e. 23-el9-release -> 23-release) + tag2=${tag/-${DEFAULT_OS}-/-} # bash syntax for search-and-replace + if [[ $tag2 != $tag ]]; then + docker buildx imagetools create --tag $tag2 $DIGESTS; + fi + done + From a51e879845e5a8a9bfc46e955f4eaa844dd83b59 Mon Sep 17 00:00:00 2001 From: mwestphall Date: Thu, 10 Oct 2024 10:12:34 -0500 Subject: [PATCH 2/3] Update buildx setup input arg --- .github/actions/push-digest-local/action.yaml | 4 ++-- .github/workflows/build-container.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/push-digest-local/action.yaml b/.github/actions/push-digest-local/action.yaml index f74ba9f..85c2cd9 100644 --- a/.github/actions/push-digest-local/action.yaml +++ b/.github/actions/push-digest-local/action.yaml @@ -58,7 +58,7 @@ runs: password: ${{ inputs.password }} - id: upload-image - uses: opensciencegrid/build-container-action@420b64817b8a5d296a4201af6e955c4410107da5 + uses: opensciencegrid/build-container-action@HEAD with: registry_url: ${{ inputs.registry }} osg_series: ${{ inputs.osg_series }} @@ -67,7 +67,7 @@ runs: platform: ${{ inputs.platform }} push_by_digest: true timestamp: ${{ inputs.timestamp }} - setup: false + buildx_setup: false output_image: ${{ inputs.output_image }} - name: Export digest diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 39b8ffa..996da65 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -44,7 +44,7 @@ jobs: - id: build-image name: Local image build - uses: opensciencegrid/build-container-action@420b64817b8a5d296a4201af6e955c4410107da5 + uses: opensciencegrid/build-container-action@HEAD with: registry_url: hub.opensciencegrid.org osg_series: ${{ matrix.osg_series }} From 9dd60fda7d67625180570c4566bc6dd05f8e0c1a Mon Sep 17 00:00:00 2001 From: Matt Westphall Date: Thu, 10 Oct 2024 16:30:28 -0500 Subject: [PATCH 3/3] Add comments for several design decisions --- .github/actions/push-digest-local/action.yaml | 8 ++++++-- .github/workflows/build-container.yml | 8 +++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/actions/push-digest-local/action.yaml b/.github/actions/push-digest-local/action.yaml index 85c2cd9..081762b 100644 --- a/.github/actions/push-digest-local/action.yaml +++ b/.github/actions/push-digest-local/action.yaml @@ -1,8 +1,12 @@ +# Local helper action that pushes an untagged image manifest to a registry via docker +# buildx, then records the digest of that manifest and expected tags as github action +# artifacts. Per the docker GHA multi-platform docs, the recommended approach to multi +# -platform builds is to push an untagged manifest for each arch, then combine them into +# a single tagged manifest in a separate GHA job. + name: 'Push Container by Digest Action' description: 'Pushes an image by digest to a given registry, then outputs its digest and tags as an artifact' - - inputs: registry: required: true diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 996da65..2cc8b4f 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -56,6 +56,8 @@ jobs: - name: Prepare CVMFS + # TODO: For all tests, GHA currently only supports amd64 runners. We will need + # to re-enable tests on arm64 when ARM runners become available. if: ${{ matrix.platform == 'linux/amd64' }} run: | sudo ./tests/setup_cvmfs.sh @@ -114,6 +116,10 @@ jobs: timestamp: ${{ steps.custom-image-name.outputs.timestamp }} output_image: ${{ steps.custom-image-name.outputs.output_image }} + # TODO: these artifacts will only be tagged if the build succeeds for every arch, + # and will remain as cruft in harbor unlesss manually removed in the case that + # some arch fails. Handling this scenario is future work: + # https://opensciencegrid.atlassian.net/browse/SOFTWARE-6010 - id: upload-by-digest-dockerhub if: >- github.ref == 'refs/heads/master' && @@ -190,7 +196,7 @@ jobs: - name: Check Artifact Count env: - EXPECTED: 2 + EXPECTED: 2 # One per build arch working-directory: /tmp/${{ matrix.registry.url }} run: | for dir in tags digests; do