build-ot3-actions.yml

name: 'Build Flex image on github workflows'
run-name: 'image: mono ${{ inputs.monorepo-ref }}, core ${{ inputs.oe-core-ref }}, fw ${{ inputs.ot3-firmware-ref }}'
on:
  workflow_dispatch:
    inputs:
      monorepo-ref:
        description: |
          Ref of https://github.com/opentrons/opentrons to build. This MUST be a full ref, e.g. refs/heads/edge, or '-' to indicate not-specified. If not specified, will be determined from the oe-core ref if specified, and then default to edge.
        required: true
        default: '-'
      oe-core-ref:
        description: |
          Ref of https://github.com/opentrons/oe-core to build. This is different from the ref specified in the github api/webUI when starting this workflow - that ref is what contains this workflow, this ref specifies what gets built. It MUST be a full ref, e.g. refs/heads/main, or '-' to indicate not-specified. If not specified, will be decided based on the monorepo ref; if that isn't specified, will be main.
        required: true
        default: '-'
      ot3-firmware-ref:
        description: |
          Ref of https://github.com/opentrons/ot3-firmware to build. It MUST be a full ref, e.g. refs/heads/main, or '-' to indicate not-specified. If not specified, will be decided based on the monorepo ref; if that isn't specified, will be main.
        required: false
        default: '-'
      infra-stage:
        description: |
          What infra stage to run on. This should almost always be prod; staging and dev are useful when you explicitly want to test those infra stages.
        required: true
        type: choice
        options:
          - 'stage-prod'
          - 'stage-staging'
          - 'stage-dev'
        default: 'stage-prod'

jobs:
  decide-refs:
    runs-on: ubuntu-latest
    outputs:
      oe-core: ${{ steps.build-refs.outputs.oe-core }}
      monorepo: ${{ steps.build-refs.outputs.monorepo }}
      ot3-firmware: ${{ steps.build-refs.outputs.ot3-firmware }}
      variant: ${{ steps.build-refs.outputs.variant }}
      build-type: ${{ steps.build-refs.outputs.build-type }}
    name: 'deciding refs to build'
    steps:
      - name: Fetch initial sources for action
        uses: 'actions/checkout@v3'
        with:
          submodules: false
          path: ./oe-core-for-workflow
      - name: Decide refs to build
        id: build-refs
        uses: './oe-core-for-workflow/.github/actions/build-refs'
        with:
          token: ${{ github.token }}
          monorepo: ${{ inputs.monorepo-ref }}
          oe-core: ${{ inputs.oe-core-ref }}
          ot3-firmware: ${{ inputs.ot3-firmware-ref }}
  run-build:
    needs: decide-refs
    strategy:
      matrix:
        build_env: [ '${{ inputs.infra-stage }}' ]
    name: 'Building ${{needs.decide-refs.outputs.variant}} images on ${{ matrix.build_env }}'
    timeout-minutes: 480
    runs-on: ['self-hosted', '${{matrix.build_env}}', '${{needs.decide-refs.outputs.variant}}']
    steps:
      - name: Fetch initial sources for action
        uses: 'actions/checkout@v3'
        with:
          submodules: false
          fetch-depth: 0
          path: ./oe-core-for-workflow
      - name: Fetch oe-core source
        uses: 'actions/checkout@v3'
        with:
          submodules: false
          fetch-depth: 0
          ref: ${{needs.decide-refs.outputs.oe-core}}
          path: ./oe-core
      - name: Fetch monorepo source
        uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
          ref: ${{ needs.decide-refs.outputs.monorepo }}
          repository: Opentrons/opentrons
          path: ./opentrons
      - name: Fetch ot3-firmware source
        uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
          ref: ${{ needs.decide-refs.outputs.ot3-firmware }}
          repository: Opentrons/ot3-firmware
          path: ./ot3-firmware
      - name: Sync oe-core submodules
        run: |
          chown -R `whoami` oe-core
          chown -R `whoami` opentrons
          chown -R `whoami` ot3-firmware
          cd oe-core
          ./update.sh
          cd ..
      - name: Configure AWS Credentials
        uses: './oe-core-for-workflow/.github/actions/aws-credentials'
        id: aws
        with:
          access_key_id: ${{ secrets.ROBOT_STACK_AWS_ACCESS_KEY_ID }}
          secret_access_key: ${{ secrets.ROBOT_STACK_AWS_SECRET_ACCESS_KEY }}
          region: us-east-2
          stage: ${{ matrix.build_env }}
      - name: Build container
        run: |
          cd oe-core
          tmp_dir=$(mktemp -d -t ci-XXXXXXX)
          cp start.sh $tmp_dir/
          docker build -f ./Dockerfile --tag "ot3-image:latest" $tmp_dir
          cd ..
      - name: Apply unconditional CI config overrides
        run: |
          cd oe-core
          echo "" >> ./build/conf/local.conf
          echo 'DL_DIR = "/volumes/cache/downloads"' >> ./build/conf/local.conf
          echo 'GITDIR = "/volumes/cache/git"' >> ./build/conf/local.conf
          echo 'SSTATE_DIR = "/volumes/cache/sstate"' >> ./build/conf/local.conf
          echo 'OT_BUILD_TYPE = "${{needs.decide-refs.outputs.build-type}}"' >> ./build/conf/local.conf
          echo 'YARN_CACHE_DIR = "/volumes/cache/yarn"' >> ./build/conf/local.conf
          echo 'ELECTRON_CACHE_DIR = "/volumes/cache/electron"' >> ./build/conf/local.conf
          cd ..
      - name: Apply internal-release variant CI config overrides
        if: needs.decide-refs.outputs.variant == 'internal-release'
        run: |
          cd oe-core
          echo 'OPENTRONS_PROJECT = "ot3"' >> ./build/conf/local.conf
          cd ..
      - name: Apply release variant CI config overrides
        if: needs.decide-refs.outputs.variant == 'release'
        run: |
          cd oe-core
          echo 'OPENTRONS_PROJECT = "robot-stack"' >> ./build/conf/local.conf
          cd ..
      - name: Apply release-build-only CI config overrides
        if: needs.decide-refs.outputs.build-type == 'release'
        run: |
          cd oe-core
          echo 'MIXPANEL_ID = "${{ secrets.MIXPANEL_ID }}"' >> ./build/conf/local.conf
          # setup signing key
          echo 'SIGNING_KEY = "${TOPDIR}/.signing-key"' >> ./build/conf/local.conf
          cat <<EOF >./build/.signing-key
          ${{secrets.ROBOT_SIGNING_KEY}}
          EOF
          cd ..

      - name: Pull S3 cache
        shell: bash
        run: |
          # fetch cache if the size is less than 20GB, so we have enough space to build image.
          sizeInBytes=`aws --profile=${{ steps.aws.outputs.profile_name }} s3 ls s3://${S3_CACHE_ARN/arn:aws:s3:::/} --recursive --summarize | awk END'{print}' | grep -o [0-9].*`
          sizeInGigabytes=$(($sizeInBytes/1024/1024/1024))
          if [[ sizeInGigabytes -gt 50 ]]; then
              echo "Doing clean build without cache, size of cache: ${sizeInGigabytes}GB!"
          else
              aws_cp="aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --no-progress"
              cachedir=${LOCAL_CACHE:-./cache}
              for cachetype in downloads sstate git ; do
                  localzip=$(realpath ${cachedir}/../${cachetype}.zip)
                  thiscache=${cachedir}/${cachetype}
                  echo "Fetching cache for ${cachetype} to ${localzip}"
                  mkdir -p ${thiscache}
                  TIME="%E" time $aws_cp s3://${S3_CACHE_ARN/arn:aws:s3:::/}/${cachetype}.zip ${localzip} 2>elapsed || continue
                  echo "Fetched $(du -h ${localzip} | cut -f 1)B in $(cat elapsed), extracting to ${thiscache}" || 0
                  TIME="%E" time unzip -q -u -o ${localzip} -d ${thiscache} 2>elapsed
                  echo "Extracted $(du -h -d 1 $thiscache | tail -n 1 | cut -f 1)B to ${thiscache} in $(cat elapsed)" || 0
              done
          fi
      - name: Download sources
        run: |
          here=$(pwd)
          oe_mount="type=bind,src=$here/oe-core,dst=/volumes/oe-core,consistency=delegated"
          monorepo_mount="type=bind,src=$here/opentrons,dst=/volumes/opentrons,consistency=delegated"
          ot3_firmware_mount="type=bind,src=$here/ot3-firmware,dst=/volumes/ot3-firmware,consistency=delegated"
          cache_mount="type=bind,src=${LOCAL_CACHE:-./cache},dst=/volumes/cache,consistency=delegated"
          tmp_mount="type=tmpfs,dst=/tmp"
          echo "docker run --rm --mount $oe_mount --mount $monorepo_mount --mount $ot3_firmware_mount --mount $cache_mount --mount $tmp_mount ot3-image:latest opentrons-ot3-image --runall=fetch"
          docker run --rm --mount $oe_mount --mount $monorepo_mount --mount $ot3_firmware_mount --mount $cache_mount --mount $tmp_mount ot3-image:latest opentrons-ot3-image --runall=fetch
      - name: Build image
        run: |
          here=$(pwd)
          oe_mount="type=bind,src=$here/oe-core,dst=/volumes/oe-core,consistency=delegated"
          monorepo_mount="type=bind,src=$here/opentrons,dst=/volumes/opentrons,consistency=delegated"
          ot3_firmware_mount="type=bind,src=$here/ot3-firmware,dst=/volumes/ot3-firmware,consistency=delegated"
          cache_mount="type=bind,src=${LOCAL_CACHE:-./cache},dst=/volumes/cache,consistency=delegated"
          tmp_mount="type=tmpfs,dst=/tmp"
          echo "docker run --rm --mount $oe_mount --mount $monorepo_mount --mount $ot3_firmware_mount --mount $cache_mount --mount $tmp_mount ot3-image:latest opentrons-ot3-image"
          docker run --rm --mount $oe_mount --mount $monorepo_mount --mount $ot3_firmware_mount --mount $cache_mount --mount $tmp_mount ot3-image:latest opentrons-ot3-image
      - name: Prune images
        if: always()
        run: docker image prune -af
      - name: Push S3 cache
        shell: bash
        continue-on-error: true
        run: |
          aws_cp="aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --no-progress"
          cachedir=${LOCAL_CACHE:-./cache}
          for cachetype in downloads sstate git ; do
              df -h
              localzip=$(realpath ${cachedir}/../${cachetype}.zip)
              thiscache=${cachedir}/${cachetype}
              cd ${thiscache}
              echo "Refreshing cache for ${cachetype} from ${thiscache} to ${localzip}"
              TIME="%E" time zip -q -r --filesync --symlinks  ${localzip} ./* 2>elapsed
              echo "Refreshed cache in $(cat elapsed)" || 0
              TIME="%E" time ${aws_cp} ${localzip} s3://${S3_CACHE_ARN/arn:aws:s3:::/}/${cachetype}.zip 2>elapsed
              echo "Uploaded $(du -h $localzip | cut -f 1)B in $(cat elapsed)" || 0
          done
      - name: Gather results
        id: artifacts
        run: |
          _oe_build=$(pwd)/oe-core/build
          echo "found build dir at ${_oe_build}"
          _artifact_root=${_oe_build}/deploy
          _artifact_unversioned_subdir=opentrons
          _artifact_s3=${_artifact_root}/${_artifact_unversioned_subdir}
          _artifact_versioned=${_artifact_root}/opentrons-versioned
          _oe_images=${_oe_build}/deploy/images
          _oe_tmp=${_oe_build}/tmp
          mkdir -p ${_artifact_s3}
          find ${_oe_images} -name "*opentrons-ot3-image-Tezi*" -exec cp {} ${_artifact_s3}/ot3-fullimage.tar \;
          find ${_oe_images} -name "ot3-system.zip" -exec cp {} ${_artifact_s3} \;
          find ${_oe_images} -name "VERSION.json" -exec cp {} ${_artifact_s3} \;
          find ${_oe_images} -name "release-notes.md" -exec cp {} ${_artifact_s3} \;
          tar czf ${_artifact_s3}/buildstats.tar.gz ${_oe_tmp}/buildstats

          mkdir -p ${_artifact_versioned}
          _fulltag="${{needs.decide-refs.outputs.monorepo}}"
          echo "monorepo_shorttag=${_fulltag:10}" >> $GITHUB_OUTPUT
          _vers_suffix=${{needs.decide-refs.outputs.variant == 'release' && '${_fulltag:11}' || '${_fulltag:14}'}}
          _system_zipname="ot3-system-${_vers_suffix}.zip"
          _version_jsonname="VERSION-${_vers_suffix}.json"
          _fullimage_tarname="ot3-fullimage-${_vers_suffix}.tar"
          cp ${_artifact_s3}/ot3-system.zip "${_artifact_versioned}/${_system_zipname}"
          cp ${_artifact_s3}/VERSION.json "${_artifact_versioned}/${_version_jsonname}"
          cp ${_artifact_s3}/ot3-fullimage.tar "${_artifact_versioned}/${_fullimage_tarname}"
          echo "versioned_system_zip=${_artifact_versioned}/${_system_zipname}" >> $GITHUB_OUTPUT
          echo "versioned_version_json=${_artifact_versioned}/${_version_jsonname}" >> $GITHUB_OUTPUT
          echo "versioned_fullimage_tar=${_artifact_versioned}/${_fullimage_tarname}" >> $GITHUB_OUTPUT

          echo "artifact_root=${_artifact_root}" >> $GITHUB_OUTPUT
          echo "artifact_unversioned_subdir=${_artifact_unversioned_subdir}" >> $GITHUB_OUTPUT
          echo "artifact_versioned=${_artifact_versioned}" >> $GITHUB_OUTPUT
      - name: Handle Release
        if: ${{ needs.decide-refs.outputs.build-type == 'release' }}
        shell: bash
        id: 'handle-release'
        run: |
          pushd oe-core/scripts
          aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --acl=public-read s3://${S3_ARTIFACT_ARN/arn:aws:s3:::/}/ot3-oe/releases.json releases.json
          base_url=https://${S3_ARTIFACT_ARN/arn:aws:s3:::/}/ot3-oe/${{ github.run_id }}
          version_file=${{ steps.artifacts.outputs.artifact_root }}/${{ steps.artifacts.outputs.artifact_unversioned_subdir }}/VERSION.json
          python3 update_releases_file.py --releases-file releases.json --version-file ${version_file} --base-url $base_url
          aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --acl=public-read releases.json s3://${S3_ARTIFACT_ARN/arn:aws:s3:::/}/ot3-oe/
          popd
      - name: Upload results to S3
        shell: bash
        id: 'upload-results'
        run: |
          pushd ${{ steps.artifacts.outputs.artifact_root }}
          aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --acl=public-read --recursive ${{ steps.artifacts.outputs.artifact_unversioned_subdir }} s3://${S3_ARTIFACT_ARN/arn:aws:s3:::/}/ot3-oe/${{ github.run_id }}
          root_url=https://${S3_ARTIFACT_ARN/arn:aws:s3:::/}/ot3-oe/${{ github.run_id }}
          echo "console_url=https://s3.console.aws.amazon.com/s3/buckets/${S3_ARTIFACT_ARN/arn:aws:s3::::/}?prefix=${{ github.run_id }}" >> $GITHUB_OUTPUT
          echo "version_file_url=$root_url/VERSION.json" >> $GITHUB_OUTPUT
          echo "release_notes_file_url=$root_url/release-notes.md" >> $GITHUB_OUTPUT
          echo "system_url=$root_url/ot3-system.zip" >> $GITHUB_OUTPUT
          echo "fullimage_url=$root_url/ot3-fullimage.tar" >> $GITHUB_OUTPUT
          popd
      - name: Upload system zip to monorepo release
        if: needs.decide-refs.outputs.build-type == 'release'
        uses: 'ncipollo/release-action@v1.12.0'
        with:
          allowUpdates: true
          omitBody: true
          omitName: true
          omitPrereleaseDuringUpdate: true
          omitDraftDuringUpdate: true
          repo: opentrons
          tag: ${{ steps.artifacts.outputs.monorepo_shorttag }}
          artifacts: ${{ steps.artifacts.outputs.versioned_system_zip }}
          artifactContentType: application/zip
          token: ${{secrets.MONOREPO_RELEASE_ARTIFACT_UPLOAD_TOKEN}}
      - name: Upload fullimage tar to monorepo release
        if: needs.decide-refs.outputs.build-type == 'release'
        uses: 'ncipollo/release-action@v1.12.0'
        with:
          allowUpdates: true
          omitBody: true
          omitName: true
          omitPrereleaseDuringUpdate: true
          omitDraftDuringUpdate: true
          repo: opentrons
          tag: ${{ steps.artifacts.outputs.monorepo_shorttag }}
          artifacts: ${{ steps.artifacts.outputs.versioned_fullimage_tar }}
          artifactContentType: application/x-tar
          token: ${{secrets.MONOREPO_RELEASE_ARTIFACT_UPLOAD_TOKEN}}
      - name: Upload version json to monorepo release
        if: needs.decide-refs.outputs.build-type == 'release'
        uses: 'ncipollo/release-action@v1.12.0'
        with:
          allowUpdates: true
          omitBody: true
          omitName: true
          omitPrereleaseDuringUpdate: true
          omitDraftDuringUpdate: true
          repo: opentrons
          tag: ${{ steps.artifacts.outputs.monorepo_shorttag }}
          artifacts: ${{ steps.artifacts.outputs.versioned_version_json }}
          artifactContentType: application/json
          token: ${{secrets.MONOREPO_RELEASE_ARTIFACT_UPLOAD_TOKEN}}
      - name: Post results as internal-release
        if: ${{ matrix.build_env == 'stage-prod' && needs.decide-refs.outputs.variant=='internal-release' }}
        uses: slackapi/slack-github-action@v1.14.0
        with:
          payload: "{\"s3-url\":\"${{ steps.upload-results.outputs.console_url }}/\",\"type\":\"branch\", \"reflike\":\"${{ needs.decide-refs.outputs.oe-core }}\",\"monorepo-reflike\":\"${{ needs.decide-refs.outputs.monorepo }}\",\"firmware-reflike\":\"${{ needs.decide-refs.outputs.ot3-firmware }}\",\"full-image\":\"${{ steps.upload-results.outputs.fullimage_url }}\", \"system-update\":\"${{ steps.upload-results.outputs.system_url }}\", \"version-file\":\"${{ steps.upload-results.outputs.version_file_url }}\", \"release-notes\":\"${{ steps.upload-results.outputs.release_notes_file_url }}\"}"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

      - name: Post results as release
        if: ${{ matrix.build_env == 'stage-prod' && needs.decide-refs.outputs.variant== 'release' }}
        uses: slackapi/slack-github-action@v1.14.0
        with:
          payload: "{\"s3-url\":\"${{ steps.upload-results.outputs.console_url }}/\",\"type\":\"branch\", \"reflike\":\"${{ needs.decide-refs.outputs.oe-core }}\",\"monorepo-reflike\":\"${{ needs.decide-refs.outputs.monorepo }}\",\"firmware-reflike\":\"${{ needs.decide-refs.outputs.ot3-firmware }}\",\"full-image\":\"${{ steps.upload-results.outputs.fullimage_url }}\", \"system-update\":\"${{ steps.upload-results.outputs.system_url }}\", \"version-file\":\"${{ steps.upload-results.outputs.version_file_url }}\", \"release-notes\":\"${{ steps.upload-results.outputs.release_notes_file_url }}\"}"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_RELEASE }}
      - name: Remove build data
        if: always()
        run: |
          rm -rf ./*
      - name: Remove poisoned cache
        if: ${{ failure() }}
        run: |
          rm -rf ${LOCAL_CACHE:-./cache}/*