container-build-publish.yml

name: 'Docker: build and publish container images'

on:
  push:
    branches:
      - master
    paths-ignore:
      - 'doc/**'
      - 'docker-k8s/examples/**'
      - 'Formula/**'
      - 'scripts/get-version'
      - 'windows/**'
      - '**.md'
  schedule:
    - cron: '22 2 */6 * *' # every 6 days to avoid gha cache being evicted
  pull_request:
    paths-ignore:
      - 'doc/**'
      - 'docker-k8s/examples/**'
      - 'Formula/**'
      - 'scripts/get-version'
      - 'windows/**'
      - '**.md'

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
  DOCKERHUB_REPO: eturnal/eturnal
  PKGREL_FILE: docker-k8s/pkgrel

jobs:
################################################################################
  #' check whether to compile from master branch or from tagged version
  detect-change:
    name: Check ctr version change
    runs-on: ubuntu-latest
    outputs:
      update: ${{ steps.check_version_changed.outputs.update }}
    steps:
      -
        name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 2
      -
        name: Compare ctr package-release vsn between commits
        id: check_version_changed
        run: |
          TAG_PKGREL=$(awk 'END{print}' ${{ env.PKGREL_FILE }})
          git checkout HEAD^
          TAG_PKGREL_BASELINE=$(awk 'END{print}' ${{ env.PKGREL_FILE }})
          if [ "$TAG_PKGREL" != "$TAG_PKGREL_BASELINE" ]
          then echo "update=true" >> $GITHUB_OUTPUT
          else echo "update=false" >> $GITHUB_OUTPUT
          fi

################################################################################
  #' build musl-libc based binary tarballs for x64/arm64
  build-musl-binary-archives:
    name: ${{ matrix.arch }} - build musl-libc based binary archives
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [x64, arm64]
      fail-fast: false
    needs: [detect-change]
    steps:
      -
        name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      -
        name: Cache toolchain directory
        uses: actions/cache@v3
        with:
          path: ~/build/
          key: ${{runner.os}}-ct-ng-1.25.0-${{ matrix.arch }}-musl
      -
        name: On push master | extract git version
        if: needs.detect-change.outputs.update == 'false'
        run: echo "TAG_VERSION=$(echo "$(./scripts/get-version)" | sed -e 's|+|-|')" >> $GITHUB_ENV
      -
        name: On release | extract release tag
        if: needs.detect-change.outputs.update == 'true'
        run: echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
      -
        name: On release | check out release to be published/updated
        uses: actions/checkout@v3
        if: needs.detect-change.outputs.update == 'true'
        with:
          ref: ${{ env.TAG_VERSION }}
      -
        name: Install prerequisites, obtain erlang/otp version & target arch ...
        run: |
          sudo apt-get -qq update
          # https://github.com/crosstool-ng/crosstool-ng/blob/master/testing/docker/ubuntu22.04/Dockerfile
          sudo apt-get -qq install makeself build-essential \
            gcc g++ gperf bison flex texinfo help2man make libncurses5-dev \
            python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 \
            xz-utils unzip patch libstdc++6 rsync git meson ninja-build \
            binfmt-support qemu-user-static
          echo "OTP_VSN=$(awk '/^otp_vsn=/ {{gsub(/[^0-9.]/, ""); print}}' scripts/make-binaries)" >> $GITHUB_ENV
          echo "ARCH=$(echo ${{ matrix.arch }} | sed -e 's|x64|x86_64|;s|arm64|aarch64|')" >> $GITHUB_ENV
      -
        name: Install erlang/OTP
        uses: erlef/setup-beam@v1
        with:
          otp-version: ${{ env.OTP_VSN }}
          version-type: strict
      -
        name: Build ${{ matrix.arch }} musl-libc based binary archives
        run: CHECK_DEPS=false scripts/make-binaries ${{ env.ARCH }}-linux-musl
      -
        name: Start container for rebar3 test suites ...
        run: |
          rebar3_path="$HOME/build/bootstrap/bin"
          otp_path="$HOME/build/eturnal/${{ env.ARCH }}-linux-musl/bin"
          alpine_path='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
          arch=$(echo ${{ matrix.arch }} | sed -e 's|x64|amd64|')
          docker run --init -d --name test-suites \
                --platform linux/$arch \
                -v $HOME/build:$HOME/build \
                -v $PWD:/eturnal \
                --workdir /eturnal \
                -e PATH=$rebar3_path:$otp_path:$alpine_path \
                docker.io/alpine:latest \
                sleep 600
      - # add build-tools & git, if rebar.config uses e.g. "github" as source
        name: add build tools to the container ...
        run: docker exec test-suites apk add build-base git yaml-dev
      -
        name: rebar3 xref ...
        run: docker exec test-suites rebar3 xref
      -
        name: rebar3 eunit ...
        run: docker exec test-suites rebar3 eunit
      -
        name: rebar3 ct ...
        run: docker exec test-suites rebar3 ct
      -
        name: Stop rebar3 test container ...
        run: docker stop test-suites
      -
        name: Upload artifact | ${{ matrix.arch }} musl-libc based binary tarball
        uses: actions/upload-artifact@v3
        with:
          name: eturnal-${{ env.TAG_VERSION }}-linux-musl-${{ matrix.arch }}.tar.gz
          path: eturnal-*-linux-musl-${{ matrix.arch }}.tar.gz
          if-no-files-found: error
          retention-days: 15

################################################################################
  #' Test musl-libc tarballs for incompatibilities between musl-libc versions
  ci-musl-libc:
    name: CI - musl-libc v${{ matrix.musl }} compatibality test on ${{ matrix.arch }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        musl: [1.1.16, 1.1.17, 1.1.18, 1.1.19, 1.1.20, 1.1.21, 1.1.22, 1.1.23, 1.1.24, 1.2.0, 1.2.1, 1.2.2, 1.2.3, 1.2.4]
        arch: [arm64, x64]
      fail-fast: false
    needs: [detect-change, build-musl-binary-archives]
    steps:
      -
        name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      -
        name: On push master | extract git version
        if: needs.detect-change.outputs.update == 'false'
        run: echo "TAG_VERSION=$(echo "$(./scripts/get-version)" | sed -e 's|+|-|')" >> $GITHUB_ENV
      -
        name: On release | extract release tag
        if: needs.detect-change.outputs.update == 'true'
        run: echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
      -
        name: Prepare build context and CI environemnt ...
        run: |
          cp $(find . -name eturnal-new-otp.yml) $PWD/eturnal.yml
          echo "  log_dir: stdout" >> $PWD/eturnal.yml
          echo "DOCKERFILE_PATH=$(dirname $(find . -name Dockerfile))" >> $GITHUB_ENV
          echo "ARCH=$(echo ${{ matrix.arch }} | sed -e 's|x64|amd64|')" >> $GITHUB_ENV
          echo "STUN_SERVICE=$(wget -O - https://raw.githubusercontent.com/pradt2/always-online-stun/master/valid_hosts.txt | awk 'END{gsub(":", " "); print}')" >> $GITHUB_ENV
      -
        name: Download artifact | ${{ matrix.arch }} musl-libc based binary tarball
        uses: actions/download-artifact@v3
        with:
          name: eturnal-${{ env.TAG_VERSION }}-linux-musl-${{ matrix.arch }}.tar.gz
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Build CI image ...
        run: |
          docker buildx build --load \
                --platform linux/${{ env.ARCH }} \
                -f ${{ env.DOCKERFILE_PATH }}/Dockerfile \
                --build-arg CI_MUSL_VSN=${{ matrix.musl }} \
                --build-arg METHOD=package \
                --target musl-ci \
                -t localhost/${{ env.IMAGE_NAME }}:musl-ci \
                .
      -
        name: Start CI container ...
        run: |
          docker run --init -d --name ci \
                --platform linux/${{ env.ARCH }} \
                --user eturnal \
                -e STUN_SERVICE="${{ env.STUN_SERVICE }}" \
                -v $PWD/eturnal.yml:/etc/eturnal.yml \
                localhost/${{ env.IMAGE_NAME }}:musl-ci \
                sleep 300
          docker exec ci eturnalctl daemon
      -
        name: CI | print container log ...
        run: docker logs ci
      -
        name: CI | send ping to eturnal container ...
        run: docker exec ci eturnalctl ping
      -
        name: CI | perform STUN IPv4 query ...
        run: docker exec ci stun -4 ${{ env.STUN_SERVICE }}
      -
        name: Stop CI container ...
        if: success() || failure()
        run: docker stop ci

################################################################################
  #' build actual images for x64/arm64 with built binary tarballs
  build-ctr-binary-based:
    name: ${{ matrix.arch }} - build & publish container image (binary-based)
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [x64, arm64]
      fail-fast: false
    needs: [detect-change, build-musl-binary-archives]
    steps:
      -
        name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      -
        name: On push master | extract git version
        if: needs.detect-change.outputs.update == 'false'
        run: echo "TAG_VERSION=$(echo "$(./scripts/get-version)" | sed -e 's|+|-|')" >> $GITHUB_ENV
      -
        name: On release | extract release tag
        if: needs.detect-change.outputs.update == 'true'
        run: echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Extract build & environment variables
        run: |
          echo "DOCKERFILE_PATH=$(dirname $(find . -name Dockerfile))" >> $GITHUB_ENV
          echo "ARCH=$(echo ${{ matrix.arch }} | sed -e 's|x64|amd64|')" >> $GITHUB_ENV
          if [ ${{ github.event_name }} = 'push' ]
          then echo "MODE=push" >> $GITHUB_ENV
          else echo "MODE=load" >> $GITHUB_ENV
          fi
      -
        name: Download artifact | ${{ matrix.arch }} musl-libc based binary tarball
        uses: actions/download-artifact@v3
        with:
          name: eturnal-${{ env.TAG_VERSION }}-linux-musl-${{ matrix.arch }}.tar.gz
      -
        name: Log in to ${{ env.REGISTRY }}
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      -
        name: Docker meta
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
      -
        name: Build and push by digest
        id: build
        uses: docker/build-push-action@v3
        with:
          build-args: |
            METHOD=package
            VERSION=${{ env.TAG_VERSION }}
            REPOSITORY=https://github.com/${{ github.repository }}.git
          context: .
          file: ${{ env.DOCKERFILE_PATH }}/Dockerfile
          platforms: linux/${{ env.ARCH }}
          labels: ${{ steps.meta.outputs.labels }}
          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,${{ env.MODE }}=true
      -
        name: Export digest | ${{ matrix.arch }}
        run: |
          mkdir -p /tmp/digests
          digest="${{ steps.build.outputs.digest }}"
          touch "/tmp/digests/${digest#sha256:}"
      -
        name: Upload digest | ${{ matrix.arch }}
        uses: actions/upload-artifact@v3
        with:
          name: digests
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 5

################################################################################
  #' build container images with local source files
  build-ctr:
    name: ${{ matrix.arch }} - build & publish container image (non-binary-based)
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [386, arm/v7, ppc64le, s390x]
      fail-fast: false
    needs: [detect-change]
    steps:
      -
        name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      -
        name: On push master | extract git version and define build mode
        if: needs.detect-change.outputs.update == 'false'
        run: |
          echo "VSN=$(git describe --tag)" >> $GITHUB_ENV
          echo "TAG_VERSION=$(echo "$(./scripts/get-version)" | sed -e 's|+|-|')" >> $GITHUB_ENV
          echo "SRC=local" >> $GITHUB_ENV
          echo "OTP_VSN=$(awk '/^otp_vsn=/ {{gsub(/[^0-9.]/, ""); print}}' scripts/make-binaries)" >> $GITHUB_ENV
      -
        name: On release | extract git version and define build mode
        if: needs.detect-change.outputs.update == 'true'
        run: |
          eturnal_vsn=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})
          echo "VSN=$eturnal_vsn" >> $GITHUB_ENV
          echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV

          # define erlang/OTP version
          echo "OTP_VSN=$(wget -O - https://raw.githubusercontent.com/${{ github.repository }}/$eturnal_vsn/scripts/make-binaries \
            | awk '/^otp_vsn=/ {{gsub(/[^0-9.]/, ""); print}}')" >> $GITHUB_ENV

          # check whether to build from archive or git, because git repo is HEAD here
          # hence, we would not build the specifc release when updating the ctr image
          pkgrel=$(awk 'END{gsub("-", " "); print $2}' ${{ env.PKGREL_FILE }})
          if [ "$pkgrel" != 'r0' ]
          then echo "SRC=web" >> $GITHUB_ENV
          else echo "SRC=local" >> $GITHUB_ENV
          fi
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Extract build & environment variables
        run: |
          echo "DOCKERFILE_PATH=$(dirname $(find . -name Dockerfile))" >> $GITHUB_ENV
          # Check if rebar3 common test should be performed
          # fix for slow architectures was introduces after version 1.10.1
          if dpkg --compare-versions ${{ env.TAG_VERSION }} le "1.10.1"
          then echo "REBAR_CT=false" >> $GITHUB_ENV
          else echo "REBAR_CT=true" >> $GITHUB_ENV
          fi
          if [ ${{ github.event_name }} = 'push' ]
          then echo "MODE=push" >> $GITHUB_ENV
          else echo "MODE=load" >> $GITHUB_ENV
          fi
      -
        name: Log in to ${{ env.REGISTRY }}
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      -
        name: Docker meta
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
      -
        name: Build and push by digest
        id: build
        uses: docker/build-push-action@v3
        with:
          build-args: |
            SOURCE=${{ env.SRC }}
            VERSION=${{ env.VSN }}
            OTP_VSN=${{ env.OTP_VSN }}
            REPOSITORY=https://github.com/${{ github.repository }}.git
            REBAR_CT=${{ env.REBAR_CT }}
          context: .
          file: ${{ env.DOCKERFILE_PATH }}/Dockerfile
          platforms: linux/${{ matrix.arch }}
          labels: ${{ steps.meta.outputs.labels }}
          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,${{ env.MODE }}=true
      -
        name: Export digest | ${{ matrix.arch }}
        run: |
          mkdir -p /tmp/digests
          digest="${{ steps.build.outputs.digest }}"
          touch "/tmp/digests/${digest#sha256:}"
      -
        name: Upload digest | ${{ matrix.arch }}
        uses: actions/upload-artifact@v3
        with:
          name: digests
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 5

################################################################################
  ## merge single images to only advertise one image plus tag
  publish:
    name: ${{ matrix.registry }} - publish image manifest
    runs-on: ubuntu-latest
    strategy:
      matrix:
        registry: [ghcr.io, docker.io]
      fail-fast: false
    if: github.event_name != 'pull_request' && github.event_name != 'schedule'
    needs: [detect-change, build-musl-binary-archives, build-ctr-binary-based, build-ctr]
    steps:
      -
        name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      -
        name: On push master | extract git version and define build variants
        if: needs.detect-change.outputs.update == 'false'
        run: |
          echo "TAG_VERSION=$(echo "$(./scripts/get-version)" | sed -e 's|+|-|')" >> $GITHUB_ENV
      -
        name: On release | extract release tags to be published/updated
        if: needs.detect-change.outputs.update == 'true'
        run: |
          echo "TAG_PKGREL=$(awk 'END{print}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
          echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
          echo "TAG_MINOR=$(awk 'END{gsub("\\.", " "); print $1"."$2}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
          echo "TAG_MAJOR=$(awk 'END{gsub("\\.", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
      -
        name: Log in to ${{ matrix.registry }}
        uses: docker/login-action@v2
        if: |
          ( matrix.registry == 'docker.io'
            && github.repository_owner == 'processone' )
          || matrix.registry == 'ghcr.io'
        with:
          registry: ${{ matrix.registry }}
          username: ${{ (matrix.registry == 'docker.io'
                         && secrets.DOCKERHUB_USERNAME)
                     || github.repository_owner }}
          password: ${{ (matrix.registry == 'docker.io'
                         && secrets.DOCKERHUB_TOKEN)
                     || secrets.GITHUB_TOKEN }}
      -
        name: Download digests
        uses: actions/download-artifact@v3
        with:
          name: digests
          path: /tmp/digests
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: On push master | Docker meta
        id: meta
        if: needs.detect-change.outputs.update == 'false'
        uses: docker/metadata-action@v4
        with:
          images: ${{ matrix.registry }}/${{ (matrix.registry == 'docker.io'
                                        && env.DOCKERHUB_REPO)
                                        || env.IMAGE_NAME }}
          tags: |
            edge
      -
        name: On release | Docker meta
        id: meta-rel
        if: needs.detect-change.outputs.update == 'true'
        uses: docker/metadata-action@v4
        with:
          images: ${{ matrix.registry }}/${{ (matrix.registry == 'docker.io'
                                        && env.DOCKERHUB_REPO)
                                        || env.IMAGE_NAME }}
          tags: |
            latest
            ${{ env.TAG_PKGREL }}
            ${{ env.TAG_VERSION }}
            ${{ env.TAG_MINOR }}
            ${{ env.TAG_MAJOR }}
      -
        name: Create manifest list and push
        if: |
          ( matrix.registry == 'docker.io'
            && github.repository_owner == 'processone' )
          || matrix.registry == 'ghcr.io'
        working-directory: /tmp/digests
        run: |
          docker buildx imagetools create $(jq -r '"-t " + (.tags | join(" -t "))' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
      -
        name: Inspect image
        if: matrix.registry == 'ghcr.io'
        run: |
          docker buildx imagetools inspect ${{ matrix.registry }}/${{ env.IMAGE_NAME }}:${{ steps.meta-rel.outputs.version }} ||
          docker buildx imagetools inspect ${{ matrix.registry }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}