Docker: build and publish container images

Add a macos CI as well as a homebrew formula #5

Workflow file for this run

.github/workflows/container-build-publish.yml at 73bc728

	name: 'Docker: build and publish container images'

	on:
	push:
	branches:
	- master
	paths-ignore:
	- 'doc/**'
	- 'docker-k8s/examples/**'
	- 'Formula/**'
	- 'scripts/get-version'
	- 'windows/**'
	- '**.md'
	schedule:
	- cron: '22 2 /6 *' # every 6 days to avoid gha cache being evicted
	pull_request:
	paths-ignore:
	- 'doc/**'
	- 'docker-k8s/examples/**'
	- 'Formula/**'
	- 'scripts/get-version'
	- 'windows/**'
	- '**.md'

	env:
	REGISTRY: ghcr.io
	IMAGE_NAME: ${{ github.repository }}
	DOCKERHUB_REPO: eturnal/eturnal
	PKGREL_FILE: docker-k8s/pkgrel

	jobs:
	################################################################################
	#' check whether to compile from master branch or from tagged version
	detect-change:
	name: Check ctr version change
	runs-on: ubuntu-latest
	outputs:
	update: ${{ steps.check_version_changed.outputs.update }}
	steps:
	-
	name: Check out repository code
	uses: actions/checkout@v3
	with:
	fetch-depth: 2
	-
	name: Compare ctr package-release vsn between commits
	id: check_version_changed
	run: \|
	TAG_PKGREL=$(awk 'END{print}' ${{ env.PKGREL_FILE }})
	git checkout HEAD^
	TAG_PKGREL_BASELINE=$(awk 'END{print}' ${{ env.PKGREL_FILE }})
	if [ "$TAG_PKGREL" != "$TAG_PKGREL_BASELINE" ]
	then echo "update=true" >> $GITHUB_OUTPUT
	else echo "update=false" >> $GITHUB_OUTPUT
	fi

	################################################################################
	#' build musl-libc based binary tarballs for x64/arm64
	build-musl-binary-archives:
	name: ${{ matrix.arch }} - build musl-libc based binary archives
	runs-on: ubuntu-latest
	strategy:
	matrix:
	arch: [x64, arm64]
	fail-fast: false
	needs: [detect-change]
	steps:
	-
	name: Check out repository code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	-
	name: Cache toolchain directory
	uses: actions/cache@v3
	with:
	path: ~/build/
	key: ${{runner.os}}-ct-ng-1.25.0-${{ matrix.arch }}-musl
	-
	name: On push master \| extract git version
	if: needs.detect-change.outputs.update == 'false'
	run: echo "TAG_VERSION=$(echo "$(./scripts/get-version)" \| sed -e 's\|+\|-\|')" >> $GITHUB_ENV
	-
	name: On release \| extract release tag
	if: needs.detect-change.outputs.update == 'true'
	run: echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
	-
	name: On release \| check out release to be published/updated
	uses: actions/checkout@v3
	if: needs.detect-change.outputs.update == 'true'
	with:
	ref: ${{ env.TAG_VERSION }}
	-
	name: Install prerequisites, obtain erlang/otp version & target arch ...
	run: \|
	sudo apt-get -qq update
	# https://github.com/crosstool-ng/crosstool-ng/blob/master/testing/docker/ubuntu22.04/Dockerfile
	sudo apt-get -qq install makeself build-essential \
	gcc g++ gperf bison flex texinfo help2man make libncurses5-dev \
	python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 \
	xz-utils unzip patch libstdc++6 rsync git meson ninja-build \
	binfmt-support qemu-user-static
	echo "OTP_VSN=$(awk '/^otp_vsn=/ {{gsub(/[^0-9.]/, ""); print}}' scripts/make-binaries)" >> $GITHUB_ENV
	echo "ARCH=$(echo ${{ matrix.arch }} \| sed -e 's\|x64\|x86_64\|;s\|arm64\|aarch64\|')" >> $GITHUB_ENV
	-
	name: Install erlang/OTP
	uses: erlef/setup-beam@v1
	with:
	otp-version: ${{ env.OTP_VSN }}
	version-type: strict
	-
	name: Build ${{ matrix.arch }} musl-libc based binary archives
	run: CHECK_DEPS=false scripts/make-binaries ${{ env.ARCH }}-linux-musl
	-
	name: Start container for rebar3 test suites ...
	run: \|
	rebar3_path="$HOME/build/bootstrap/bin"
	otp_path="$HOME/build/eturnal/${{ env.ARCH }}-linux-musl/bin"
	alpine_path='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
	arch=$(echo ${{ matrix.arch }} \| sed -e 's\|x64\|amd64\|')
	docker run --init -d --name test-suites \
	--platform linux/$arch \
	-v $HOME/build:$HOME/build \
	-v $PWD:/eturnal \
	--workdir /eturnal \
	-e PATH=$rebar3_path:$otp_path:$alpine_path \
	docker.io/alpine:latest \
	sleep 600
	- # add build-tools & git, if rebar.config uses e.g. "github" as source
	name: add build tools to the container ...
	run: docker exec test-suites apk add build-base git yaml-dev
	-
	name: rebar3 xref ...
	run: docker exec test-suites rebar3 xref
	-
	name: rebar3 eunit ...
	run: docker exec test-suites rebar3 eunit
	-
	name: rebar3 ct ...
	run: docker exec test-suites rebar3 ct
	-
	name: Stop rebar3 test container ...
	run: docker stop test-suites
	-
	name: Upload artifact \| ${{ matrix.arch }} musl-libc based binary tarball
	uses: actions/upload-artifact@v3
	with:
	name: eturnal-${{ env.TAG_VERSION }}-linux-musl-${{ matrix.arch }}.tar.gz
	path: eturnal-*-linux-musl-${{ matrix.arch }}.tar.gz
	if-no-files-found: error
	retention-days: 15

	################################################################################
	#' Test musl-libc tarballs for incompatibilities between musl-libc versions
	ci-musl-libc:
	name: CI - musl-libc v${{ matrix.musl }} compatibality test on ${{ matrix.arch }}
	runs-on: ubuntu-latest
	strategy:
	matrix:
	musl: [1.1.16, 1.1.17, 1.1.18, 1.1.19, 1.1.20, 1.1.21, 1.1.22, 1.1.23, 1.1.24, 1.2.0, 1.2.1, 1.2.2, 1.2.3, 1.2.4]
	arch: [arm64, x64]
	fail-fast: false
	needs: [detect-change, build-musl-binary-archives]
	steps:
	-
	name: Check out repository code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	-
	name: Cache toolchain directory
	uses: actions/cache@v3
	with:
	path: ~/build/
	key: ${{runner.os}}-ct-ng-1.25.0-${{ matrix.arch }}-musl
	-
	name: On push master \| extract git version
	if: needs.detect-change.outputs.update == 'false'
	run: echo "TAG_VERSION=$(echo "$(./scripts/get-version)" \| sed -e 's\|+\|-\|')" >> $GITHUB_ENV
	-
	name: On release \| extract release tag
	if: needs.detect-change.outputs.update == 'true'
	run: echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
	-
	name: Prepare build context and CI environemnt ...
	run: \|
	cp $(find . -name eturnal-new-otp.yml) $PWD/eturnal.yml
	echo " log_dir: stdout" >> $PWD/eturnal.yml
	echo "DOCKERFILE_PATH=$(dirname $(find . -name Dockerfile))" >> $GITHUB_ENV
	echo "ARCH=$(echo ${{ matrix.arch }} \| sed -e 's\|x64\|amd64\|')" >> $GITHUB_ENV
	echo "STUN_SERVICE=$(wget -O - https://raw.githubusercontent.com/pradt2/always-online-stun/master/valid_hosts.txt \| awk 'END{gsub(":", " "); print}')" >> $GITHUB_ENV
	-
	name: Download artifact \| ${{ matrix.arch }} musl-libc based binary tarball
	uses: actions/download-artifact@v3
	with:
	name: eturnal-${{ env.TAG_VERSION }}-linux-musl-${{ matrix.arch }}.tar.gz
	-
	name: Set up QEMU
	uses: docker/setup-qemu-action@v2
	-
	name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2
	-
	name: Build CI image ...
	run: \|
	docker buildx build --load \
	--platform linux/${{ env.ARCH }} \
	-f ${{ env.DOCKERFILE_PATH }}/Dockerfile \
	--build-arg CI_MUSL_VSN=${{ matrix.musl }} \
	--build-arg METHOD=package \
	--target musl-ci \
	-t localhost/${{ env.IMAGE_NAME }}:musl-ci \
	.
	-
	name: Start CI container ...
	run: \|
	docker run --init -d --name ci \
	--platform linux/${{ env.ARCH }} \
	--user eturnal \
	-e STUN_SERVICE="${{ env.STUN_SERVICE }}" \
	-v $PWD/eturnal.yml:/etc/eturnal.yml \
	localhost/${{ env.IMAGE_NAME }}:musl-ci \
	eturnalctl foreground
	echo ">> sleep shortly to have the service up ..." && sleep 15s
	-
	name: CI \| print container log ...
	run: docker logs ci
	-
	name: CI \| send ping to eturnal container ...
	run: docker exec ci eturnalctl ping
	-
	name: CI \| perform STUN IPv4 query ...
	run: docker exec ci stun -4 ${{ env.STUN_SERVICE }}
	-
	name: Stop CI container ...
	if: success() \|\| failure()
	run: docker stop ci

	################################################################################
	#' build actual images for x64/arm64 with built binary tarballs
	build-ctr-binary-based:
	name: ${{ matrix.arch }} - build & publish container image (binary-based)
	runs-on: ubuntu-latest
	strategy:
	matrix:
	arch: [x64, arm64]
	fail-fast: false
	needs: [detect-change, build-musl-binary-archives]
	steps:
	-
	name: Check out repository code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	-
	name: On push master \| extract git version
	if: needs.detect-change.outputs.update == 'false'
	run: echo "TAG_VERSION=$(echo "$(./scripts/get-version)" \| sed -e 's\|+\|-\|')" >> $GITHUB_ENV
	-
	name: On release \| extract release tag
	if: needs.detect-change.outputs.update == 'true'
	run: echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
	-
	name: Set up QEMU
	uses: docker/setup-qemu-action@v2
	-
	name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2
	-
	name: Extract build & environment variables
	run: \|
	echo "DOCKERFILE_PATH=$(dirname $(find . -name Dockerfile))" >> $GITHUB_ENV
	echo "ARCH=$(echo ${{ matrix.arch }} \| sed -e 's\|x64\|amd64\|')" >> $GITHUB_ENV
	-
	name: Download artifact \| ${{ matrix.arch }} musl-libc based binary tarball
	uses: actions/download-artifact@v3
	with:
	name: eturnal-${{ env.TAG_VERSION }}-linux-musl-${{ matrix.arch }}.tar.gz
	-
	name: Log in to ${{ env.REGISTRY }}
	uses: docker/login-action@v2
	with:
	registry: ${{ env.REGISTRY }}
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}
	-
	name: Docker meta
	id: meta
	uses: docker/metadata-action@v4
	with:
	images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
	-
	name: Build and push by digest
	id: build
	uses: docker/build-push-action@v3
	with:
	build-args: \|
	METHOD=package
	VERSION=${{ env.TAG_VERSION }}
	REPOSITORY=https://github.com/${{ github.repository }}.git
	context: .
	file: ${{ env.DOCKERFILE_PATH }}/Dockerfile
	platforms: linux/${{ env.ARCH }}
	labels: ${{ steps.meta.outputs.labels }}
	outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
	-
	name: Export digest \| ${{ matrix.arch }}
	run: \|
	mkdir -p /tmp/digests
	digest="${{ steps.build.outputs.digest }}"
	touch "/tmp/digests/${digest#sha256:}"
	-
	name: Upload digest \| ${{ matrix.arch }}
	uses: actions/upload-artifact@v3
	with:
	name: digests
	path: /tmp/digests/*
	if-no-files-found: error
	retention-days: 1

	################################################################################
	#' build container images with local source files
	build-ctr:
	name: ${{ matrix.arch }} - build & publish container image (non-binary-based)
	runs-on: ubuntu-latest
	strategy:
	matrix:
	arch: [386, arm/v7, ppc64le, s390x]
	fail-fast: false
	needs: [detect-change]
	steps:
	-
	name: Check out repository code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	-
	name: On push master \| extract git version and define build mode
	if: needs.detect-change.outputs.update == 'false'
	run: \|
	echo "VSN=$(git describe --tag)" >> $GITHUB_ENV
	echo "TAG_VERSION=$(echo "$(./scripts/get-version)" \| sed -e 's\|+\|-\|')" >> $GITHUB_ENV
	echo "SRC=local" >> $GITHUB_ENV
	echo "OTP_VSN=$(awk '/^otp_vsn=/ {{gsub(/[^0-9.]/, ""); print}}' scripts/make-binaries)" >> $GITHUB_ENV
	-
	name: On release \| extract git version and define build mode
	if: needs.detect-change.outputs.update == 'true'
	run: \|
	eturnal_vsn=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})
	echo "VSN=$eturnal_vsn" >> $GITHUB_ENV
	echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV

	# define erlang/OTP version
	echo "OTP_VSN=$(wget -O - https://raw.githubusercontent.com/${{ github.repository }}/$eturnal_vsn/scripts/make-binaries \
	\| awk '/^otp_vsn=/ {{gsub(/[^0-9.]/, ""); print}}')" >> $GITHUB_ENV

	# check whether to build from archive or git, because git repo is HEAD here
	# hence, we would not build the specifc release when updating the ctr image
	pkgrel=$(awk 'END{gsub("-", " "); print $2}' ${{ env.PKGREL_FILE }})
	if [ "$pkgrel" != 'r0' ]
	then echo "SRC=web" >> $GITHUB_ENV
	else echo "SRC=local" >> $GITHUB_ENV
	fi
	-
	name: Set up QEMU
	uses: docker/setup-qemu-action@v2
	-
	name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2
	-
	name: Extract build & environment variables
	run: \|
	echo "DOCKERFILE_PATH=$(dirname $(find . -name Dockerfile))" >> $GITHUB_ENV
	# Check if rebar3 common test should be performed
	# fix for slow architectures was introduces after version 1.10.1
	if dpkg --compare-versions ${{ env.TAG_VERSION }} le "1.10.1"
	then echo "REBAR_CT=false" >> $GITHUB_ENV
	else echo "REBAR_CT=true" >> $GITHUB_ENV
	fi
	-
	name: Log in to ${{ env.REGISTRY }}
	uses: docker/login-action@v2
	with:
	registry: ${{ env.REGISTRY }}
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}
	-
	name: Docker meta
	id: meta
	uses: docker/metadata-action@v4
	with:
	images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
	-
	name: Build and push by digest
	id: build
	uses: docker/build-push-action@v3
	with:
	build-args: \|
	SOURCE=${{ env.SRC }}
	VERSION=${{ env.VSN }}
	OTP_VSN=${{ env.OTP_VSN }}
	REPOSITORY=https://github.com/${{ github.repository }}.git
	REBAR_CT=${{ env.REBAR_CT }}
	context: .
	file: ${{ env.DOCKERFILE_PATH }}/Dockerfile
	platforms: linux/${{ matrix.arch }}
	labels: ${{ steps.meta.outputs.labels }}
	outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
	-
	name: Export digest \| ${{ matrix.arch }}
	run: \|
	mkdir -p /tmp/digests
	digest="${{ steps.build.outputs.digest }}"
	touch "/tmp/digests/${digest#sha256:}"
	-
	name: Upload digest \| ${{ matrix.arch }}
	uses: actions/upload-artifact@v3
	with:
	name: digests
	path: /tmp/digests/*
	if-no-files-found: error
	retention-days: 1

	################################################################################
	## merge single images to only advertise one image plus tag
	publish:
	name: ${{ matrix.registry }} - publish image manifest
	runs-on: ubuntu-latest
	strategy:
	matrix:
	registry: [ghcr.io, docker.io]
	fail-fast: false
	if: github.event_name != 'pull_request' && github.event_name != 'schedule'
	needs: [detect-change, build-musl-binary-archives, build-ctr-binary-based, build-ctr]
	steps:
	-
	name: Check out repository code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	-
	name: On push master \| extract git version and define build variants
	if: needs.detect-change.outputs.update == 'false'
	run: \|
	echo "TAG_VERSION=$(echo "$(./scripts/get-version)" \| sed -e 's\|+\|-\|')" >> $GITHUB_ENV
	-
	name: On release \| extract release tags to be published/updated
	if: needs.detect-change.outputs.update == 'true'
	run: \|
	echo "TAG_PKGREL=$(awk 'END{print}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
	echo "TAG_VERSION=$(awk 'END{gsub("-", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
	echo "TAG_MINOR=$(awk 'END{gsub("\\.", " "); print $1"."$2}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
	echo "TAG_MAJOR=$(awk 'END{gsub("\\.", " "); print $1}' ${{ env.PKGREL_FILE }})" >> $GITHUB_ENV
	-
	name: Log in to ${{ matrix.registry }}
	uses: docker/login-action@v2
	if: \|
	( matrix.registry == 'docker.io'
	&& github.repository_owner == 'processone' )
	\|\| matrix.registry == 'ghcr.io'
	with:
	registry: ${{ matrix.registry }}
	username: ${{ (matrix.registry == 'docker.io'
	&& secrets.DOCKERHUB_USERNAME)
	\|\| github.repository_owner }}
	password: ${{ (matrix.registry == 'docker.io'
	&& secrets.DOCKERHUB_TOKEN)
	\|\| secrets.GITHUB_TOKEN }}
	-
	name: Download digests
	uses: actions/download-artifact@v3
	with:
	name: digests
	path: /tmp/digests
	-
	name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2
	-
	name: On push master \| Docker meta
	id: meta
	if: needs.detect-change.outputs.update == 'false'
	uses: docker/metadata-action@v4
	with:
	images: ${{ matrix.registry }}/${{ (matrix.registry == 'docker.io'
	&& env.DOCKERHUB_REPO)
	\|\| env.IMAGE_NAME }}
	tags: \|
	edge
	-
	name: On release \| Docker meta
	id: meta-rel
	if: needs.detect-change.outputs.update == 'true'
	uses: docker/metadata-action@v4
	with:
	images: ${{ matrix.registry }}/${{ (matrix.registry == 'docker.io'
	&& env.DOCKERHUB_REPO)
	\|\| env.IMAGE_NAME }}
	tags: \|
	latest
	${{ env.TAG_PKGREL }}
	${{ env.TAG_VERSION }}
	${{ env.TAG_MINOR }}
	${{ env.TAG_MAJOR }}
	-
	name: Create manifest list and push
	if: \|
	( matrix.registry == 'docker.io'
	&& github.repository_owner == 'processone' )
	\|\| matrix.registry == 'ghcr.io'
	working-directory: /tmp/digests
	run: \|
	docker buildx imagetools create $(jq -r '"-t " + (.tags \| join(" -t "))' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
	$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
	-
	name: Inspect image
	if: matrix.registry == 'ghcr.io'
	run: \|
	docker buildx imagetools inspect ${{ matrix.registry }}/${{ env.IMAGE_NAME }}:${{ steps.meta-rel.outputs.version }} \|\|
	docker buildx imagetools inspect ${{ matrix.registry }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add a macos CI as well as a homebrew formula #5

Workflow file

Add a macos CI as well as a homebrew formula #5

Jobs

Run details

Workflow file for this run