Rework build to avoid emulation

.github/workflows/pr.yml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - Dockerfile.tmpl
+      - Dockerfile.build.tmpl
       - .github/workflows/**
       - lib/**
       - Makefile
@@ -60,10 +61,27 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Generate Dockerfile
-        run: make Dockerfile
+        run: make Dockerfile Dockerfile.build
 
-      - # This will only build the container image, not publish it
-        name: Build container image and export to Docker
+      - name: Build auxiliary image and export to docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.build
+          push: false
+          load: true
+          tags: binaries:local
+          build-args: |
+            TARGET_GOOS=linux
+            TARGET_GOARCH=amd64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+      - name: Capture binaries
+        run: |
+          ./scripts/build-os-arch --image binaries:local --no-build linux amd64
+
+      - name: Build container image and export to Docker
         uses: docker/build-push-action@v5
         with:
           context: .

.github/workflows/publish.yml

@@ -12,9 +12,83 @@ on:
   workflow_dispatch:
 
 jobs:
+  build:
+    name: Build binaries
+    runs-on: ubuntu-latest-8-cores
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+
+      - name: Unshallow
+        run: git fetch --prune --tags --unshallow
+
+      - name: Describe the current state
+        run: git describe --tags --always
+
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ghcr.io/${{ github.repository }}
+          tags: |
+            type=raw,value=sha-${{ github.sha }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Package Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate Dockerfile
+        run: make Dockerfile Dockerfile.build
+
+      - name: Build auxiliary image and export to docker (linux/amd64)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.build
+          push: false
+          load: true
+          tags: binaries:local-linux-amd64
+          build-args: |
+            TARGET_GOOS=linux
+            TARGET_GOARCH=amd64
+
+      - name: Build auxiliary image and export to docker (linux/arm64)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.build
+          push: false
+          load: true
+          tags: binaries:local-linux-arm64
+          build-args: |
+            TARGET_GOOS=linux
+            TARGET_GOARCH=arm64
+
+      - name: Capture binaries
+        run: |
+          ./scripts/build-os-arch --image binaries:local-linux-amd64 --no-build linux amd64
+          ./scripts/build-os-arch --image binaries:local-linux-arm64 --no-build linux arm64
+          tar -czf dist/binaries.tar.gz dist/linux-amd64 dist/linux-arm64
+
+      - name: Archive binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries
+          path: dist/binaries.tar.gz
+          retention-days: 1
+
   publish:
     name: Publish container images
     runs-on: ubuntu-latest-8-cores
+    needs: build
     steps:
       - uses: actions/checkout@v4
 
@@ -46,16 +120,8 @@ jobs:
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-        with:
-          config: .github/workflows/buildkitd.toml
-
-      - name: Cache Docker layers
-        uses: actions/cache@v4
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+        # with:
+        #   config: .github/workflows/buildkitd.toml
 
       - name: Login to GitHub Package Registry
         uses: docker/login-action@v3
@@ -64,24 +130,36 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Retrieve binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: binaries
+
+      - name: Load binaries
+        run: |
+          find -type f -print0 | xargs -0r ls -ld
+          tar -xzf binaries.tar.gz
+
+      - name: Build image (linux/amd64)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          push: false
+          load: true
+          platforms: linux/amd64
+          tags: test:local
+
+      - name: Test image
+        run: |
+          docker run --rm test:local image-test
+
       - name: Build and push image
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
-
-      -
-        # Temp fix
-        # https://github.com/docker/build-push-action/issues/252
-        # https://github.com/moby/buildkit/issues/1896
-        name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-        shell: bash

.gitignore

@@ -1 +1,2 @@
 Dockerfile
+Dockerfile.build

Dockerfile.build.tmpl

@@ -0,0 +1,195 @@
+FROM docker.io/library/golang:1.22.6 AS go
+	COPY lib/go.env /usr/local/go/
+	RUN mkdir -p /build/bin
+
+FROM go AS go_jsonnet
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]
+
+FROM go AS wire
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/wire/cmd/[email protected]
+
+FROM go AS bingo
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/bwplotka/[email protected]
+
+FROM go AS lefthook
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/evilmartians/[email protected]
+
+FROM go AS dockerfile_json
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN git clone --depth 1 --branch 1.0.8 https://github.com/keilerkonzept/dockerfile-json dockerfile-json && \
+		cd dockerfile-json && \
+		env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install .
+
+FROM go AS enumer
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/dmarkham/[email protected]
+
+FROM go AS protoc_gen_go
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install google.golang.org/protobuf/cmd/[email protected]
+
+FROM go AS protoc_gen_go_grpc
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install google.golang.org/grpc/cmd/[email protected]
+
+FROM go AS buf
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/bufbuild/buf/cmd/[email protected]
+
+FROM go AS mage
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN git clone --depth 1 --branch v1.15.0 https://github.com/magefile/mage mage && \
+		cd mage && \
+		mkdir -p /host/bin /build/bin/${TARGET_GOOS}-${TARGET_GOARCH} && \
+		env GOPATH=/host go run bootstrap.go && \
+		env /host/bin/mage -compile /build/bin/${TARGET_GOOS}-${TARGET_GOARCH}/mage -goos ${TARGET_GOOS} -goarch ${TARGET_GOARCH}
+
+FROM go AS nilaway
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install go.uber.org/nilaway/cmd/[email protected]
+
+FROM go AS grizzly
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/grafana/grizzly/cmd/[email protected]
+
+FROM go AS semversort
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/whereswaldon/[email protected]
+
+FROM go AS golangci_lint
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/golangci/golangci-lint/cmd/[email protected]
+
+FROM go AS git_chglog
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/git-chglog/git-chglog/cmd/[email protected]
+
+FROM go AS gotestsum
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install gotest.tools/[email protected]
+
+FROM go AS xk6
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	# The grafana/xk6 image only exists for amd64, so we need to build it for
+	# the target architecture.
+	RUN mkdir -p /host/bin /build/bin/${TARGET_GOOS}-${TARGET_GOARCH} && \
+		env GOPATH=/host go install go.k6.io/xk6/cmd/[email protected] && \
+		env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install go.k6.io/xk6/cmd/[email protected]
+
+FROM xk6 AS k6
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	# The grafana/k6 image only exists for amd64, so we need to build it for
+	# the architecture we are targeting. The simplest way to build k6 is to
+	# (ab)use xk6 to build a binary without any extensions. In the future, if
+	# we wanted additional extensions, this is the place to add them.
+	RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} /host/bin/xk6 build v0.52.0 --output /build/bin/${TARGET_GOOS}-${TARGET_GOARCH}/k6
+
+FROM docker.io/library/debian:stable-slim AS skopeo
+	ARG TARGET_GOOS
+	ARG TARGET_GOARCH
+
+	COPY --from=go /usr/local/go /usr/local/go
+
+	ENV PATH="/usr/local/go/bin:${PATH}"
+
+	RUN apt-get update && \
+	    apt-get install -y \
+	    build-essential \
+	    libgpgme-dev \
+	    libassuan-dev \
+	    libdevmapper-dev \
+	    pkg-config \
+	    git
+
+	# skopeo is used to inspect container registries. This can be used to
+	# inspect the available versions without pulling the repos.
+	RUN git clone https://github.com/containers/skopeo && \
+	    cd skopeo && \
+	    git checkout "v1.16.0" && \
+	    make GOPATH=/build DISABLE_DOCS=1 bin/skopeo.${TARGET_GOOS}.${TARGET_GOARCH} && \
+	    mkdir -p /build/bin && \
+	    cp bin/skopeo.${TARGET_GOOS}.${TARGET_GOARCH} /build/bin/skopeo
+
+FROM docker.io/library/debian:stable-slim AS final
+	RUN mkdir -p /dist
+
+	COPY --from=bingo /build/bin/* /dist/
+
+	COPY --from=buf /build/bin/* /dist/
+
+	COPY --from=dockerfile_json /build/bin/* /dist/
+
+	COPY --from=enumer /build/bin/* /dist/
+
+	COPY --from=git_chglog /build/bin/* /dist/
+
+	COPY --from=go_jsonnet /build/bin/* /dist/
+
+	COPY --from=golangci_lint /build/bin/* /dist/
+
+	COPY --from=gotestsum /build/bin/* /dist/
+
+	COPY --from=grizzly /build/bin/* /dist/
+
+	COPY --from=k6 /build/bin/* /dist/
+
+	COPY --from=lefthook /build/bin/* /dist/
+
+	COPY --from=mage /build/bin/* /dist/
+
+	COPY --from=nilaway /build/bin/* /dist/
+
+	COPY --from=protoc_gen_go /build/bin/* /dist/
+
+	COPY --from=protoc_gen_go_grpc /build/bin/* /dist/
+
+	COPY --from=semversort /build/bin/* /dist/
+
+	COPY --from=skopeo /build/bin/* /dist/
+
+	COPY --from=wire /build/bin/* /dist/
+
+	COPY --from=xk6 /build/bin/* /dist/