Skip to content

Commit

Permalink
Rework build to avoid emulation
Browse files Browse the repository at this point in the history
I'm sick of issues related to emulation, the most recent one being that
stuff times out for some reason (and it's not necessarily that the build
is slow).

It would be much easier to build natively, but since that doesn't seem
to be an option yet, change things so that binaries are cross-compiled.

Signed-off-by: Marcelo E. Magallon <[email protected]>
  • Loading branch information
grafanarenovatebot[bot] authored and mem committed Aug 9, 2024
1 parent 6ffafbc commit 12743d7
Show file tree
Hide file tree
Showing 9 changed files with 368 additions and 144 deletions.
24 changes: 21 additions & 3 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ on:
- main
paths:
- Dockerfile.tmpl
- Dockerfile.build.tmpl
- .github/workflows/**
- lib/**
- Makefile
Expand Down Expand Up @@ -60,10 +61,27 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}

- name: Generate Dockerfile
run: make Dockerfile
run: make Dockerfile Dockerfile.build

- # This will only build the container image, not publish it
name: Build container image and export to Docker
- name: Build auxiliary image and export to docker
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile.build
push: false
load: true
tags: binaries:local
build-args: |
TARGET_GOOS=linux
TARGET_GOARCH=amd64
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max

- name: Capture binaries
run: |
./scripts/build-os-arch --image binaries:local --no-build linux amd64
- name: Build container image and export to Docker
uses: docker/build-push-action@v5
with:
context: .
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
Dockerfile
Dockerfile.build
195 changes: 195 additions & 0 deletions Dockerfile.build.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
FROM docker.io/library/golang:1.22.6 AS go
COPY lib/go.env /usr/local/go/
RUN mkdir -p /build/bin

FROM go AS go_jsonnet
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]
RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]
RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]
RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/go-jsonnet/cmd/[email protected]

FROM go AS wire
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/google/wire/cmd/[email protected]

FROM go AS bingo
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/bwplotka/[email protected]

FROM go AS lefthook
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/evilmartians/[email protected]

FROM go AS dockerfile_json
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN git clone --depth 1 --branch 1.0.8 https://github.com/keilerkonzept/dockerfile-json dockerfile-json && \
cd dockerfile-json && \
env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install .

FROM go AS enumer
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/dmarkham/[email protected]

FROM go AS protoc_gen_go
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install google.golang.org/protobuf/cmd/[email protected]

FROM go AS protoc_gen_go_grpc
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install google.golang.org/grpc/cmd/[email protected]

FROM go AS buf
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/bufbuild/buf/cmd/[email protected]

FROM go AS mage
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN git clone --depth 1 --branch v1.15.0 https://github.com/magefile/mage mage && \
cd mage && \
mkdir -p /host/bin /build/bin/${TARGET_GOOS}-${TARGET_GOARCH} && \
env GOPATH=/host go run bootstrap.go && \
env /host/bin/mage -compile /build/bin/${TARGET_GOOS}-${TARGET_GOARCH}/mage -goos ${TARGET_GOOS} -goarch ${TARGET_GOARCH}

FROM go AS nilaway
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install go.uber.org/nilaway/cmd/[email protected]

FROM go AS grizzly
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/grafana/grizzly/cmd/[email protected]

FROM go AS semversort
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/whereswaldon/[email protected]

FROM go AS golangci_lint
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/golangci/golangci-lint/cmd/[email protected]

FROM go AS git_chglog
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install github.com/git-chglog/git-chglog/cmd/[email protected]

FROM go AS gotestsum
ARG TARGET_GOOS
ARG TARGET_GOARCH

RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install gotest.tools/[email protected]

FROM go AS xk6
ARG TARGET_GOOS
ARG TARGET_GOARCH

# The grafana/xk6 image only exists for amd64, so we need to build it for
# the target architecture.
RUN mkdir -p /host/bin /build/bin/${TARGET_GOOS}-${TARGET_GOARCH} && \
env GOPATH=/host go install go.k6.io/xk6/cmd/[email protected] && \
env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} go install go.k6.io/xk6/cmd/[email protected]

FROM xk6 AS k6
ARG TARGET_GOOS
ARG TARGET_GOARCH

# The grafana/k6 image only exists for amd64, so we need to build it for
# the architecture we are targeting. The simplest way to build k6 is to
# (ab)use xk6 to build a binary without any extensions. In the future, if
# we wanted additional extensions, this is the place to add them.
RUN env GOPATH=/build GOOS=${TARGET_GOOS} GOARCH=${TARGET_GOARCH} /host/bin/xk6 build v0.52.0 --output /build/bin/${TARGET_GOOS}-${TARGET_GOARCH}/k6

FROM docker.io/library/debian:stable-slim AS skopeo
ARG TARGET_GOOS
ARG TARGET_GOARCH

COPY --from=go /usr/local/go /usr/local/go

ENV PATH="/usr/local/go/bin:${PATH}"

RUN apt-get update && \
apt-get install -y \
build-essential \
libgpgme-dev \
libassuan-dev \
libdevmapper-dev \
pkg-config \
git

# skopeo is used to inspect container registries. This can be used to
# inspect the available versions without pulling the repos.
RUN git clone https://github.com/containers/skopeo && \
cd skopeo && \
git checkout "v1.16.0" && \
make GOPATH=/build DISABLE_DOCS=1 bin/skopeo.${TARGET_GOOS}.${TARGET_GOARCH} && \
mkdir -p /build/bin && \
cp bin/skopeo.${TARGET_GOOS}.${TARGET_GOARCH} /build/bin/skopeo

FROM docker.io/library/debian:stable-slim AS final
RUN mkdir -p /dist

COPY --from=bingo /build/bin/* /dist/

COPY --from=buf /build/bin/* /dist/

COPY --from=dockerfile_json /build/bin/* /dist/

COPY --from=enumer /build/bin/* /dist/

COPY --from=git_chglog /build/bin/* /dist/

COPY --from=go_jsonnet /build/bin/* /dist/

COPY --from=golangci_lint /build/bin/* /dist/

COPY --from=gotestsum /build/bin/* /dist/

COPY --from=grizzly /build/bin/* /dist/

COPY --from=k6 /build/bin/* /dist/

COPY --from=lefthook /build/bin/* /dist/

COPY --from=mage /build/bin/* /dist/

COPY --from=nilaway /build/bin/* /dist/

COPY --from=protoc_gen_go /build/bin/* /dist/

COPY --from=protoc_gen_go_grpc /build/bin/* /dist/

COPY --from=semversort /build/bin/* /dist/

COPY --from=skopeo /build/bin/* /dist/

COPY --from=wire /build/bin/* /dist/

COPY --from=xk6 /build/bin/* /dist/
128 changes: 14 additions & 114 deletions Dockerfile.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -2,122 +2,13 @@ FROM docker.io/library/golang:{{ .data.go }} as go
COPY lib/go.env /usr/local/go/
RUN mkdir -p /build/bin

FROM go as go_jsonnet
RUN env GOBIN=/build/bin go install github.com/google/go-jsonnet/cmd/jsonnet@{{ .data.go_jsonnet }}
RUN env GOBIN=/build/bin go install github.com/google/go-jsonnet/cmd/jsonnetfmt@{{ .data.go_jsonnet }}
RUN env GOBIN=/build/bin go install github.com/google/go-jsonnet/cmd/jsonnet-deps@{{ .data.go_jsonnet }}
RUN env GOBIN=/build/bin go install github.com/google/go-jsonnet/cmd/jsonnet-lint@{{ .data.go_jsonnet }}
FROM scratch AS shellcheck
COPY --from=docker.io/koalaman/shellcheck:v0.10.0 /bin/shellcheck /build/bin/

FROM go as wire
# Add wire
RUN env GOBIN=/build/bin go install github.com/google/wire/cmd/wire@{{ .data.wire }}

FROM go as bingo
# Add bingo
RUN env GOBIN=/build/bin go install github.com/bwplotka/bingo@{{ .data.bingo }}

FROM go as lefthook
# Add lefthook
RUN env GOBIN=/build/bin go install github.com/evilmartians/lefthook@{{ .data.lefthook }}

FROM go as dockerfile_json
# Add dockerfile-json
RUN git clone --depth 1 --branch {{ .data.dockerfile_json }} https://github.com/keilerkonzept/dockerfile-json dockerfile-json && \
cd dockerfile-json && \
env GOBIN=/build/bin go install .

FROM go as enumer
# Add enumer
RUN env GOBIN=/build/bin go install github.com/dmarkham/enumer@{{ .data.enumer }}

FROM go as protoc_gen_go
# Add protoc-gen-go
RUN env GOBIN=/build/bin go install google.golang.org/protobuf/cmd/protoc-gen-go@{{ .data.protoc_gen_go }}

FROM go as protoc_gen_go_grpc
# Add protoc-gen-go-grpc
RUN env GOBIN=/build/bin go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@{{ .data.protoc_gen_go_grpc }}

FROM go as buf
# Add buf
RUN env GOBIN=/build/bin go install github.com/bufbuild/buf/cmd/buf@{{ .data.buf }}

FROM go as mage
# Add mage
RUN git clone --depth 1 --branch {{ .data.mage }} https://github.com/magefile/mage mage && \
cd mage && \
mkdir -p /build/bin && \
env GOBIN=/build/bin go run bootstrap.go

FROM go as nilaway
# Add nilaway
RUN env GOBIN=/build/bin go install go.uber.org/nilaway/cmd/nilaway@{{ .data.nilaway }}

FROM go as grizzly
# Add grizzly
RUN env GOBIN=/build/bin go install github.com/grafana/grizzly/cmd/grr@{{ .data.grizzly }}

FROM go as semversort
# Add semversort
RUN env GOBIN=/build/bin go install github.com/whereswaldon/semversort@{{ .data.semversort }}

FROM go as golangci_lint
# Add golangci-lint
RUN env GOBIN=/build/bin go install github.com/golangci/golangci-lint/cmd/golangci-lint@{{ .data.golangci_lint }}

FROM go as shellcheck
# Add shellcheck
COPY --from=docker.io/koalaman/shellcheck:{{ .data.shellcheck }} /bin/shellcheck /build/bin/

FROM go as git_chglog
# Add git-chglog
RUN env GOBIN=/build/bin go install github.com/git-chglog/git-chglog/cmd/git-chglog@{{ .data.git_chglog }}

FROM go as gotestsum
# Add gotestsum
RUN env GOBIN=/build/bin go install gotest.tools/gotestsum@{{ .data.gotestsum }}

FROM go as jq
# Add jq
COPY --from=ghcr.io/jqlang/jq:{{ .data.jq }} /jq /build/bin/

FROM go AS xk6
# The grafana/xk6 image only exists for amd64, so we need to build it for
# the target architecture.
RUN env GOBIN=/build/bin go install go.k6.io/xk6/cmd/xk6@{{ .data.xk6 }}

FROM xk6 AS k6
# The grafana/k6 image only exists for amd64, so we need to build it for
# the architecture we are targeting. The simplest way to build k6 is to
# (ab)use xk6 to build a binary without any extensions. In the future, if
# we wanted additional extensions, this is the place to add them.
RUN /build/bin/xk6 build {{ .data.k6 }} --output /build/bin/k6

FROM docker.io/library/debian:stable-slim as skopeo
COPY --from=go /usr/local/go /usr/local/go

ENV PATH="/usr/local/go/bin:${PATH}"

RUN apt-get update && \
apt-get install -y \
build-essential \
libgpgme-dev \
libassuan-dev \
libdevmapper-dev \
pkg-config \
git

# skopeo is used to inspect container registries. This can be used to
# inspect the available versions without pulling the repos.
RUN git clone https://github.com/containers/skopeo && \
cd skopeo && \
git checkout "{{ .data.skopeo }}" && \
make GOBIN=/build/bin DISABLE_DOCS=1 bin/skopeo && \
mkdir -p /build/bin && \
cp bin/skopeo /build/bin/
FROM scratch AS jq
COPY --from=ghcr.io/jqlang/jq:1.7.1 /jq /build/bin/

FROM docker.io/library/debian:stable-slim AS final

RUN apt-get update && \
apt-get install -y \
build-essential \
Expand All @@ -136,8 +27,17 @@ FROM docker.io/library/debian:stable-slim AS final

COPY --from=go /usr/local/go /usr/local/go

ARG TARGETOS
ARG TARGETARCH

ADD dist/${TARGETOS}-${TARGETARCH}/* /usr/local/bin/

COPY --from=shellcheck /build/bin/* /usr/local/bin/

COPY --from=jq /build/bin/* /usr/local/bin/

ENV PATH="/usr/local/go/bin:${PATH}"

{{ range $pkg, $info := .data }}
COPY --from={{ $pkg }} /build/bin/* /usr/local/bin/
### COPY --from={{ $pkg }} /build/bin/* /usr/local/bin/
{{ end }}
Loading

0 comments on commit 12743d7

Please sign in to comment.