From f6b3c7d548b0fcf8ce33b153d7d3258f21d0ad2a Mon Sep 17 00:00:00 2001 From: dubo-dubon-duponey Date: Fri, 15 Mar 2024 16:45:39 -0700 Subject: [PATCH] March Cake --- .github/workflows/CI.yml | 28 +++--- .gitignore | 5 +- ADVANCED.md | 12 +-- Dockerfile | 6 +- README.md | 5 - .../buildkit/buildctl/buildctl.cue | 2 +- cue.mod/pkg/duponey.cloud/scullery/icing.cue | 2 +- hack/build.sh | 93 +++++++++++-------- hack/helpers/install-tools.sh | 34 +++---- hack/helpers/start-buildkit.sh | 16 ++-- hack/lint.sh | 16 ++-- hack/recipe.cue | 2 +- hack/test.sh | 4 +- 13 files changed, 119 insertions(+), 106 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f1c3070..c069344 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -10,24 +10,24 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Checks-out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true - name: Enable cache - uses: actions/cache@v2.1.1 + uses: actions/cache@v4 with: - path: /cache + path: $GITHUB_WORKSPACE/cache key: cache - name: Prepare directories run: | - sudo mkdir -p "/cache/apt" - sudo mkdir -p "/cache/certs" - sudo mkdir -p "/cache/bin" - sudo chown -R 2000:root "/cache/apt" - sudo chown -R 2000:root "/cache/certs" - sudo chown -R $(id -u) "/cache/bin" + sudo mkdir -p "$GITHUB_WORKSPACE/cache/apt" + sudo mkdir -p "$GITHUB_WORKSPACE/cache/certs" + sudo mkdir -p "$GITHUB_WORKSPACE/cache/bin" + sudo chown -R 2000:root "$GITHUB_WORKSPACE/cache/apt" + sudo chown -R 2000:root "$GITHUB_WORKSPACE/cache/certs" + sudo chown -R $(id -u) "$GITHUB_WORKSPACE/cache/bin" - name: Start apt proxy run: | @@ -43,15 +43,15 @@ jobs: --env PORT=443 \ --env PORT_HTTP=80 \ --env ADDITIONAL_DOMAINS=*.debian.org \ - --volume "/cache/certs":/certs \ - --volume "/cache/apt":/data \ + --volume "$GITHUB_WORKSPACE/cache/certs":/certs \ + --volume "$GITHUB_WORKSPACE/cache/apt":/data \ docker.io/dubodubonduponey/aptutil:bullseye-2021-08-01 - name: test run: | # Set the path and install the tools - export PATH="/cache/bin:$PATH" - ./hack/helpers/install-tools.sh "/cache/bin" + export PATH="$HOME/bin:$PATH" + ./hack/helpers/install-tools.sh # Start buildkit bkaddr="$(./hack/helpers/start-buildkit.sh 2>/dev/null)" # Sanity check @@ -68,5 +68,5 @@ jobs: ./hack/test.sh \ --inject bk="$bkaddr" \ --inject ip="$(docker inspect apt-front | jq -rc .[0].NetworkSettings.Networks.bridge.IPAddress)" \ - --inject trust="$(sudo cat "/cache/certs/pki/authorities/local/root.crt")" \ + --inject trust="$(sudo cat "$GITHUB_WORKSPACE/cache/certs/pki/authorities/local/root.crt")" \ ".github/workflows/environment.cue" diff --git a/.gitignore b/.gitignore index a1be2dc..22894af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .DS_Store /.idea -*/cache/* -cache -buildctl.trace.json +cache/ +/xxx-* diff --git a/ADVANCED.md b/ADVANCED.md index 1768a20..a3214a2 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -3,20 +3,20 @@ ## Acknowledgements The Dockerfile of this project relies heavily on - * [debuerreotype](https://github.com/debuerreotype/debuerreotype), + * [debuerreotype](https://github.com/debuerreotype/debuerreotype) * [debootstrap](https://wiki.debian.org/Debootstrap) The build toolchain relies on: - * [qemu](https://www.qemu.org/), + * [qemu](https://www.qemu.org/) * [cue](https://cuelang.org/) - * [buildkit](https://github.com/moby/buildkit). + * [buildkit](https://github.com/moby/buildkit) ## Cue environment The build supports advanced environment control, allowing you to use apt mirrors (or proxy cache), complete with TLS, authentication and gpg signing. -To access these features, create a cue file, for example `env.cue`, as follow: +To access these features, create a cue file, for example `env.cue`, as follows: ```cue package cake @@ -123,9 +123,9 @@ If you experience any issue, please share configuration details. The `context/cache` folder is part of the build context. -As such, if it grows really big (with many different versions), assembling the final image will become slooooooow. +As such, if it grows real big (with many different versions), assembling the final image will become slooooooow. -It is recommended to clean-up this folder from older / useless versions from time to time to avoid such adverse side-effects. +It is recommended to clean up this folder from older / useless versions from time to time to avoid such adverse side-effects. You may also override the cue `input: context:` and `output: directory` to better control where these artifacts are going. diff --git a/Dockerfile b/Dockerfile index 2feae7d..2ce2bba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,8 +194,8 @@ FROM $FROM_IMAGE_RUNTIME SHELL ["/bin/bash", "-o", "errexit", "-o", "errtrace", "-o", "functrace", "-o", "nounset", "-o", "pipefail", "-c"] # What we want -ARG TARGET_SUITE="buster" -ARG TARGET_DATE="2020-07-01" +ARG TARGET_SUITE="bookworm" +ARG TARGET_DATE="2024-03-01" ARG TARGETPLATFORM # Load it! @@ -245,7 +245,7 @@ ONBUILD ARG PRELOAD_PACKAGES="" ONBUILD ARG UNLOAD_PACKAGES="" ONBUILD ARG L3="" -# hadolint ignore=DL3008 +# hadolint ignore=DL3008,SC2086 ONBUILD RUN --mount=type=secret,uid=100,id=CA \ --mount=type=secret,uid=100,id=CERTIFICATE \ --mount=type=secret,uid=100,id=KEY \ diff --git a/README.md b/README.md index 84584fc..598a5b2 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,7 @@ Features: * resulting images are in the range of 25MB * multi-architecture * amd64 - * 386 * arm64 - * arm/v7 - * arm/v6 - * s390x - * ppc64le ## Important diff --git a/cue.mod/pkg/duponey.cloud/buildkit/buildctl/buildctl.cue b/cue.mod/pkg/duponey.cloud/buildkit/buildctl/buildctl.cue index 7a74ef0..e91385a 100644 --- a/cue.mod/pkg/duponey.cloud/buildkit/buildctl/buildctl.cue +++ b/cue.mod/pkg/duponey.cloud/buildkit/buildctl/buildctl.cue @@ -72,7 +72,7 @@ import ( ["--local", "dockerfile=\(dockerfile)"] + ["--frontend", frontend] + - ["--trace", "buildctl.trace.json"] + + ["--trace", "cache/buildctl.trace.json"] + ["--opt", "filename=\(filename)"] + ["--local", "context=\(context)"] + diff --git a/cue.mod/pkg/duponey.cloud/scullery/icing.cue b/cue.mod/pkg/duponey.cloud/scullery/icing.cue index 6fbfcb2..fe17b30 100644 --- a/cue.mod/pkg/duponey.cloud/scullery/icing.cue +++ b/cue.mod/pkg/duponey.cloud/scullery/icing.cue @@ -64,7 +64,7 @@ import ( #Icing: { // XXX make this type more specific buildkit?: { - address?: string | * "docker-container://buildkitd" + address?: string | * "docker-container://dbdbdp-buildkit" name?: string ca?: types.#Path cert?: types.#Path diff --git a/hack/build.sh b/hack/build.sh index 0535755..0133f44 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -1,46 +1,61 @@ #!/usr/bin/env bash set -o errexit -o errtrace -o functrace -o nounset -o pipefail -cd "$(dirname "${BASH_SOURCE[0]}")/.." +# shellcheck source=/dev/null +root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)/../" +readonly root -PATH="$(pwd)/cache/bin:$PATH" -export PATH +# shellcheck source=/dev/null +BIN_LOCATION="${BIN_LOCATION:-$root/cache/bin}" . "$root/hack/helpers/install-tools.sh" -"./hack/helpers/install-tools.sh" "./cache/bin" +rm -f "$root/cache/buildctl.trace.json" -rm -f "./buildctl.trace.json" +# Build the cue invocation +params=(cue) +case "${1:-}" in + # Provisional + "--version") + exit + ;; + # Provisional + "--help") + exit + ;; + *) + cd "$root" + target=image + files=("$root/hack/recipe.cue" "$root/hack/helpers/cue_tool.cue") + isparam= + for i in "$@"; do + if [ "${i:0:2}" == "--" ]; then + params+=("$i") + isparam=true + elif [ "$isparam" == true ]; then + params+=("$i") + isparam= + elif [ "${i##*.}" == "cue" ]; then + files+=("$i") + else + target="$i" + fi + done + com=("${params[@]}") + com+=("$target") + com+=("${files[@]}") -com=(cue) -files=("./hack/recipe.cue" "./hack/helpers/cue_tool.cue") -isflagvalue= -for i in "$@"; do - if [ "${i:0:2}" == "--" ]; then - com+=("$i") - isflagvalue=true - elif [ "$isflagvalue" == true ]; then - com+=("$i") - isflagvalue= - elif [ "${i##*.}" == "cue" ]; then - files+=("$i") - else - target="$i" - fi -done -com+=("${target:-image}") -com+=("${files[@]}") - -echo "------------------------------------------------------------------" -for i in "${com[@]}"; do - if [ "${i:0:2}" == -- ]; then - >&2 printf " %s" "$i" - else - >&2 printf " %s\n" "$i" - fi -done -echo "------------------------------------------------------------------" -"${com[@]}" || { - cd - > /dev/null - >&2 printf "Execution failure" - exit 1 -} -cd - > /dev/null + echo "------------------------------------------------------------------" + for i in "${com[@]}"; do + if [ "${i:0:2}" == -- ]; then + >&2 printf " %s" "$i" + else + >&2 printf " %s\n" "$i" + fi + done + "${com[@]}" || { + cd - > /dev/null + echo "Execution failure" + exit 1 + } + cd - > /dev/null + ;; +esac diff --git a/hack/helpers/install-tools.sh b/hack/helpers/install-tools.sh index 258c349..0166b3a 100755 --- a/hack/helpers/install-tools.sh +++ b/hack/helpers/install-tools.sh @@ -1,27 +1,27 @@ #!/usr/bin/env bash set -o errexit -o errtrace -o functrace -o nounset -o pipefail -readonly SUITE=bullseye -readonly DATE=2023-01-01 +export SUITE=bookworm +export DATE=2024-02-20 + +export BIN_LOCATION="${BIN_LOCATION:-$HOME/bin}" +export PATH="$BIN_LOCATION:$PATH" readonly IMAGE_TOOLS="${IMAGE_TOOLS:-dubodubonduponey/tools:$(uname -s | grep -q Darwin && printf "macos" || printf "linux-dev")-$SUITE-$DATE}" -readonly SHELLCHECK_VERSION=0.8.0 -readonly HADOLINT_VERSION=2.10.0 + +export SHELLCHECK_VERSION=0.10.0 +export HADOLINT_VERSION=2.12.0 setup::tools(){ local location="$1" - mkdir -p "$location" - - local item - local missing - for item in cue buildctl docker hadolint shellcheck; do - command -v "$location/$item" > /dev/null || { - missing=true - break - } - done - - [ "${missing:-}" ] || return 0 + if command -v "$location/cue" > /dev/null && + command -v "$location/buildctl" > /dev/null && + command -v "$location/docker" > /dev/null && + command -v "$location/hadolint" > /dev/null && + command -v "$location/shellcheck" > /dev/null; then + return + fi + mkdir -p "$location" docker rm -f dubo-tools >/dev/null 2>&1 || true docker create --pull always --name dubo-tools "$IMAGE_TOOLS" bash > /dev/null docker cp dubo-tools:/boot/bin/cue "$location" @@ -40,4 +40,4 @@ setup::tools(){ rm -Rf ./shellcheck-v$SHELLCHECK_VERSION } -setup::tools "${1:-./cache/bin}" +setup::tools "$BIN_LOCATION" diff --git a/hack/helpers/start-buildkit.sh b/hack/helpers/start-buildkit.sh index 7d38b25..b1b98b3 100755 --- a/hack/helpers/start-buildkit.sh +++ b/hack/helpers/start-buildkit.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash set -o errexit -o errtrace -o functrace -o nounset -o pipefail -readonly SUITE=bullseye -readonly DATE=2023-01-01 -readonly IMAGE_BLDKT="${IMAGE_BLDKT:-docker.io/dubodubonduponey/buildkit:$SUITE-$DATE}" +export TAG=latest +readonly IMAGE_BLDKT="${IMAGE_BLDKT:-docker.io/dubodubonduponey/buildkit:$TAG}" setup::buildkit() { [ "$(docker container inspect -f '{{.State.Running}}' dbdbdp-buildkit 2>/dev/null)" == "true" ] || { @@ -11,20 +10,21 @@ setup::buildkit() { -p 4242:4242 \ --network host \ --name dbdbdp-buildkit \ - --env MDNS_ENABLED=true \ - --env MDNS_HOST=buildkit-machina \ - --env MDNS_NAME="Dubo Buildkit on la machina" \ + --env MOD_MDNS_ENABLED=true \ + --env MOD_MDNS_HOST=buildkit-machina \ + --env MOD_MDNS_NAME="Dubo Buildkit on la machina" \ --entrypoint buildkitd \ --user root \ --privileged \ "$IMAGE_BLDKT" docker exec --env QEMU_BINARY_PATH=/boot/bin/ dbdbdp-buildkit binfmt --install all + docker exec dbdbdp-buildkit mkdir /tmp/runtime } } setup::buildkit 1>&2 || { - printf >&2 "Something wrong with starting buildkit\n" + echo >&2 "Something wrong with starting buildkit" exit 1 } -printf "docker-container://dbdbdp-buildkit\n" +echo "docker-container://dbdbdp-buildkit" diff --git a/hack/lint.sh b/hack/lint.sh index 0621eb7..127b850 100755 --- a/hack/lint.sh +++ b/hack/lint.sh @@ -4,21 +4,25 @@ set -o errexit -o errtrace -o functrace -o nounset -o pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)/../" readonly root -"$root/hack/helpers/install-tools.sh" "$root/cache/bin" +# shellcheck source=/dev/null +BIN_LOCATION="${BIN_LOCATION:-$root/cache/bin}" . "$root/hack/helpers/install-tools.sh" # Ignore some hadolint warnings that do not make much sense -# DL3006 complains about unpinned images (which is not true, we are just using ARGs for that) -# DL3029 is about "dO nOT UsE --platform", which is really ludicrous +# DL3006 is about "dO nOT UsE --platform", which is really ludicrous +# DL3029 complains about unpinned images (which is not true, we are just using ARGs for that) # DL4006 is about setting pipefail (which we do, in our base SHELL) # DL3059 is about not having multiple successive RUN statements, and this is moronic # SC2039 is about array ref in POSIX shells (we are using bash, so) # SC2027 is about quotes inside quotes, and is moronic too -readonly hadolint_ignore=(--ignore DL3006) +# XXX For some hard to fathom reason, the CI reports errors that the local test does not - specifically SC3014 SC3054 SC3010, so, also ignoring these since we use bash +readonly hadolint_ignore=(--ignore DL3006 --ignore DL3029 --ignore DL4006 --ignore DL3059 --ignore SC2039 --ignore SC2027 --ignore SC3014 --ignore SC3054 --ignore SC3010) -if ! "$root/cache/bin/hadolint" "${hadolint_ignore[@]}" "$root"/*Dockerfile*; then +if ! hadolint "${hadolint_ignore[@]}" "$root"/*Dockerfile*; then printf >&2 "Failed linting on Dockerfile\n" exit 1 fi -find "$root" -iname "*.sh" -not -path "*debuerreotype*" -not -path "*cache*" -exec "$root/cache/bin/shellcheck" {} \; +while read -r line; do + shellcheck "$line" +done < <(find "$root" -iname "*.sh" -not -path "*debuerreotype*" -not -path "*cache*" -not -path "*xxx*" 2>/dev/null || true) diff --git a/hack/recipe.cue b/hack/recipe.cue index 7e338f1..b463fc4 100644 --- a/hack/recipe.cue +++ b/hack/recipe.cue @@ -15,7 +15,7 @@ cakes: { platforms: types.#Platforms | * [ types.#Platforms.#AMD64, types.#Platforms.#ARM64, - types.#Platforms.#V7, + // types.#Platforms.#V7, // types.#Platforms.#I386, // types.#Platforms.#V6, // types.#Platforms.#V5, diff --git a/hack/test.sh b/hack/test.sh index e6afa41..a01636d 100755 --- a/hack/test.sh +++ b/hack/test.sh @@ -9,7 +9,7 @@ if ! "$root/hack/build.sh" \ --inject registry="docker.io/dubodubonduponey" \ --inject progress=plain \ --inject date=2021-08-01 \ - --inject suite=bullseye \ + --inject suite=bookworm \ --inject platforms=linux/amd64,linux/arm64 \ debootstrap "$@"; then printf >&2 "Failed building tooling rootfs from online debian\n" @@ -22,7 +22,7 @@ if ! "$root/hack/build.sh" \ --inject registry="" \ --inject progress=plain \ --inject date=2021-08-01 \ - --inject suite=bullseye \ + --inject suite=bookworm \ --inject platforms=linux/amd64,linux/arm64 \ debootstrap "$@"; then printf >&2 "Failed building tooling rootfs from existing rootfs\n"