From 78a2acb65965155daac7024d5d0a02df191eaf8e Mon Sep 17 00:00:00 2001 From: Renan Santos Date: Thu, 6 Jun 2024 14:38:47 -0300 Subject: [PATCH] feat: update the Dockerfile with emulator dependencies - Adds linux.bin and rootfs.ext2 download to the emulator stage. - Adds a emulator-devel stage that install libcmt and xgenext2fs. - Configures the CI image to use CGO. - Refactors the Dockerfile to improve readability. --- .github/workflows/build.yml | 6 +- .github/workflows/clean-up-images.yml | 2 +- build/Dockerfile | 452 +++++++++++++++++--------- build/docker-bake.override.hcl | 4 +- build/docker-bake.platforms.hcl | 6 +- internal/node/machinehash_test.go | 3 +- 6 files changed, 308 insertions(+), 165 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f0b157164..ba1b16077 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,7 +140,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | - name=ghcr.io/cartesi/rollups-node-ci-base + name=ghcr.io/cartesi/rollups-node-ci tags: | type=semver,pattern={{version}} type=ref,event=branch @@ -162,7 +162,7 @@ jobs: ./docker-bake.hcl ${{ steps.docker_meta.outputs.bake-file }} ./docker-bake.platforms.hcl - targets: rollups-node-ci-base + targets: rollups-node-ci push: true project: ${{ vars.DEPOT_PROJECT }} workdir: build @@ -174,7 +174,7 @@ jobs: test-go: runs-on: ubuntu-22.04 container: - image: ghcr.io/cartesi/rollups-node-ci-base:${{needs.build-ci-base.outputs.output}} + image: ghcr.io/cartesi/rollups-node-ci:${{needs.build-ci-base.outputs.output}} needs: - build-ci-base steps: diff --git a/.github/workflows/clean-up-images.yml b/.github/workflows/clean-up-images.yml index d3d030c78..acccfeb41 100644 --- a/.github/workflows/clean-up-images.yml +++ b/.github/workflows/clean-up-images.yml @@ -17,7 +17,7 @@ jobs: matrix: image: - rollups-node - - rollups-node-ci-base + - rollups-node-ci steps: - uses: vlaurin/action-ghcr-prune@v0.6.0 with: diff --git a/build/Dockerfile b/build/Dockerfile index a65da65e6..6a9ffb7c1 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -4,19 +4,19 @@ # syntax=docker.io/docker/dockerfile:1 # This dockerfile contains multiple stages to build three final targets. -# The file was split in a section for each final target. -# Version config that should be set in the bake file. +# Version configuration that should be set in the bake file. ARG BASE_IMAGE ARG RUST_VERSION ARG GO_VERSION ARG FOUNDRY_NIGHTLY_VERSION -ARG MACHINE_EMULATOR_VERSION -ARG ROOTFS_VERSION -ARG LINUX_VERSION -ARG LINUX_KERNEL_VERSION ARG ROM_VERSION ARG ROLLUPS_NODE_VERSION +ARG MACHINE_EMULATOR_VERSION +ARG MACHINE_TOOLS_VERSION +ARG MACHINE_IMAGE_KERNEL_VERSION +ARG MACHINE_KERNEL_VERSION +ARG MACHINE_XGENEXT2FS_VERSION # Build directories. ARG SNAPSHOT_BUILD_PATH=/build/snapshot @@ -24,177 +24,320 @@ ARG DEVNET_BUILD_PATH=/build/devnet ARG RUST_BUILD_PATH=/build/rollups-node/rust ARG GO_BUILD_PATH=/build/rollups-node/go -# Runtime dir for the cartesi-machine snapshot. +# Runtime directory for the cartesi-machine snapshot. ARG SNAPSHOT_RUNTIME_PATH=/usr/share/cartesi/snapshot -#################################################################################################### -# STAGE: emulator-base +# ============================================================================= +# STAGE: emulator +# +# - Install ca-certificates and curl (setup). +# - Install the machine-emulator. +# - Download linux.bin. +# - Download rootfs.ext2. # -# This stage creates a base-image with the Cartesi machine emulator. -# The result is used as the base for the snapshot and the node targets. -# We do this instead of using the cartesi/machine-emulator image to have control over the distro -# used by the base image. -FROM ${BASE_IMAGE} as emulator-base +# NOTE: We do not use the cartesi/machine-emulator image to have control over +# the distro used by the base image. +# ============================================================================= + +FROM ${BASE_IMAGE} as emulator -# Install machine-emulator ARG MACHINE_EMULATOR_VERSION +ARG MACHINE_TOOLS_VERSION +ARG MACHINE_IMAGE_KERNEL_VERSION +ARG MACHINE_KERNEL_VERSION +ARG DEBIAN_FRONTEND=noninteractive + +# Install ca-certificates and curl (setup). +RUN < /dev/null + apt-get update + apt-get install -y --no-install-recommends \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin + # Cartesi Machine Emulator + SDK_URL=https://github.com/cartesi/machine-emulator-sdk +EOF + +# ============================================================================= # STAGE: snapshot-builder # -# This stage builds the snapshot using the machine emulator as base image. -FROM emulator-base as snapshot-builder - -# Download rootfs and linux. -# Add these files to the directories the cartesi-machine expects. -WORKDIR /usr/share/cartesi-machine/images/ -ARG TOOLS_VERSION -ARG LINUX_VERSION -ARG LINUX_KERNEL_VERSION -ADD https://github.com/cartesi/machine-emulator-tools/releases/download/v${TOOLS_VERSION}/rootfs-tools-v${TOOLS_VERSION}.ext2 rootfs.ext2 -ADD https://github.com/cartesi/image-kernel/releases/download/v${LINUX_VERSION}/linux-${LINUX_KERNEL_VERSION}.bin linux.bin - -# Generate snapshot with echo and store it. +# - Build an echo snapshot. +# +# DEPRECATED: this stage is going to be deleted, as the CI won't be generating +# machine snapshots in the future. +# ============================================================================= + +FROM emulator as snapshot-builder + WORKDIR /build ARG SNAPSHOT_BUILD_PATH RUN cartesi-machine \ --ram-length=128Mi \ --store=$SNAPSHOT_BUILD_PATH \ + --no-rollback \ -- "ioctl-echo-loop --vouchers=1 --notices=1 --reports=1 --verbose=1" +# ============================================================================= # STAGE: rollups-node-snapshot # -# This stage copies the image from the builder. -# We use the emulator as base image so we can easily create a container with a volume shared with -# the rollups-node container. -FROM emulator-base as rollups-node-snapshot +# DEPRECATED: this stage is going to be deleted, as the CI won't be generating +# machine snapshots in the future. +# ============================================================================= + +FROM emulator as rollups-node-snapshot # Copy image from the builder stage. ARG SNAPSHOT_BUILD_PATH ARG SNAPSHOT_RUNTIME_PATH WORKDIR ${SNAPSHOT_RUNTIME_PATH} -COPY --from=snapshot-builder --chown=cartesi:cartesi ${SNAPSHOT_BUILD_PATH} ${SNAPSHOT_RUNTIME_PATH} +COPY --from=snapshot-builder --chown=cartesi:cartesi \ + ${SNAPSHOT_BUILD_PATH} ${SNAPSHOT_RUNTIME_PATH} # Set dummy command. CMD /bin/bash -#################################################################################################### -# TARGET: rollups-node-devnet -# -# This target contains the Ethereum node that rollups node uses for testing. -# This target requires the machine-snapshot built in the snapshot-builder stage. - +# ============================================================================= # STAGE: devnet-base # -# This stage installs Foundry. +# - Install ca-certificates, curl, and git (setup). +# - Install Foundry from downloaded pre-compiled binaries. +# ============================================================================= + FROM ${BASE_IMAGE} as devnet-base -# Install system dependencies. +# Install ca-certificates, curl, and git (setup). ARG DEBIAN_FRONTEND=noninteractive RUN < /dev/null - apt-get update - apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -EOF \ No newline at end of file diff --git a/build/docker-bake.override.hcl b/build/docker-bake.override.hcl index d2204f478..69177bd88 100644 --- a/build/docker-bake.override.hcl +++ b/build/docker-bake.override.hcl @@ -21,6 +21,6 @@ target "rollups-node-devnet" { tags = ["${DOCKER_ORGANIZATION}/rollups-node-devnet:${TAG}"] } -target "rollups-node-ci-base" { - tags = ["${DOCKER_ORGANIZATION}/rollups-node-ci-base:${TAG}"] +target "rollups-node-ci" { + tags = ["${DOCKER_ORGANIZATION}/rollups-node-ci:${TAG}"] } diff --git a/build/docker-bake.platforms.hcl b/build/docker-bake.platforms.hcl index 18f87a522..f128f619e 100644 --- a/build/docker-bake.platforms.hcl +++ b/build/docker-bake.platforms.hcl @@ -4,6 +4,10 @@ target "docker-platforms" { platforms = [ "linux/amd64", - "linux/arm64" + # TODO: libarchive13 (required by xgenext2fs in the emulator-devel + # stage) is not available for arm64. We are temporarily disabling this + # platform now, but will come back to it before merging next/2.0 into + # main. + # "linux/arm64" ] } diff --git a/internal/node/machinehash_test.go b/internal/node/machinehash_test.go index c1aefc8fb..76dc1b24d 100644 --- a/internal/node/machinehash_test.go +++ b/internal/node/machinehash_test.go @@ -116,8 +116,7 @@ func mockMachineDir(hash string) (string, error) { return temp, nil } -// Generates a new Cartesi Machine snapshot in a temporary directory and returns -// its path +// Generates a new Cartesi Machine snapshot in a temporary directory and returns its path func createMachineSnapshot() (string, error) { tmpDir, err := os.MkdirTemp("", "") if err != nil {