Skip to content

Commit

Permalink
fix: allow passing rayon threads when building aztec images (#9096)
Browse files Browse the repository at this point in the history
also, do not do parallel vk generation for mock circuits
  • Loading branch information
just-mitch authored Oct 9, 2024
1 parent b62235e commit 05de539
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 15 deletions.
3 changes: 1 addition & 2 deletions noir-projects/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,10 @@ build-mock-protocol-circuits:

ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
RUN echo "building with num threads $RAYON_NUM_THREADS"
RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo ./bootstrap.sh
RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
SAVE ARTIFACT mock-protocol-circuits

build:
ARG RAYON_NUM_THREADS
FROM +source
BUILD +build-contracts
BUILD +build-protocol-circuits
Expand Down
16 changes: 13 additions & 3 deletions noir-projects/mock-protocol-circuits/bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,19 @@ BB_HASH=${BB_HASH:-$(cd ../../ && git ls-tree -r HEAD | grep 'barretenberg/cpp'
echo Using BB hash $BB_HASH
mkdir -p "./target/keys"

for pathname in "./target"/*.json; do
BB_HASH=$BB_HASH node ../scripts/generate_vk_json.js "$pathname" "./target/keys" &
done
PARALLEL_VK=${PARALLEL_VK:-true}

if [[ $PARALLEL_VK == "true" ]]; then
echo "Generating vks in parallel..."
for pathname in "./target"/*.json; do
BB_HASH=$BB_HASH node ../scripts/generate_vk_json.js "$pathname" "./target/keys" &
done
else
echo "Generating vks sequentially..."
for pathname in "./target"/*.json; do
BB_HASH=$BB_HASH node ../scripts/generate_vk_json.js "$pathname" "./target/keys"
done
fi

for job in $(jobs -p); do
wait $job || exit 1
Expand Down
11 changes: 1 addition & 10 deletions yarn-project/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,11 @@ deps:
RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules

build:
ARG RAYON_NUM_THREADS
# Prefetch targets to not wait for +deps.
BUILD ../barretenberg/cpp/+build
BUILD ../barretenberg/ts/+build
BUILD ../noir/+nargo
BUILD ../noir-projects/+build
BUILD --pass-args ../noir-projects/+build
BUILD ../l1-contracts/+build
BUILD ../barretenberg/ts/+build
BUILD ../noir/+packages
Expand All @@ -58,7 +57,6 @@ build-dev:
SAVE ARTIFACT /usr/src /usr/src

cli-base:
ARG RAYON_NUM_THREADS
FROM +build
# Remove a bunch of stuff that we don't need that takes up space.
RUN rm -rf \
Expand Down Expand Up @@ -147,14 +145,12 @@ export-cli-wallet:
SAVE IMAGE --push aztecprotocol/cli-wallet:${DIST_TAG}${ARCH:+-$ARCH}

aztec-prod:
ARG RAYON_NUM_THREADS
FROM +cli-base
RUN yarn workspaces focus @aztec/aztec --production && yarn cache clean
COPY --dir +rollup-verifier-contract/usr/src/bb /usr/src
SAVE ARTIFACT /usr/src /usr/src

aztec:
ARG RAYON_NUM_THREADS
FROM ubuntu:noble
RUN apt update && apt install nodejs curl jq -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
COPY +aztec-prod/usr/src /usr/src
Expand Down Expand Up @@ -194,7 +190,6 @@ export-aztec-faucet:

# We care about creating a slimmed down e2e image because we have to serialize it from earthly to docker for running.
end-to-end-prod:
ARG RAYON_NUM_THREADS
FROM +cli-base
RUN yarn workspaces focus @aztec/end-to-end @aztec/cli-wallet --production && yarn cache clean
COPY --dir +rollup-verifier-contract/usr/src/bb /usr/src
Expand Down Expand Up @@ -228,7 +223,6 @@ end-to-end-base:
RUN ln -s /usr/src/yarn-project/.yarn/releases/yarn-3.6.3.cjs /usr/local/bin/yarn

end-to-end:
ARG RAYON_NUM_THREADS
FROM +end-to-end-base

COPY +anvil/anvil /opt/foundry/bin/anvil
Expand All @@ -242,12 +236,10 @@ scripts-prod:
SAVE ARTIFACT /usr/src /usr/src

all:
ARG RAYON_NUM_THREADS
BUILD +aztec
BUILD +end-to-end

export-aztec:
ARG RAYON_NUM_THREADS
ARG EARTHLY_GIT_HASH
FROM +aztec
SAVE IMAGE aztecprotocol/aztec:$EARTHLY_GIT_HASH
Expand All @@ -264,7 +256,6 @@ export-end-to-end:
SAVE IMAGE aztecprotocol/end-to-end:$EARTHLY_GIT_HASH

export-e2e-test-images:
ARG RAYON_NUM_THREADS
BUILD +export-aztec
BUILD +export-end-to-end

Expand Down

0 comments on commit 05de539

Please sign in to comment.