From c6f90e1475970850f556a141ec7f5447f3ed7a35 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 18 Dec 2023 13:13:49 +0100 Subject: [PATCH] Manually download `bitcoind`/`electrs` for CI tests Previously, we used the auto-download feature of the `electrsd`/`bitcoind` crates. While convenient, they unnecessarily introduced a lot of dependecies (`zip`, `zstd`, `time`, etc.) to our test environment which needed pinning for the MSRV of 1.63. Here, we introduce a new `no_download` config flag allowing us to disable this auto-download feature in CI, where we now opt to download the corresponding binaries manually. We keep the default-auto-download as a convenience feature for running tests locally though. --- .github/workflows/rust.yml | 17 +++++++---- Cargo.toml | 7 ++++- scripts/download_bitcoind_electrs.sh | 43 ++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 6 deletions(-) create mode 100755 scripts/download_bitcoind_electrs.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a76e1e663..1b5ca7b43 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -46,13 +46,18 @@ jobs: cargo update -p proptest --precise "1.2.0" --verbose # proptest 1.3.0 requires rustc 1.64.0 cargo update -p reqwest --precise "0.11.20" --verbose # reqwest 0.11.21 broke 1.63.0 MSRV cargo update -p regex --precise "1.9.6" --verbose # regex 1.10.0 requires rustc 1.65.0 - cargo update -p jobserver --precise "0.1.26" --verbose # jobserver 0.1.27 requires rustc 1.66.0 - cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5" --verbose # zstd-sys 2.0.9+zstd.1.5.5 requires rustc 1.64.0 - cargo update -p petgraph --precise "0.6.3" --verbose # petgraph v0.6.4, requires rustc 1.64 or newer cargo update -p home --precise "0.5.5" --verbose # home v0.5.9, requires rustc 1.70 or newer - name: Set RUSTFLAGS to deny warnings if: "matrix.toolchain == 'stable'" run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" + - name: Download bitcoind/electrs and set environment variables + if: "matrix.platform != 'windows-latest'" + run: | + source ./scripts/download_bitcoind_electrs.sh + cp "$BITCOIND_EXE" "$HOME"/bitcoind + cp "$ELECTRS_EXE" "$HOME"/electrs + echo "BITCOIND_EXE=$HOME/bitcoind" >> "$GITHUB_ENV" + echo "ELECTRS_EXE=$HOME/electrs" >> "$GITHUB_ENV" - name: Build on Rust ${{ matrix.toolchain }} run: cargo build --verbose --color always - name: Build with UniFFI support on Rust ${{ matrix.toolchain }} @@ -69,10 +74,12 @@ jobs: run: cargo check --release --features uniffi --verbose --color always - name: Test on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest'" - run: cargo test + run: | + RUSTFLAGS="--cfg no_download" cargo test - name: Test with UniFFI support on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest' && matrix.build-uniffi" - run: cargo test --features uniffi + run: | + RUSTFLAGS="--cfg no_download" cargo test --features uniffi - name: Check formatting on Rust ${{ matrix.toolchain }} if: matrix.check-fmt run: rustup component add rustfmt && cargo fmt --all -- --check diff --git a/Cargo.toml b/Cargo.toml index cf1292a30..b831c1d75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,12 +76,17 @@ winapi = { version = "0.3", features = ["winbase"] } [dev-dependencies] lightning = { version = "0.0.119", features = ["std", "_test_utils"] } #lightning = { git = "https://github.com/lightningdevkit/rust-lightning", branch="main", features = ["std", "_test_utils"] } -electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] } electrum-client = { version = "0.15.1", default-features = true } bitcoincore-rpc = { version = "0.17.0", default-features = false } proptest = "1.0.0" regex = "1.5.6" +[target.'cfg(not(no_download))'.dev-dependencies] +electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] } + +[target.'cfg(no_download)'.dev-dependencies] +electrsd = { version = "0.26.0", features = ["legacy"] } + [target.'cfg(cln_test)'.dev-dependencies] clightningrpc = { version = "0.3.0-beta.8", default-features = false } diff --git a/scripts/download_bitcoind_electrs.sh b/scripts/download_bitcoind_electrs.sh new file mode 100755 index 000000000..fcdb31891 --- /dev/null +++ b/scripts/download_bitcoind_electrs.sh @@ -0,0 +1,43 @@ +# Our Esplora-based tests require `electrs` and `bitcoind` +# binaries. Here, we download the binaries, validate them, and export their +# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the +# `electrsd`/`bitcoind` crates in our tests. + +HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')" +ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases" +ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254" +BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/" +BITCOIND_VERSION="25.1" +if [[ "$HOST_PLATFORM" == *linux* ]]; then + ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip + ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1" + BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz + BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b" +elif [[ "$HOST_PLATFORM" == *darwin* ]]; then + ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip + ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd" + BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz + BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1" +else + echo "\n\nUnsupported platform: $HOST_PLATFORM Exiting.." + exit 1 +fi + +DL_TMP_DIR=$(mktemp -d) +trap 'rm -rf -- "$DL_TMP_DIR"' EXIT + +pushd "$DL_TMP_DIR" +ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME" +curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL" +echo "$ELECTRS_DL_HASH $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c +unzip "$ELECTRS_DL_FILE_NAME" +export ELECTRS_EXE="$DL_TMP_DIR"/electrs +chmod +x "$ELECTRS_EXE" + +BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME" +curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL" +echo "$BITCOIND_DL_HASH $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c +tar xzf "$BITCOIND_DL_FILE_NAME" +export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind +chmod +x "$BITCOIND_EXE" +popd