Manually download bitcoind/electrs for CI tests

Previously, we used the auto-download feature of the
`electrsd`/`bitcoind` crates. While convenient, they unnecessarily
introduced a lot of dependecies (`zip`, `zstd`, `time`, etc.) to our
test environment which needed pinning for the MSRV of 1.63.

Here, we introduce a new `no_download` config flag allowing us to
disable this auto-download feature in CI, where we now opt to download
the corresponding binaries manually. We keep the default-auto-download
as a convenience feature for running tests locally though.

.github/workflows/rust.yml

@@ -46,13 +46,18 @@ jobs:
           cargo update -p proptest --precise "1.2.0" --verbose # proptest 1.3.0 requires rustc 1.64.0
           cargo update -p reqwest --precise "0.11.20" --verbose # reqwest 0.11.21 broke 1.63.0 MSRV
           cargo update -p regex --precise "1.9.6" --verbose # regex 1.10.0 requires rustc 1.65.0
-          cargo update -p jobserver --precise "0.1.26" --verbose # jobserver 0.1.27 requires rustc 1.66.0
-          cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5" --verbose # zstd-sys 2.0.9+zstd.1.5.5 requires rustc 1.64.0
-          cargo update -p petgraph --precise "0.6.3" --verbose # petgraph v0.6.4, requires rustc 1.64 or newer
           cargo update -p home --precise "0.5.5" --verbose # home v0.5.9, requires rustc 1.70 or newer
       - name: Set RUSTFLAGS to deny warnings
         if: "matrix.toolchain == 'stable'"
         run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV"
+      - name: Download bitcoind/electrs and set environment variables
+        if: "matrix.platform != 'windows-latest'"
+        run: |
+          source ./scripts/download_bitcoind_electrs.sh
+          cp "$BITCOIND_EXE" "$HOME"/bitcoind
+          cp "$ELECTRS_EXE" "$HOME"/electrs
+          echo "BITCOIND_EXE=$HOME/bitcoind" >> "$GITHUB_ENV"
+          echo "ELECTRS_EXE=$HOME/electrs" >> "$GITHUB_ENV"
       - name: Build on Rust ${{ matrix.toolchain }}
         run: cargo build --verbose --color always
       - name: Build with UniFFI support on Rust ${{ matrix.toolchain }}
@@ -69,10 +74,12 @@ jobs:
         run: cargo check --release --features uniffi --verbose --color always
       - name: Test on Rust ${{ matrix.toolchain }}
         if: "matrix.platform != 'windows-latest'"
-        run: cargo test
+        run: |
+          RUSTFLAGS="--cfg no_download" cargo test
       - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
         if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
-        run: cargo test --features uniffi
+        run: |
+          RUSTFLAGS="--cfg no_download" cargo test --features uniffi
       - name: Check formatting on Rust ${{ matrix.toolchain }}
         if: matrix.check-fmt
         run: rustup component add rustfmt && cargo fmt --all -- --check

Cargo.toml

@@ -76,12 +76,17 @@ winapi = { version = "0.3", features = ["winbase"] }
 [dev-dependencies]
 lightning = { version = "0.0.119", features = ["std", "_test_utils"] }
 #lightning = { git = "https://github.com/lightningdevkit/rust-lightning", branch="main", features = ["std", "_test_utils"] }
-electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
 electrum-client = { version = "0.15.1", default-features = true }
 bitcoincore-rpc = { version = "0.17.0", default-features = false }
 proptest = "1.0.0"
 regex = "1.5.6"
 
+[target.'cfg(not(no_download))'.dev-dependencies]
+electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
+
+[target.'cfg(no_download)'.dev-dependencies]
+electrsd = { version = "0.26.0", features = ["legacy"] }
+
 [target.'cfg(cln_test)'.dev-dependencies]
 clightningrpc = { version = "0.3.0-beta.8", default-features = false }
 

scripts/download_bitcoind_electrs.sh

@@ -0,0 +1,43 @@
+# Our Esplora-based tests require `electrs` and `bitcoind`
+# binaries. Here, we download the binaries, validate them, and export their
+# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
+# `electrsd`/`bitcoind` crates in our tests.
+
+HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')"
+ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
+ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
+BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
+BITCOIND_VERSION="25.1"
+if [[ "$HOST_PLATFORM" == *linux* ]]; then
+	ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
+	ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
+	BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
+	BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
+elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
+	ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
+	ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
+	BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
+	BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
+else
+	echo "\n\nUnsupported platform: $HOST_PLATFORM Exiting.."
+	exit 1
+fi
+
+DL_TMP_DIR=$(mktemp -d)
+trap 'rm -rf -- "$DL_TMP_DIR"' EXIT
+
+pushd "$DL_TMP_DIR"
+ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
+curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
+echo "$ELECTRS_DL_HASH  $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
+unzip "$ELECTRS_DL_FILE_NAME"
+export ELECTRS_EXE="$DL_TMP_DIR"/electrs
+chmod +x "$ELECTRS_EXE"
+
+BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
+curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
+echo "$BITCOIND_DL_HASH  $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
+tar xzf "$BITCOIND_DL_FILE_NAME"
+export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
+chmod +x "$BITCOIND_EXE"
+popd