Merge current solution wip (dunxen branch)

.github/workflows/build.yml

@@ -22,7 +22,6 @@ jobs:
         include:
           - toolchain: stable
             platform: ubuntu-latest
-            coverage: true
           # 1.48.0 is the MSRV for all crates except lightning-transaction-sync and Win/Mac
           - toolchain: 1.48.0
             platform: ubuntu-latest
@@ -50,46 +49,31 @@ jobs:
         run: |
           sudo apt-get -y install shellcheck
           shellcheck ci/ci-tests.sh
-      - name: Run CI script with coverage generation
-        if: matrix.coverage
-        shell: bash # Default on Winblows is powershell
-        run: LDK_COVERAGE_BUILD=true ./ci/ci-tests.sh
       - name: Run CI script
-        if: "!matrix.coverage"
         shell: bash # Default on Winblows is powershell
         run: ./ci/ci-tests.sh
-      - name: Install deps for kcov
-        if: matrix.coverage
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install binutils-dev libcurl4-openssl-dev zlib1g-dev libdw-dev libiberty-dev
-      - name: Install kcov
-        if: matrix.coverage
-        run: |
-          wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz
-          tar xzf master.tar.gz
-          cd kcov-master && mkdir build && cd build
-          cmake ..
-          make
-          make install DESTDIR=../../kcov-build
-          cd ../.. && rm -rf kcov-master master.tar.gz
-      - name: Generate coverage report
-        if: matrix.coverage
-        run: |
-          for file in target/debug/deps/lightning*; do
-            [ -x "${file}" ] || continue;
-            mkdir -p "target/cov/$(basename $file)";
-            ./kcov-build/usr/local/bin/kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$(basename $file)" "$file";
-          done
-      - name: Upload coverage
-        if: matrix.coverage
-        uses: codecov/codecov-action@v3
+
+  coverage:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v3
         with:
+          fetch-depth: 0
+      - name: Install Rust stable toolchain
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal
+      - name: Run tests with coverage generation
+        run: |
+          cargo install cargo-llvm-cov
+          export RUSTFLAGS="-Clink-dead-code -Coverflow-checks=off"
+          cargo llvm-cov --features rest-client,rpc-client,tokio,futures,serde --codecov --hide-instantiations --output-path=target/codecov.json
           # Could you use this to fake the coverage report for your PR? Sure.
           # Will anyone be impressed by your amazing coverage? No
           # Maybe if codecov wasn't broken we wouldn't need to do this...
-          token: f421b687-4dc2-4387-ac3d-dc3b2528af57
-          fail_ci_if_error: true
+          bash <(curl -s https://codecov.io/bash) -f target/codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57"
 
   benchmark:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -1,3 +1,178 @@
+# 0.0.117 - Oct 3, 2023 - "Everything but the Twelve Sinks"
+
+## API Updates
+ * `ProbabilisticScorer`'s internal models have been substantially improved,
+   including better decaying (#1789), a more granular historical channel
+   liquidity tracker (#2176) and a now-default option to make our estimate for a
+   channel's current liquidity nonlinear in the channel's capacity (#2547). In
+   total, these changes should result in improved payment success rates at the
+   cost of slightly worse routefinding performance.
+ * Support for custom TLVs for recipients of HTLCs has been added (#2308).
+ * Support for generating transactions for third-party watchtowers has been
+   added to `ChannelMonitor/Update`s (#2337).
+ * `KVStorePersister` has been replaced with a more generic and featureful
+   `KVStore` interface (#2472).
+ * A new `MonitorUpdatingPersister` is provided which wraps a `KVStore` and
+   implements `Persist` by writing differential updates rather than full
+   `ChannelMonitor`s (#2359).
+ * Batch funding of outbound channels is now supported using the new
+   `ChannelManager::batch_funding_transaction_generated` method (#2486).
+ * `ChannelManager::send_preflight_probes` has been added to probe a payment's
+   potential paths while a user is providing approval for a payment (#2534).
+ * Fully asynchronous `ChannelMonitor` updating is available as an alpha
+   preview. There remain a few known but incredibly rare race conditions which
+   may lead to loss of funds (#2112, #2169, #2562).
+ * `ChannelMonitorUpdateStatus::PermanentFailure` has been removed in favor of a
+   new `ChannelMonitorUpdateStatus::UnrecoverableError`. The new variant panics
+   on use, rather than force-closing a channel in an unsafe manner, which the
+   previous variant did (#2562). Rather than panicking with the new variant,
+   users may wish to use the new asynchronous `ChannelMonitor` updating using
+   `ChannelMonitorUpdateStatus::InProgress`.
+ * `RouteParameters::max_total_routing_fee_msat` was added to limit the fees
+   paid when routing, defaulting to 1% + 50sats when using the new
+   `from_payment_params_and_value` constructor (#2417, #2603, #2604).
+ * Implementations of `UtxoSource` are now provided in `lightning-block-sync`.
+   Those running with a full node should use this to validate gossip (#2248).
+ * `LockableScore` now supports read locking for parallel routefinding (#2197).
+ * `ChannelMonitor::get_spendable_outputs` was added to allow for re-generation
+   of `SpendableOutputDescriptor`s for a channel after they were provided via
+   `Event::SpendableOutputs` (#2609, #2624).
+ * `[u8; 32]` has been replaced with a `ChannelId` newtype for chan ids (#2485).
+ * `NetAddress` was renamed `SocketAddress` (#2549) and `FromStr` impl'd (#2134)
+ * For `no-std` users, `parse_onion_address` was added which creates a
+   `NetAddress` from a "...onion" string and port (#2134, #2633).
+ * HTLC information is now provided in `Event::PaymentClaimed::htlcs` (#2478).
+ * The success probability used in historical penalties when scoring is now
+   available via `historical_estimated_payment_success_probability` (#2466).
+ * `RecentPaymentDetails::*::payment_id` has been added (#2567).
+ * `Route` now contains a `RouteParameters` rather than a `PaymentParameters`,
+   tracking the original arguments passed to routefinding (#2555).
+ * `Balance::*::claimable_amount_satoshis` was renamed `amount_satoshis` (#2460)
+ * `*Features::set_*_feature_bit` have been added for non-custom flags (#2522).
+ * `channel_id` was added to `SpendableOutputs` events (#2511).
+ * `counterparty_node_id` and `channel_capacity_sats` were added to
+   `ChannelClosed` events (#2387).
+ * `ChannelMonitor` now implements `Clone` for `Clone`able signers (#2448).
+ * `create_onion_message` was added to build an onion message (#2583, #2595).
+ * `HTLCDescriptor` now implements `Writeable`/`Readable` (#2571).
+ * `SpendableOutputDescriptor` now implements `Hash` (#2602).
+ * `MonitorUpdateId` now implements `Debug` (#2594).
+ * `Payment{Hash,Id,Preimage}` now implement `Display` (#2492).
+ * `NodeSigner::sign_bolt12_invoice{,request}` were added for future use (#2432)
+
+## Backwards Compatibility
+ * Users migrating to the new `KVStore` can use a concatentation of
+   `[{primary_namespace}/[{secondary_namespace}/]]{key}` to build a key
+   compatible with the previous `KVStorePersister` interface (#2472).
+ * Downgrading after receipt of a payment with custom HTLC TLVs may result in
+   unintentionally accepting payments with TLVs you do not understand (#2308).
+ * `Route` objects (including pending payments) written by LDK versions prior
+   to 0.0.117 won't be retryable after being deserialized by LDK 0.0.117 or
+   above (#2555).
+ * Users of the `MonitorUpdatingPersister` can upgrade seamlessly from the
+   default `KVStore` `Persist` implementation, however the stored
+   `ChannelMonitor`s are deliberately unreadable by the default `Persist`. This
+   ensures the correct downgrade procedure is followed, which is: (#2359)
+   * First, make a backup copy of all channel state,
+   * then ensure all `ChannelMonitorUpdate`s stored are fully applied to the
+     relevant `ChannelMonitor`,
+   * finally, write each full `ChannelMonitor` using your new `Persist` impl.
+
+## Bug Fixes
+ * Anchor channels which were closed by a counterparty broadcasting its
+   commitment transaction (i.e. force-closing) would previously not generate a
+   `SpendableOutputs` event for our `to_remote` (i.e. non-HTLC-encumbered)
+   balance. Those with such balances available should fetch the missing
+   `SpendableOutputDescriptor`s using the new
+   `ChannelMonitor::get_spendable_outputs` method (#2605).
+ * Anchor channels may result in spurious or missing `Balance` entries for HTLC
+   balances (#2610).
+ * `ChannelManager::send_spontaneous_payment_with_retry` spuriously did not
+   provide the recipient with enough information to claim the payment, leading
+   to all spontaneous payments failing (#2475).
+   `send_spontaneous_payment_with_route` was unaffected.
+ * The `keysend` feature on node announcements was spuriously un-set in 0.0.112
+   and has been re-enabled (#2465).
+ * Fixed several races which could lead to deadlock when force-closing a channel
+   (#2597). These races have not been seen in production.
+ * The `ChannelManager` is persisted substantially less when it has not changed,
+   leading to substantially less I/O traffic for it (#2521, #2617).
+ * Passing new block data to `ChainMonitor` no longer results in all other
+   monitor operations being blocked until it completes (#2528).
+ * When retrying payments, any excess amount sent to the recipient in order to
+   meet an `htlc_minimum` constraint on the path is now no longer included in
+   the amount we send in the retry (#2575).
+ * Several edge cases in route-finding around HTLC minimums were fixed which
+   could have caused invalid routes or panics when built with debug assertions
+   (#2570, #2575).
+ * Several edge cases in route-finding around HTLC minimums and route hints
+   were fixed which would spuriously result in no route found (#2575, #2604).
+ * The `user_channel_id` passed to `SignerProvider::generate_channel_keys_id`
+   for inbound channels is now correctly using the one passed to
+   `ChannelManager::accept_inbound_channel` rather than a default value (#2428).
+ * Users of `impl_writeable_tlv_based!` no longer have use requirements (#2506).
+ * No longer force-close channels when counterparties send a `channel_update`
+   with a bogus `htlc_minimum_msat`, which LND users can manually build (#2611).
+
+## Node Compatibility
+ * LDK now ignores `error` messages generated by LND in response to a
+   `shutdown` message, avoiding force-closes due to LND bug 6039. This may
+   lead to non-trivial bandwidth usage with LND peers exhibiting this bug
+   during the cooperative shutdown process (#2507).
+
+## Security
+0.0.117 fixes several loss-of-funds vulnerabilities in anchor output channels,
+support for which was added in 0.0.116, in reorg handling, and when accepting
+channel(s) from counterparties which are miners.
+ * When a counterparty broadcasts their latest commitment transaction for a
+   channel with anchor outputs, we'd previously fail to build claiming
+   transactions against any HTLC outputs in that transaction. This could lead
+   to loss of funds if the counterparty is able to eventually claim the HTLC
+   after a timeout (#2606).
+ * Anchor channels HTLC claims on-chain previously spent the entire value of any
+   HTLCs as fee, which has now been fixed (#2587).
+ * If a channel is closed via an on-chain commitment transaction confirmation
+   with a pending outbound HTLC in the commitment transaction, followed by a
+   reorg which replaces the confirmed commitment transaction with a different
+   (but non-revoked) commitment transaction, all before we learn the payment
+   preimage for this HTLC, we may previously have not generated a proper
+   claiming transaction for the HTLC's value (#2623).
+ * 0.0.117 now correctly handles channels for which our counterparty funded the
+   channel with a coinbase transaction. As such transactions are not spendable
+   until they've reached 100 confirmations, this could have resulted in
+   accepting HTLC(s) which are not enforcible on-chain (#1924).
+
+In total, this release features 121 files changed, 20477 insertions, 8184
+deletions in 381 commits from 27 authors, in alphabetical order:
+ * Alec Chen
+ * Allan Douglas R. de Oliveira
+ * Antonio Yang
+ * Arik Sosman
+ * Chris Waterson
+ * David Caseria
+ * DhananjayPurohit
+ * Dom Zippilli
+ * Duncan Dean
+ * Elias Rohrer
+ * Erik De Smedt
+ * Evan Feenstra
+ * Gabor Szabo
+ * Gursharan Singh
+ * Jeffrey Czyz
+ * Joseph Goulden
+ * Lalitmohansharma1
+ * Matt Corallo
+ * Rachel Malonson
+ * Sergi Delgado Segura
+ * Valentine Wallace
+ * Vladimir Fomene
+ * Willem Van Lint
+ * Wilmer Paulino
+ * benthecarman
+ * jbesraa
+ * optout
+
+
 # 0.0.116 - Jul 21, 2023 - "Anchoring the Roadmap"
 
 ## API Updates

bench/benches/bench.rs

@@ -13,6 +13,9 @@ criterion_group!(benches,
 	lightning::routing::router::benches::generate_routes_with_probabilistic_scorer,
 	lightning::routing::router::benches::generate_mpp_routes_with_probabilistic_scorer,
 	lightning::routing::router::benches::generate_large_mpp_routes_with_probabilistic_scorer,
+	lightning::routing::router::benches::generate_routes_with_nonlinear_probabilistic_scorer,
+	lightning::routing::router::benches::generate_mpp_routes_with_nonlinear_probabilistic_scorer,
+	lightning::routing::router::benches::generate_large_mpp_routes_with_nonlinear_probabilistic_scorer,
 	lightning::sign::benches::bench_get_secure_random_bytes,
 	lightning::ln::channelmanager::bench::bench_sends,
 	lightning_persister::fs_store::bench::bench_sends,

ci/ci-tests.sh

@@ -29,41 +29,45 @@ PIN_RELEASE_DEPS # pin the release dependencies in our main workspace
 # The quote crate switched to Rust edition 2021 starting with v1.0.31, i.e., has MSRV of 1.56
 [ "$RUSTC_MINOR_VERSION" -lt 56 ] && cargo update -p quote --precise "1.0.30" --verbose
 
+# The syn crate depends on too-new proc-macro2 starting with v2.0.33, i.e., has MSRV of 1.56
+if [ "$RUSTC_MINOR_VERSION" -lt 56 ]; then
+	SYN_2_DEP=$(grep -o '"syn 2.*' Cargo.lock | tr -d '",' | tr ' ' ':')
+	cargo update -p "$SYN_2_DEP" --precise "2.0.32" --verbose
+fi
+
 # The proc-macro2 crate switched to Rust edition 2021 starting with v1.0.66, i.e., has MSRV of 1.56
 [ "$RUSTC_MINOR_VERSION" -lt 56 ] && cargo update -p proc-macro2 --precise "1.0.65" --verbose
 
 # The memchr crate switched to an MSRV of 1.60 starting with v2.6.0
 [ "$RUSTC_MINOR_VERSION" -lt 60 ] && cargo update -p memchr --precise "2.5.0" --verbose
 
-[ "$LDK_COVERAGE_BUILD" != "" ] && export RUSTFLAGS="-C link-dead-code"
-
 export RUST_BACKTRACE=1
 
 echo -e "\n\nBuilding and testing all workspace crates..."
 cargo test --verbose --color always
-cargo build --verbose --color always
+cargo check --verbose --color always
 
 echo -e "\n\nBuilding and testing Block Sync Clients with features"
 pushd lightning-block-sync
 cargo test --verbose --color always --features rest-client
-cargo build --verbose --color always --features rest-client
+cargo check --verbose --color always --features rest-client
 cargo test --verbose --color always --features rpc-client
-cargo build --verbose --color always --features rpc-client
+cargo check --verbose --color always --features rpc-client
 cargo test --verbose --color always --features rpc-client,rest-client
-cargo build --verbose --color always --features rpc-client,rest-client
+cargo check --verbose --color always --features rpc-client,rest-client
 cargo test --verbose --color always --features rpc-client,rest-client,tokio
-cargo build --verbose --color always --features rpc-client,rest-client,tokio
+cargo check --verbose --color always --features rpc-client,rest-client,tokio
 popd
 
 if [[ $RUSTC_MINOR_VERSION -gt 67 && "$HOST_PLATFORM" != *windows* ]]; then
 	echo -e "\n\nBuilding and testing Transaction Sync Clients with features"
 	pushd lightning-transaction-sync
 	cargo test --verbose --color always --features esplora-blocking
-	cargo build --verbose --color always --features esplora-blocking
+	cargo check --verbose --color always --features esplora-blocking
 	cargo test --verbose --color always --features esplora-async
-	cargo build --verbose --color always --features esplora-async
+	cargo check --verbose --color always --features esplora-async
 	cargo test --verbose --color always --features esplora-async-https
-	cargo build --verbose --color always --features esplora-async-https
+	cargo check --verbose --color always --features esplora-async-https
 	popd
 fi
 
@@ -89,7 +93,7 @@ fi
 echo -e "\n\nBuilding with all Log-Limiting features"
 pushd lightning
 grep '^max_level_' Cargo.toml | awk '{ print $1 }'| while read -r FEATURE; do
-	cargo build --verbose --color always --features "$FEATURE"
+	cargo check --verbose --color always --features "$FEATURE"
 done
 popd
 
@@ -120,6 +124,10 @@ if [[ $RUSTC_MINOR_VERSION -gt 67 ]]; then
 	# lightning-transaction-sync's MSRV is 1.67
 	cargo check --verbose --color always --features lightning-transaction-sync
 else
+	# The memchr crate switched to an MSRV of 1.60 starting with v2.6.0
+	# This is currently only a release dependency via core2, which we intend to work with
+	# rust-bitcoin to remove soon.
+	[ "$RUSTC_MINOR_VERSION" -lt 60 ] && cargo update -p memchr --precise "2.5.0" --verbose
 	cargo check --verbose --color always
 fi
 popd

fuzz/src/chanmon_consistency.rs

@@ -138,7 +138,7 @@ impl TestChainMonitor {
 	}
 }
 impl chain::Watch<TestChannelSigner> for TestChainMonitor {
-	fn watch_channel(&self, funding_txo: OutPoint, monitor: channelmonitor::ChannelMonitor<TestChannelSigner>) -> chain::ChannelMonitorUpdateStatus {
+	fn watch_channel(&self, funding_txo: OutPoint, monitor: channelmonitor::ChannelMonitor<TestChannelSigner>) -> Result<chain::ChannelMonitorUpdateStatus, ()> {
 		let mut ser = VecWriter(Vec::new());
 		monitor.write(&mut ser).unwrap();
 		if let Some(_) = self.latest_monitors.lock().unwrap().insert(funding_txo, (monitor.get_latest_update_id(), ser.0)) {
@@ -155,7 +155,7 @@ impl chain::Watch<TestChannelSigner> for TestChainMonitor {
 		};
 		let deserialized_monitor = <(BlockHash, channelmonitor::ChannelMonitor<TestChannelSigner>)>::
 			read(&mut Cursor::new(&map_entry.get().1), (&*self.keys, &*self.keys)).unwrap().1;
-		deserialized_monitor.update_monitor(update, &&TestBroadcaster{}, &FuzzEstimator { ret_val: atomic::AtomicU32::new(253) }, &self.logger).unwrap();
+		deserialized_monitor.update_monitor(update, &&TestBroadcaster{}, &&FuzzEstimator { ret_val: atomic::AtomicU32::new(253) }, &self.logger).unwrap();
 		let mut ser = VecWriter(Vec::new());
 		deserialized_monitor.write(&mut ser).unwrap();
 		map_entry.insert((update.update_id, ser.0));
@@ -371,6 +371,7 @@ fn send_payment(source: &ChanMan, dest: &ChanMan, dest_chan_id: u64, amt: u64, p
 			channel_features: dest.channel_features(),
 			fee_msat: amt,
 			cltv_expiry_delta: 200,
+			maybe_announced_channel: true,
 		}], blinded_tail: None }],
 		route_params: None,
 	}, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) {
@@ -405,13 +406,15 @@ fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, des
 			channel_features: middle.channel_features(),
 			fee_msat: first_hop_fee,
 			cltv_expiry_delta: 100,
+			maybe_announced_channel: true,
 		}, RouteHop {
 			pubkey: dest.get_our_node_id(),
 			node_features: dest.node_features(),
 			short_channel_id: dest_chan_id,
 			channel_features: dest.channel_features(),
 			fee_msat: amt,
 			cltv_expiry_delta: 200,
+			maybe_announced_channel: true,
 		}], blinded_tail: None }],
 		route_params: None,
 	}, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) {
@@ -497,7 +500,7 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
 			let res = (<(BlockHash, ChanMan)>::read(&mut Cursor::new(&$ser.0), read_args).expect("Failed to read manager").1, chain_monitor.clone());
 			for (funding_txo, mon) in monitors.drain() {
 				assert_eq!(chain_monitor.chain_monitor.watch_channel(funding_txo, mon),
-					ChannelMonitorUpdateStatus::Completed);
+					Ok(ChannelMonitorUpdateStatus::Completed));
 			}
 			res
 		} }