expose functional tests under _test_utils feature flag

introduce DynSigner and TestSignerFactory

.gitignore

@@ -12,5 +12,6 @@ lightning/net_graph-*.bin
 lightning-rapid-gossip-sync/res/full_graph.lngossip
 lightning-custom-message/target
 lightning-transaction-sync/target
+ext-functional-test-demo/target
 no-std-check/target
 msrv-no-dev-deps-check/target

Cargo.toml

@@ -19,6 +19,7 @@ members = [
 ]
 
 exclude = [
+    "ext-functional-test-demo",
     "no-std-check",
     "msrv-no-dev-deps-check",
     "bench",

ci/ci-tests.sh

@@ -138,6 +138,12 @@ cargo check --verbose --color always --features lightning-transaction-sync
 [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean
 popd
 
+echo -e "\n\Running functional tests from outside the workspace"
+pushd ext-functional-test-demo
+cargo test --color always
+[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean
+popd
+
 # Test that we can build downstream code with only the "release pins".
 pushd msrv-no-dev-deps-check
 PIN_RELEASE_DEPS

ext-functional-test-demo/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "ext-functional-tester"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+lightning = { path = "../lightning", features = ["_test_utils"] }

ext-functional-test-demo/src/main.rs

@@ -0,0 +1,31 @@
+fn main() {
+	println!("Hello, world!");
+}
+
+#[cfg(test)]
+mod tests {
+	use lightning::util::dyn_signer::{DynKeysInterfaceTrait, DynSigner};
+	use lightning::util::test_utils::{TestSignerFactory, SIGNER_FACTORY};
+	use std::panic::catch_unwind;
+	use std::sync::Arc;
+	use std::time::Duration;
+
+	struct BrokenSignerFactory();
+
+	impl TestSignerFactory for BrokenSignerFactory {
+		fn make_signer(
+			&self, _seed: &[u8; 32], _now: Duration,
+		) -> Box<dyn DynKeysInterfaceTrait<EcdsaSigner = DynSigner>> {
+			panic!()
+		}
+	}
+
+	#[test]
+	fn test_functional() {
+		lightning::ln::functional_tests::test_insane_channel_opens();
+		lightning::ln::functional_tests::fake_network_test();
+
+		SIGNER_FACTORY.set(Arc::new(BrokenSignerFactory()));
+		catch_unwind(|| lightning::ln::functional_tests::fake_network_test()).unwrap_err();
+	}
+}

fuzz/src/chanmon_consistency.rs

@@ -81,6 +81,7 @@ use bitcoin::secp256k1::schnorr;
 use bitcoin::secp256k1::{self, Message, PublicKey, Scalar, Secp256k1, SecretKey};
 
 use lightning::io::Cursor;
+use lightning::util::dyn_signer::DynSigner;
 use std::cmp::{self, Ordering};
 use std::mem;
 use std::sync::atomic;
@@ -391,6 +392,7 @@ impl SignerProvider for KeyProvider {
 			channel_keys_id,
 		);
 		let revoked_commitment = self.make_enforcement_state_cell(keys.commitment_seed);
+		let keys = DynSigner::new(keys);
 		TestChannelSigner::new_with_revoked(keys, revoked_commitment, false)
 	}
 
@@ -399,6 +401,7 @@ impl SignerProvider for KeyProvider {
 
 		let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
 		let state = self.make_enforcement_state_cell(inner.commitment_seed);
+		let inner = DynSigner::new(inner);
 
 		Ok(TestChannelSigner::new_with_revoked(inner, state, false))
 	}

fuzz/src/full_stack.rs

@@ -77,6 +77,7 @@ use bitcoin::secp256k1::ecdsa::{RecoverableSignature, Signature};
 use bitcoin::secp256k1::schnorr;
 use bitcoin::secp256k1::{self, Message, PublicKey, Scalar, Secp256k1, SecretKey};
 
+use lightning::util::dyn_signer::DynSigner;
 use std::cell::RefCell;
 use std::cmp;
 use std::convert::TryInto;
@@ -455,7 +456,7 @@ impl SignerProvider for KeyProvider {
 		let ctr = channel_keys_id[0];
 		let (inbound, state) = self.signer_state.borrow().get(&ctr).unwrap().clone();
 		TestChannelSigner::new_with_revoked(
-			if inbound {
+			DynSigner::new(if inbound {
 				InMemorySigner::new(
 					&secp_ctx,
 					SecretKey::from_slice(&[
@@ -527,7 +528,7 @@ impl SignerProvider for KeyProvider {
 					channel_keys_id,
 					channel_keys_id,
 				)
-			},
+			}),
 			state,
 			false,
 		)
@@ -536,6 +537,7 @@ impl SignerProvider for KeyProvider {
 	fn read_chan_signer(&self, mut data: &[u8]) -> Result<TestChannelSigner, DecodeError> {
 		let inner: InMemorySigner = ReadableArgs::read(&mut data, self)?;
 		let state = Arc::new(Mutex::new(EnforcementState::new()));
+		let inner = DynSigner::new(inner);
 
 		Ok(TestChannelSigner::new_with_revoked(inner, state, false))
 	}

lightning-background-processor/src/lib.rs

@@ -2557,6 +2557,8 @@ mod tests {
 				failure: PathFailure::OnPath { network_update: None },
 				path: path.clone(),
 				short_channel_id: Some(scored_scid),
+				error_code: None,
+				error_data: None,
 			});
 			let event = $receive.expect("PaymentPathFailed not handled within deadline");
 			match event {
@@ -2574,6 +2576,8 @@ mod tests {
 				failure: PathFailure::OnPath { network_update: None },
 				path: path.clone(),
 				short_channel_id: None,
+				error_code: None,
+				error_data: None,
 			});
 			let event = $receive.expect("PaymentPathFailed not handled within deadline");
 			match event {

lightning/Cargo.toml

@@ -17,7 +17,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [features]
 # Internal test utilities exposed to other repo crates
-_test_utils = ["regex", "bitcoin/bitcoinconsensus", "lightning-types/_test_utils"]
+_test_utils = ["regex", "bitcoin/bitcoinconsensus", "ext-test-macro", "lightning-types/_test_utils"]
 # Unlog messages superior at targeted level.
 max_level_off = []
 max_level_error = []
@@ -50,6 +50,7 @@ backtrace = { version = "0.3", optional = true }
 
 libm = { version = "0.2", default-features = false }
 delegate = "0.12.0"
+ext-test-macro = { path = "../ext-test-macro", optional = true }
 
 [dev-dependencies]
 regex = "1.5.6"

lightning/src/chain/chainmonitor.rs

@@ -464,7 +464,7 @@ where C::Target: chain::Filter,
 	}
 
 
-	#[cfg(test)]
+	#[cfg(any(test, feature = "_test_utils"))]
 	pub fn remove_monitor(&self, funding_txo: &OutPoint) -> ChannelMonitor<ChannelSigner> {
 		self.monitors.write().unwrap().remove(funding_txo).unwrap().monitor
 	}

lightning/src/chain/channelmonitor.rs

@@ -844,10 +844,7 @@ impl Readable for IrrevocablyResolvedHTLC {
 /// returned block hash and the the current chain and then reconnecting blocks to get to the
 /// best chain) upon deserializing the object!
 pub struct ChannelMonitor<Signer: EcdsaChannelSigner> {
-	#[cfg(test)]
 	pub(crate) inner: Mutex<ChannelMonitorImpl<Signer>>,
-	#[cfg(not(test))]
-	pub(super) inner: Mutex<ChannelMonitorImpl<Signer>>,
 }
 
 impl<Signer: EcdsaChannelSigner> Clone for ChannelMonitor<Signer> where Signer: Clone {
@@ -946,9 +943,9 @@ pub(crate) struct ChannelMonitorImpl<Signer: EcdsaChannelSigner> {
 	// Obviously Correct (tm) if we just keep track of them explicitly.
 	outputs_to_watch: HashMap<Txid, Vec<(u32, ScriptBuf)>>,
 
-	#[cfg(test)]
+	#[cfg(any(test, feature = "_test_utils"))]
 	pub onchain_tx_handler: OnchainTxHandler<Signer>,
-	#[cfg(not(test))]
+	#[cfg(not(any(test, feature = "_test_utils")))]
 	onchain_tx_handler: OnchainTxHandler<Signer>,
 
 	// This is set when the Channel[Manager] generated a ChannelMonitorUpdate which indicated the
@@ -1726,7 +1723,7 @@ impl<Signer: EcdsaChannelSigner> ChannelMonitor<Signer> {
 	/// Unsafe test-only version of `broadcast_latest_holder_commitment_txn` used by our test framework
 	/// to bypass HolderCommitmentTransaction state update lockdown after signature and generate
 	/// revoked commitment transaction.
-	#[cfg(any(test, feature = "unsafe_revoked_tx_signing"))]
+	#[cfg(any(test, feature = "_test_utils", feature = "unsafe_revoked_tx_signing"))]
 	pub fn unsafe_get_latest_holder_commitment_txn<L: Deref>(&self, logger: &L) -> Vec<Transaction>
 	where L::Target: Logger {
 		let mut inner = self.inner.lock().unwrap();
@@ -2026,7 +2023,7 @@ impl<Signer: EcdsaChannelSigner> ChannelMonitor<Signer> {
 		self.inner.lock().unwrap().counterparty_payment_script = script;
 	}
 
-	#[cfg(test)]
+	#[cfg(any(test, feature = "_test_utils"))]
 	pub fn do_mut_signer_call<F: FnMut(&mut Signer) -> ()>(&self, mut f: F) {
 		let mut inner = self.inner.lock().unwrap();
 		f(&mut inner.onchain_tx_handler.signer);
@@ -2668,7 +2665,7 @@ macro_rules! fail_unbroadcast_htlcs {
 // witness length match (ie is 136 bytes long). We generate one here which we also use in some
 // in-line tests later.
 
-#[cfg(test)]
+#[cfg(any(test, feature = "_test_utils"))]
 pub fn deliberately_bogus_accepted_htlc_witness_program() -> Vec<u8> {
 	use bitcoin::opcodes;
 	let mut ret = [opcodes::all::OP_NOP.to_u8(); 136];
@@ -2680,7 +2677,7 @@ pub fn deliberately_bogus_accepted_htlc_witness_program() -> Vec<u8> {
 	Vec::from(&ret[..])
 }
 
-#[cfg(test)]
+#[cfg(any(test, feature = "_test_utils"))]
 pub fn deliberately_bogus_accepted_htlc_witness() -> Vec<Vec<u8>> {
 	vec![Vec::new(), Vec::new(), Vec::new(), Vec::new(), deliberately_bogus_accepted_htlc_witness_program().into()].into()
 }
@@ -3808,7 +3805,7 @@ impl<Signer: EcdsaChannelSigner> ChannelMonitorImpl<Signer> {
 		}
 	}
 
-	#[cfg(any(test,feature = "unsafe_revoked_tx_signing"))]
+	#[cfg(any(test, feature = "_test_utils", feature = "unsafe_revoked_tx_signing"))]
 	/// Note that this includes possibly-locktimed-in-the-future transactions!
 	fn unsafe_get_latest_holder_commitment_txn<L: Deref>(
 		&mut self, logger: &WithChannelMonitor<L>
@@ -5066,7 +5063,7 @@ mod tests {
 		nodes[1].chain_monitor.chain_monitor.transactions_confirmed(&new_header,
 			&[(0, broadcast_tx)], conf_height);
 
-		let (_, pre_update_monitor) = <(BlockHash, ChannelMonitor<InMemorySigner>)>::read(
+		let (_, pre_update_monitor) = <(BlockHash, ChannelMonitor<_>)>::read(
 						&mut io::Cursor::new(&get_monitor!(nodes[1], channel.2).encode()),
 						(&nodes[1].keys_manager.backing, &nodes[1].keys_manager.backing)).unwrap();
 

lightning/src/chain/onchaintx.rs

@@ -249,10 +249,7 @@ pub struct OnchainTxHandler<ChannelSigner: EcdsaChannelSigner> {
 	// Key is identifier of the pending claim request, i.e the txid of the initial claiming transaction generated by
 	// us and is immutable until all outpoint of the claimable set are post-anti-reorg-delay solved.
 	// Entry is cache of elements need to generate a bumped claiming transaction (see ClaimTxBumpMaterial)
-	#[cfg(test)] // Used in functional_test to verify sanitization
 	pub(crate) pending_claim_requests: HashMap<ClaimId, PackageTemplate>,
-	#[cfg(not(test))]
-	pending_claim_requests: HashMap<ClaimId, PackageTemplate>,
 
 	// Used to track external events that need to be forwarded to the `ChainMonitor`. This `Vec`
 	// essentially acts as an insertion-ordered `HashMap` – there should only ever be one occurrence
@@ -272,10 +269,7 @@ pub struct OnchainTxHandler<ChannelSigner: EcdsaChannelSigner> {
 	// block height, and are immutable until the outpoint has enough confirmations to meet our
 	// [`ANTI_REORG_DELAY`]. The initial confirmation block height is used to remove the entry if
 	// the block gets disconnected.
-	#[cfg(test)] // Used in functional_test to verify sanitization
-	pub claimable_outpoints: HashMap<BitcoinOutPoint, (ClaimId, u32)>,
-	#[cfg(not(test))]
-	claimable_outpoints: HashMap<BitcoinOutPoint, (ClaimId, u32)>,
+	pub(crate) claimable_outpoints: HashMap<BitcoinOutPoint, (ClaimId, u32)>,
 
 	locktimed_packages: BTreeMap<u32, Vec<PackageTemplate>>,
 
@@ -1176,7 +1170,7 @@ impl<ChannelSigner: EcdsaChannelSigner> OnchainTxHandler<ChannelSigner> {
 		MaybeSignedTransaction(tx)
 	}
 
-	#[cfg(any(test, feature="unsafe_revoked_tx_signing"))]
+	#[cfg(any(test, feature="_test_utils", feature="unsafe_revoked_tx_signing"))]
 	pub(crate) fn get_fully_signed_copy_holder_tx(&mut self, funding_redeemscript: &Script) -> Transaction {
 		let sig = self.signer.unsafe_sign_holder_commitment(&self.holder_commitment, &self.secp_ctx).expect("sign holder commitment");
 		self.holder_commitment.add_holder_sig(funding_redeemscript, sig)

lightning/src/events/mod.rs

@@ -1009,9 +1009,9 @@ pub enum Event {
 		/// If this is `Some`, then the corresponding channel should be avoided when the payment is
 		/// retried. May be `None` for older [`Event`] serializations.
 		short_channel_id: Option<u64>,
-#[cfg(test)]
+		#[cfg(any(test, feature = "_test_utils"))]
 		error_code: Option<u16>,
-#[cfg(test)]
+		#[cfg(any(test, feature = "_test_utils"))]
 		error_data: Option<Vec<u8>>,
 	},
 	/// Indicates that a probe payment we sent returned successful, i.e., only failed at the destination.
@@ -1509,15 +1509,15 @@ impl Writeable for Event {
 			&Event::PaymentPathFailed {
 				ref payment_id, ref payment_hash, ref payment_failed_permanently, ref failure,
 				ref path, ref short_channel_id,
-				#[cfg(test)]
+				#[cfg(any(test, feature = "_test_utils"))]
 				ref error_code,
-				#[cfg(test)]
+				#[cfg(any(test, feature = "_test_utils"))]
 				ref error_data,
 			} => {
 				3u8.write(writer)?;
-				#[cfg(test)]
+				#[cfg(any(test, feature = "_test_utils"))]
 				error_code.write(writer)?;
-				#[cfg(test)]
+				#[cfg(any(test, feature = "_test_utils"))]
 				error_data.write(writer)?;
 				write_tlv_fields!(writer, {
 					(0, payment_hash, required),
@@ -1850,9 +1850,9 @@ impl MaybeReadable for Event {
 			},
 			3u8 => {
 				let mut f = || {
-					#[cfg(test)]
+					#[cfg(any(test, feature = "_test_utils"))]
 					let error_code = Readable::read(reader)?;
-					#[cfg(test)]
+					#[cfg(any(test, feature = "_test_utils"))]
 					let error_data = Readable::read(reader)?;
 					let mut payment_hash = PaymentHash([0; 32]);
 					let mut payment_failed_permanently = false;
@@ -1882,9 +1882,9 @@ impl MaybeReadable for Event {
 						failure,
 						path: Path { hops: path.unwrap(), blinded_tail },
 						short_channel_id,
-						#[cfg(test)]
+						#[cfg(any(test, feature = "_test_utils"))]
 						error_code,
-						#[cfg(test)]
+						#[cfg(any(test, feature = "_test_utils"))]
 						error_data,
 					}))
 				};

lightning/src/ln/bolt11_payment.rs

@@ -89,6 +89,7 @@ mod tests {
 	use super::*;
 	use crate::ln::types::PaymentSecret;
 	use crate::routing::router::Payee;
+	use crate::sign::{NodeSigner, Recipient};
 	use bitcoin::hashes::sha256::Hash as Sha256;
 	use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
 	use lightning_invoice::{Currency, InvoiceBuilder};
@@ -178,8 +179,6 @@ mod tests {
 		let (payment_hash, payment_secret) =
 			nodes[1].node.create_inbound_payment(None, 7200, None).unwrap();
 
-		let secp_ctx = Secp256k1::new();
-		let node_secret = nodes[1].keys_manager.backing.get_node_secret_key();
 		let timestamp = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap();
 		let invoice = InvoiceBuilder::new(Currency::Bitcoin)
 			.description("test".into())
@@ -189,8 +188,11 @@ mod tests {
 			.min_final_cltv_expiry_delta(144)
 			.amount_milli_satoshis(50_000)
 			.payment_metadata(payment_metadata.clone())
-			.build_signed(|hash| secp_ctx.sign_ecdsa_recoverable(hash, &node_secret))
+			.build_raw()
 			.unwrap();
+		let sig = nodes[1].keys_manager.backing.sign_invoice(&invoice, Recipient::Node).unwrap();
+		let invoice = invoice.sign::<_, ()>(|_| Ok(sig)).unwrap();
+		let invoice = Bolt11Invoice::from_signed(invoice).unwrap();
 
 		let (hash, onion, params) = payment_parameters_from_invoice(&invoice).unwrap();
 		nodes[0]

lightning/src/ln/chan_utils.rs

@@ -176,9 +176,9 @@ impl HTLCClaim {
 	}
 }
 
-#[cfg(not(test))]
+#[cfg(not(any(test, feature = "_test_utils")))]
 const COMMITMENT_TX_WEIGHT_PER_HTLC: u64 = 172;
-#[cfg(test)]
+#[cfg(any(test, feature = "_test_utils"))]
 pub const COMMITMENT_TX_WEIGHT_PER_HTLC: u64 = 172;
 
 pub(crate) fn commitment_tx_base_weight(channel_type_features: &ChannelTypeFeatures) -> u64 {