Add availability-recovery from systematic chunks

cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs

@@ -23,7 +23,7 @@ use polkadot_core_primitives::{Block, BlockNumber, Hash, Header};
 use polkadot_overseer::RuntimeApiSubsystemClient;
 use polkadot_primitives::{
 	slashing,
-	vstaging::{AsyncBackingParams, BackingState},
+	vstaging::{AsyncBackingParams, BackingState, ClientFeatures},
 };
 use sc_authority_discovery::{AuthorityDiscovery, Error as AuthorityDiscoveryError};
 use sp_api::{ApiError, RuntimeApiInfo};
@@ -346,6 +346,10 @@ impl RuntimeApiSubsystemClient for BlockChainRpcClient {
 		Ok(self.rpc_client.parachain_host_minimum_backing_votes(at, session_index).await?)
 	}
 
+	async fn client_features(&self, at: Hash) -> Result<ClientFeatures, ApiError> {
+		Ok(self.rpc_client.parachain_host_client_features(at).await?)
+	}
+
 	async fn staging_async_backing_params(&self, at: Hash) -> Result<AsyncBackingParams, ApiError> {
 		Ok(self.rpc_client.parachain_host_staging_async_backing_params(at).await?)
 	}

cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs

@@ -31,7 +31,7 @@ use parity_scale_codec::{Decode, Encode};
 use cumulus_primitives_core::{
 	relay_chain::{
 		slashing,
-		vstaging::{AsyncBackingParams, BackingState},
+		vstaging::{AsyncBackingParams, BackingState, ClientFeatures},
 		BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash,
 		CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo,
 		Hash as RelayHash, Header as RelayHeader, InboundHrmpMessage, OccupiedCoreAssumption,
@@ -598,6 +598,14 @@ impl RelayChainRpcClient {
 			.await
 	}
 
+	pub async fn parachain_host_client_features(
+		&self,
+		at: RelayHash,
+	) -> Result<ClientFeatures, RelayChainError> {
+		self.call_remote_runtime_function("ParachainHost_client_features", at, None::<()>)
+			.await
+	}
+
 	#[allow(missing_docs)]
 	pub async fn parachain_host_staging_async_backing_params(
 		&self,

cumulus/parachains/integration-tests/emulated/common/src/lib.rs

@@ -23,6 +23,7 @@ use constants::{
 	bridge_hub_polkadot, bridge_hub_rococo, collectives, kusama, penpal, polkadot, rococo, westend,
 };
 use impls::{RococoWococoMessageHandler, WococoRococoMessageHandler};
+use polkadot_primitives::runtime_api::runtime_decl_for_parachain_host::ParachainHostV7;
 
 // Substrate
 use frame_support::traits::OnInitialize;

polkadot/erasure-coding/Cargo.toml

@@ -8,14 +8,15 @@ license.workspace = true
 [dependencies]
 polkadot-primitives = { path = "../primitives" }
 polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../node/primitives" }
-novelpoly = { package = "reed-solomon-novelpoly", version = "1.0.0" }
+novelpoly = { package = "reed-solomon-novelpoly", git = "https://github.com/paritytech/reed-solomon-novelpoly.git", rev="6864dbf2018a8a92d976835447ba29bcdc8af6a0" }
 parity-scale-codec = { version = "3.6.1", default-features = false, features = ["std", "derive"] }
 sp-core = { path = "../../substrate/primitives/core" }
 sp-trie = { path = "../../substrate/primitives/trie" }
 thiserror = "1.0.48"
 
 [dev-dependencies]
 criterion = { version = "0.4.0", default-features = false, features = ["cargo_bench_support"] }
+quickcheck = { version = "1.0.3", default-features = false }
 
 [[bench]]
 name = "scaling_with_validators"

polkadot/erasure-coding/benches/README.md

@@ -7,7 +7,8 @@ cargo bench
 ## `scaling_with_validators`
 
 This benchmark evaluates the performance of constructing the chunks and the erasure root from PoV and
-reconstructing the PoV from chunks. You can see the results of running this bench on 5950x below.
+reconstructing the PoV from chunks (either from systematic chunks or regular chunks).
+You can see the results of running this bench on 5950x below (only including recovery from regular chunks).
 Interestingly, with `10_000` chunks (validators) its slower than with `50_000` for both construction
 and reconstruction.
 ```
@@ -37,3 +38,6 @@ reconstruct/10000       time:   [496.35 ms 505.17 ms 515.42 ms]
 reconstruct/50000       time:   [276.56 ms 277.53 ms 278.58 ms]
                         thrpt:  [17.948 MiB/s 18.016 MiB/s 18.079 MiB/s]
 ```
+
+Results from running on an Apple M2 Pro, systematic recovery is generally 40 times faster than
+regular recovery, achieving 1 Gib/s.

polkadot/erasure-coding/benches/scaling_with_validators.rs

@@ -53,12 +53,16 @@ fn construct_and_reconstruct_5mb_pov(c: &mut Criterion) {
 	}
 	group.finish();
 
-	let mut group = c.benchmark_group("reconstruct");
+	let mut group = c.benchmark_group("reconstruct_regular");
 	for n_validators in N_VALIDATORS {
 		let all_chunks = chunks(n_validators, &pov);
 
-		let mut c: Vec<_> = all_chunks.iter().enumerate().map(|(i, c)| (&c[..], i)).collect();
-		let last_chunks = c.split_off((c.len() - 1) * 2 / 3);
+		let chunks: Vec<_> = all_chunks
+			.iter()
+			.enumerate()
+			.take(polkadot_erasure_coding::recovery_threshold(n_validators).unwrap())
+			.map(|(i, c)| (&c[..], i))
+			.collect();
 
 		group.throughput(Throughput::Bytes(pov.len() as u64));
 		group.bench_with_input(
@@ -67,7 +71,32 @@ fn construct_and_reconstruct_5mb_pov(c: &mut Criterion) {
 			|b, &n| {
 				b.iter(|| {
 					let _pov: Vec<u8> =
-						polkadot_erasure_coding::reconstruct(n, last_chunks.clone()).unwrap();
+						polkadot_erasure_coding::reconstruct(n, chunks.clone()).unwrap();
+				});
+			},
+		);
+	}
+	group.finish();
+
+	let mut group = c.benchmark_group("reconstruct_systematic");
+	for n_validators in N_VALIDATORS {
+		let all_chunks = chunks(n_validators, &pov);
+
+		let chunks = all_chunks
+			.iter()
+			.take(polkadot_erasure_coding::systematic_recovery_threshold(n_validators).unwrap())
+			.map(|c| &c[..])
+			.collect::<Vec<_>>();
+
+		group.throughput(Throughput::Bytes(pov.len() as u64));
+		group.bench_with_input(
+			BenchmarkId::from_parameter(n_validators),
+			&n_validators,
+			|b, &n| {
+				b.iter(|| {
+					let _pov: Vec<u8> =
+						polkadot_erasure_coding::reconstruct_from_systematic(n, chunks.clone())
+							.unwrap();
 				});
 			},
 		);

polkadot/erasure-coding/src/lib.rs

@@ -96,6 +96,14 @@ pub const fn recovery_threshold(n_validators: usize) -> Result<usize, Error> {
 	Ok(needed + 1)
 }
 
+/// Obtain the threshold of systematic chunks that should be enough to recover the data.
+///
+/// If the regular `recovery_threshold` is a power of two, then it returns the same value.
+/// Otherwise, it returns the next lower power of two.
+pub fn systematic_recovery_threshold(n_validators: usize) -> Result<usize, Error> {
+	code_params(n_validators).map(|params| params.k())
+}
+
 fn code_params(n_validators: usize) -> Result<CodeParams, Error> {
 	// we need to be able to reconstruct from 1/3 - eps
 
@@ -113,6 +121,66 @@ fn code_params(n_validators: usize) -> Result<CodeParams, Error> {
 	})
 }
 
+/// Reconstruct the v1 available data from the set of systematic chunks.
+///
+/// Provide a vector containing chunk data. If too few chunks are provided, recovery is not
+/// possible.
+pub fn reconstruct_from_systematic_v1(
+	n_validators: usize,
+	chunks: Vec<&[u8]>,
+) -> Result<AvailableData, Error> {
+	reconstruct_from_systematic(n_validators, chunks)
+}
+
+/// Reconstruct the available data from the set of systematic chunks.
+///
+/// Provide a vector containing chunk data. If too few chunks are provided, recovery is not
+/// possible.
+pub fn reconstruct_from_systematic<T: Decode>(
+	n_validators: usize,
+	chunks: Vec<&[u8]>,
+) -> Result<T, Error> {
+	let kpow2 = systematic_recovery_threshold(n_validators)?;
+
+	let Some(first_shard) = chunks.iter().next() else { return Err(Error::NotEnoughChunks) };
+	let shard_len = first_shard.len();
+
+	if shard_len == 0 {
+		return Err(Error::NonUniformChunks)
+	}
+
+	if shard_len % 2 != 0 {
+		return Err(Error::UnevenLength)
+	}
+
+	if chunks.len() < kpow2 {
+		return Err(Error::NotEnoughChunks)
+	}
+
+	let mut check_shard_len = true;
+	let mut systematic_bytes = Vec::with_capacity(shard_len * kpow2);
+
+	for i in (0..shard_len).step_by(2) {
+		for chunk in chunks.iter().take(kpow2) {
+			if check_shard_len {
+				if chunk.len() != shard_len {
+					return Err(Error::NonUniformChunks)
+				}
+			}
+
+			// No need to check for index out of bounds because i goes up to shard_len and
+			// we return an error for non uniform chunks.
+			systematic_bytes.push(chunk[i]);
+			systematic_bytes.push(chunk[i + 1]);
+		}
+
+		// After the first check, stop checking the shard lengths.
+		check_shard_len = false;
+	}
+
+	Decode::decode(&mut &systematic_bytes[..]).map_err(|_| Error::BadPayload)
+}
+
 /// Obtain erasure-coded chunks for v1 `AvailableData`, one for each validator.
 ///
 /// Works only up to 65536 validators, and `n_validators` must be non-zero.
@@ -201,7 +269,7 @@ where
 		Ok(payload_bytes) => payload_bytes,
 	};
 
-	Decode::decode(&mut &payload_bytes[..]).or_else(|_e| Err(Error::BadPayload))
+	Decode::decode(&mut &payload_bytes[..]).map_err(|_| Error::BadPayload)
 }
 
 /// An iterator that yields merkle branches and chunk data for all chunks to
@@ -346,13 +414,41 @@ impl<'a, I: Iterator<Item = &'a [u8]>> parity_scale_codec::Input for ShardInput<
 
 #[cfg(test)]
 mod tests {
+	use std::sync::Arc;
+
 	use super::*;
 	use polkadot_node_primitives::{AvailableData, BlockData, PoV};
+	use polkadot_primitives::{HeadData, PersistedValidationData};
+	use quickcheck::{Arbitrary, Gen, QuickCheck};
 
 	// In order to adequately compute the number of entries in the Merkle
 	// trie, we must account for the fixed 16-ary trie structure.
 	const KEY_INDEX_NIBBLE_SIZE: usize = 4;
 
+	#[derive(Clone, Debug)]
+	struct ArbitraryAvailableData(AvailableData);
+
+	impl Arbitrary for ArbitraryAvailableData {
+		fn arbitrary(g: &mut Gen) -> Self {
+			// Limit the POV len to 1 mib, otherwise the test will take forever
+			let pov_len = u32::arbitrary(g).saturating_add(2) % (1024 * 1024);
+
+			let pov = (0..pov_len).map(|_| u8::arbitrary(g)).collect();
+
+			let pvd = PersistedValidationData {
+				parent_head: HeadData((0..u16::arbitrary(g)).map(|_| u8::arbitrary(g)).collect()),
+				relay_parent_number: u32::arbitrary(g),
+				relay_parent_storage_root: [u8::arbitrary(g); 32].into(),
+				max_pov_size: u32::arbitrary(g),
+			};
+
+			ArbitraryAvailableData(AvailableData {
+				pov: Arc::new(PoV { block_data: BlockData(pov) }),
+				validation_data: pvd,
+			})
+		}
+	}
+
 	#[test]
 	fn field_order_is_right_size() {
 		assert_eq!(MAX_VALIDATORS, 65536);
@@ -379,6 +475,25 @@ mod tests {
 		assert_eq!(reconstructed, available_data);
 	}
 
+	#[test]
+	fn round_trip_systematic_works() {
+		fn property(available_data: ArbitraryAvailableData, n_validators: u16) {
+			let n_validators = n_validators.saturating_add(2);
+			let kpow2 = systematic_recovery_threshold(n_validators as usize).unwrap();
+			let chunks = obtain_chunks(n_validators as usize, &available_data.0).unwrap();
+			assert_eq!(
+				reconstruct_from_systematic_v1(
+					n_validators as usize,
+					chunks.iter().take(kpow2).map(|v| &v[..]).collect()
+				)
+				.unwrap(),
+				available_data.0
+			);
+		}
+
+		QuickCheck::new().quickcheck(property as fn(ArbitraryAvailableData, u16))
+	}
+
 	#[test]
 	fn reconstruct_does_not_panic_on_low_validator_count() {
 		let reconstructed = reconstruct_v1(1, [].iter().cloned());