Include block header in leaf commitment (#3647)

crates/examples/infra/mod.rs

@@ -368,7 +368,7 @@ pub trait RunDa<
     /// Note: sequencing leaf does not have state, so does not return state
     async fn initialize_state_and_hotshot(&self) -> SystemContextHandle<TYPES, NODE, V> {
         let initializer =
-            hotshot::HotShotInitializer::<TYPES>::from_genesis(TestInstanceState::default())
+            hotshot::HotShotInitializer::<TYPES>::from_genesis::<V>(TestInstanceState::default())
                 .await
                 .expect("Couldn't generate genesis block");
 

crates/hotshot/src/lib.rs

@@ -37,7 +37,6 @@ use async_broadcast::{broadcast, InactiveReceiver, Receiver, Sender};
 use async_compatibility_layer::art::async_spawn;
 use async_lock::RwLock;
 use async_trait::async_trait;
-use committable::Committable;
 use futures::join;
 use hotshot_task::task::{ConsensusTaskRegistry, NetworkTaskRegistry};
 use hotshot_task_impls::{events::HotShotEvent, helpers::broadcast_event};
@@ -194,7 +193,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
     ///
     /// Use this instead of `init` if you want to start the tasks manually
     #[allow(clippy::too_many_arguments)]
-    pub fn new(
+    pub async fn new(
         public_key: TYPES::SignatureKey,
         private_key: <TYPES::SignatureKey as SignatureKey>::PrivateKey,
         nonce: u64,
@@ -223,6 +222,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
             interal_chan,
             external_chan,
         )
+        .await
     }
 
     /// Creates a new [`Arc<SystemContext>`] with the given configuration options.
@@ -233,7 +233,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
     /// Use this function if you want to use some prexisting channels and to spin up the tasks
     /// and start consensus manually.  Mostly useful for tests
     #[allow(clippy::too_many_arguments, clippy::type_complexity)]
-    pub fn new_from_channels(
+    pub async fn new_from_channels(
         public_key: TYPES::SignatureKey,
         private_key: <TYPES::SignatureKey as SignatureKey>::PrivateKey,
         nonce: u64,
@@ -279,7 +279,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
             anchored_leaf.view_number(),
             View {
                 view_inner: ViewInner::Leaf {
-                    leaf: anchored_leaf.commit(),
+                    leaf: anchored_leaf.commit(&upgrade_lock).await,
                     state: Arc::clone(&validated_state),
                     delta: initializer.state_delta.clone(),
                 },
@@ -291,10 +291,13 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
 
         let mut saved_leaves = HashMap::new();
         let mut saved_payloads = BTreeMap::new();
-        saved_leaves.insert(anchored_leaf.commit(), anchored_leaf.clone());
+        saved_leaves.insert(
+            anchored_leaf.commit(&upgrade_lock).await,
+            anchored_leaf.clone(),
+        );
 
         for leaf in initializer.undecided_leafs {
-            saved_leaves.insert(leaf.commit(), leaf.clone());
+            saved_leaves.insert(leaf.commit(&upgrade_lock).await, leaf.clone());
         }
         if let Some(payload) = anchored_leaf.block_payload() {
             let encoded_txns = payload.encode();
@@ -411,7 +414,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
                     TYPES::ValidatedState::genesis(&self.instance_state);
 
                 let qc = Arc::new(
-                    QuorumCertificate::genesis(&validated_state, self.instance_state.as_ref())
+                    QuorumCertificate::genesis::<V>(&validated_state, self.instance_state.as_ref())
                         .await,
                 );
 
@@ -596,7 +599,8 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
             metrics,
             storage,
             marketplace_config,
-        );
+        )
+        .await;
         let handle = Arc::clone(&hotshot).run_tasks().await;
         let (tx, rx) = hotshot.internal_event_stream.clone();
 
@@ -753,7 +757,8 @@ where
             metrics.clone(),
             storage.clone(),
             marketplace_config.clone(),
-        );
+        )
+        .await;
         let right_system_context = SystemContext::new(
             public_key,
             private_key,
@@ -765,7 +770,8 @@ where
             metrics,
             storage,
             marketplace_config,
-        );
+        )
+        .await;
 
         // create registries for both handles
         let left_consensus_registry = ConsensusTaskRegistry::new();
@@ -972,11 +978,11 @@ impl<TYPES: NodeType> HotShotInitializer<TYPES> {
     /// initialize from genesis
     /// # Errors
     /// If we are unable to apply the genesis block to the default state
-    pub async fn from_genesis(
+    pub async fn from_genesis<V: Versions>(
         instance_state: TYPES::InstanceState,
     ) -> Result<Self, HotShotError<TYPES>> {
         let (validated_state, state_delta) = TYPES::ValidatedState::genesis(&instance_state);
-        let high_qc = QuorumCertificate::genesis(&validated_state, &instance_state).await;
+        let high_qc = QuorumCertificate::genesis::<V>(&validated_state, &instance_state).await;
 
         Ok(Self {
             inner: Leaf::genesis(&validated_state, &instance_state).await,

crates/hotshot/src/tasks/mod.rs

@@ -318,7 +318,8 @@ where
             metrics,
             storage,
             marketplace_config,
-        );
+        )
+        .await;
         let consensus_registry = ConsensusTaskRegistry::new();
         let network_registry = NetworkTaskRegistry::new();
 

crates/task-impls/src/consensus/handlers.rs

@@ -14,7 +14,6 @@ use async_lock::RwLock;
 #[cfg(async_executor_impl = "async-std")]
 use async_std::task::JoinHandle;
 use chrono::Utc;
-use committable::Committable;
 use futures::FutureExt;
 use hotshot_types::{
     consensus::{CommitmentAndMetadata, OuterConsensus, View},
@@ -37,7 +36,7 @@ use hotshot_types::{
 #[cfg(async_executor_impl = "tokio")]
 use tokio::task::JoinHandle;
 use tracing::{debug, error, info, instrument, warn};
-use vbs::version::{StaticVersionType, Version};
+use vbs::version::StaticVersionType;
 
 use super::ConsensusTaskState;
 use crate::{
@@ -67,7 +66,7 @@ pub async fn create_and_send_proposal<TYPES: NodeType, V: Versions>(
     proposal_cert: Option<ViewChangeEvidence<TYPES>>,
     round_start_delay: u64,
     instance_state: Arc<TYPES::InstanceState>,
-    version: Version,
+    upgrade_lock: UpgradeLock<TYPES, V>,
     id: u64,
 ) -> Result<()> {
     let consensus_read = consensus.read().await;
@@ -81,6 +80,11 @@ pub async fn create_and_send_proposal<TYPES: NodeType, V: Versions>(
         .context("Failed to get vid share")?;
     drop(consensus_read);
 
+    let version = upgrade_lock
+        .version(view)
+        .await
+        .context("Failed to get version number")?;
+
     let block_header = if version < V::Marketplace::VERSION {
         TYPES::BlockHeader::new_legacy(
             state.as_ref(),
@@ -122,9 +126,12 @@ pub async fn create_and_send_proposal<TYPES: NodeType, V: Versions>(
 
     let proposed_leaf = Leaf::from_quorum_proposal(&proposal);
 
-    ensure!(proposed_leaf.parent_commitment() == parent_leaf.commit());
+    ensure!(proposed_leaf.parent_commitment() == parent_leaf.commit(&upgrade_lock).await);
 
-    let signature = TYPES::SignatureKey::sign(&private_key, proposed_leaf.commit().as_ref())?;
+    let signature = TYPES::SignatureKey::sign(
+        &private_key,
+        proposed_leaf.commit(&upgrade_lock).await.as_ref(),
+    )?;
 
     let message = Proposal {
         data: proposal,
@@ -221,8 +228,6 @@ pub async fn publish_proposal_from_commitment_and_metadata<TYPES: NodeType, V: V
         "Cannot propose because our VID payload commitment and metadata is for an older view."
     );
 
-    let version = upgrade_lock.version(view).await?;
-
     let create_and_send_proposal_handle = async_spawn(async move {
         match create_and_send_proposal::<TYPES, V>(
             public_key,
@@ -237,7 +242,7 @@ pub async fn publish_proposal_from_commitment_and_metadata<TYPES: NodeType, V: V
             proposal_certificate,
             delay,
             instance_state,
-            version,
+            upgrade_lock,
             id,
         )
         .await
@@ -432,7 +437,7 @@ pub(crate) async fn handle_quorum_proposal_recv<
             view,
             View {
                 view_inner: ViewInner::Leaf {
-                    leaf: leaf.commit(),
+                    leaf: leaf.commit(&task_state.upgrade_lock).await,
                     state,
                     delta: None,
                 },
@@ -441,7 +446,9 @@ pub(crate) async fn handle_quorum_proposal_recv<
             tracing::trace!("{e:?}");
         }
 
-        consensus_write.update_saved_leaves(leaf.clone());
+        consensus_write
+            .update_saved_leaves(leaf.clone(), &task_state.upgrade_lock)
+            .await;
         let new_leaves = consensus_write.saved_leaves().clone();
         let new_state = consensus_write.validated_state_map().clone();
         drop(consensus_write);
@@ -812,7 +819,7 @@ pub async fn update_state_and_vote_if_able<
 
     let state = Arc::new(validated_state);
     let delta = Arc::new(state_delta);
-    let parent_commitment = parent.commit();
+    let parent_commitment = parent.commit(upgrade_lock).await;
 
     let proposed_leaf = Leaf::from_quorum_proposal(&proposal);
     if proposed_leaf.parent_commitment() != parent_commitment {
@@ -834,7 +841,7 @@ pub async fn update_state_and_vote_if_able<
         }
         if let Ok(vote) = QuorumVote::<TYPES>::create_signed_vote(
             QuorumData {
-                leaf_commit: proposed_leaf.commit(),
+                leaf_commit: proposed_leaf.commit(upgrade_lock).await,
             },
             view,
             &public_key,
@@ -861,15 +868,17 @@ pub async fn update_state_and_vote_if_able<
         cur_view,
         View {
             view_inner: ViewInner::Leaf {
-                leaf: proposed_leaf.commit(),
+                leaf: proposed_leaf.commit(upgrade_lock).await,
                 state: Arc::clone(&state),
                 delta: Some(Arc::clone(&delta)),
             },
         },
     ) {
         tracing::trace!("{e:?}");
     }
-    consensus.update_saved_leaves(proposed_leaf.clone());
+    consensus
+        .update_saved_leaves(proposed_leaf.clone(), upgrade_lock)
+        .await;
     let new_leaves = consensus.saved_leaves().clone();
     let new_state = consensus.validated_state_map().clone();
     drop(consensus);

crates/task-impls/src/helpers.rs

@@ -108,7 +108,7 @@ pub(crate) async fn fetch_proposal<TYPES: NodeType, V: Versions>(
                         hs_event.as_ref()
                     {
                         // Make sure that the quorum_proposal is valid
-                        if quorum_proposal.validate_signature(&mem).is_ok() {
+                        if quorum_proposal.validate_signature(&mem, upgrade_lock).await.is_ok() {
                             proposal = Some(quorum_proposal.clone());
                         }
 
@@ -140,7 +140,7 @@ pub(crate) async fn fetch_proposal<TYPES: NodeType, V: Versions>(
 
     let view = View {
         view_inner: ViewInner::Leaf {
-            leaf: leaf.commit(),
+            leaf: leaf.commit(upgrade_lock).await,
             state,
             delta: None,
         },
@@ -149,7 +149,9 @@ pub(crate) async fn fetch_proposal<TYPES: NodeType, V: Versions>(
         tracing::trace!("{e:?}");
     }
 
-    consensus_write.update_saved_leaves(leaf.clone());
+    consensus_write
+        .update_saved_leaves(leaf.clone(), upgrade_lock)
+        .await;
     broadcast_event(
         HotShotEvent::ValidatedStateUpdated(view_number, view).into(),
         &event_sender,
@@ -410,7 +412,7 @@ pub(crate) async fn parent_leaf_and_state<TYPES: NodeType, V: Versions>(
 
     let reached_decided = leaf.view_number() == consensus_reader.last_decided_view();
     let parent_leaf = leaf.clone();
-    let original_parent_hash = parent_leaf.commit();
+    let original_parent_hash = parent_leaf.commit(upgrade_lock).await;
     let mut next_parent_hash = original_parent_hash;
 
     // Walk back until we find a decide
@@ -460,7 +462,7 @@ pub async fn validate_proposal_safety_and_liveness<
 
     let proposed_leaf = Leaf::from_quorum_proposal(&proposal.data);
     ensure!(
-        proposed_leaf.parent_commitment() == parent_leaf.commit(),
+        proposed_leaf.parent_commitment() == parent_leaf.commit(&upgrade_lock).await,
         "Proposed leaf does not extend the parent leaf."
     );
 
@@ -469,7 +471,7 @@ pub async fn validate_proposal_safety_and_liveness<
     );
     let view = View {
         view_inner: ViewInner::Leaf {
-            leaf: proposed_leaf.commit(),
+            leaf: proposed_leaf.commit(&upgrade_lock).await,
             state,
             delta: None, // May be updated to `Some` in the vote task.
         },
@@ -480,7 +482,9 @@ pub async fn validate_proposal_safety_and_liveness<
         if let Err(e) = consensus_write.update_validated_state_map(view_number, view.clone()) {
             tracing::trace!("{e:?}");
         }
-        consensus_write.update_saved_leaves(proposed_leaf.clone());
+        consensus_write
+            .update_saved_leaves(proposed_leaf.clone(), &upgrade_lock)
+            .await;
 
         // Update our internal storage of the proposal. The proposal is valid, so
         // we swallow this error and just log if it occurs.
@@ -602,7 +606,9 @@ pub async fn validate_proposal_view_and_certs<TYPES: NodeType, V: Versions>(
     );
 
     // Validate the proposal's signature. This should also catch if the leaf_commitment does not equal our calculated parent commitment
-    proposal.validate_signature(quorum_membership)?;
+    proposal
+        .validate_signature(quorum_membership, upgrade_lock)
+        .await?;
 
     // Verify a timeout certificate OR a view sync certificate exists and is valid.
     if proposal.data.justify_qc.view_number() != view - 1 {

crates/task-impls/src/quorum_proposal/handlers.rs

@@ -13,7 +13,6 @@ use anyhow::{ensure, Context, Result};
 use async_broadcast::{Receiver, Sender};
 use async_compatibility_layer::art::{async_sleep, async_spawn};
 use async_lock::RwLock;
-use committable::Committable;
 use hotshot_task::{
     dependency::{Dependency, EventDependency},
     dependency_task::HandleDepOutput,
@@ -211,13 +210,15 @@ impl<TYPES: NodeType, V: Versions> ProposalDependencyHandle<TYPES, V> {
 
         let proposed_leaf = Leaf::from_quorum_proposal(&proposal);
         ensure!(
-            proposed_leaf.parent_commitment() == parent_leaf.commit(),
+            proposed_leaf.parent_commitment() == parent_leaf.commit(&self.upgrade_lock).await,
             "Proposed leaf parent does not equal high qc"
         );
 
-        let signature =
-            TYPES::SignatureKey::sign(&self.private_key, proposed_leaf.commit().as_ref())
-                .context("Failed to compute proposed_leaf.commit()")?;
+        let signature = TYPES::SignatureKey::sign(
+            &self.private_key,
+            proposed_leaf.commit(&self.upgrade_lock).await.as_ref(),
+        )
+        .context("Failed to compute proposed_leaf.commit()")?;
 
         let message = Proposal {
             data: proposal,

crates/task-impls/src/quorum_proposal_recv/handlers.rs

@@ -55,7 +55,7 @@ async fn validate_proposal_liveness<TYPES: NodeType, I: NodeImplementation<TYPES
     );
     let view = View {
         view_inner: ViewInner::Leaf {
-            leaf: leaf.commit(),
+            leaf: leaf.commit(&task_state.upgrade_lock).await,
             state,
             delta: None, // May be updated to `Some` in the vote task.
         },
@@ -64,7 +64,9 @@ async fn validate_proposal_liveness<TYPES: NodeType, I: NodeImplementation<TYPES
     if let Err(e) = consensus_write.update_validated_state_map(view_number, view.clone()) {
         tracing::trace!("{e:?}");
     }
-    consensus_write.update_saved_leaves(leaf.clone());
+    consensus_write
+        .update_saved_leaves(leaf.clone(), &task_state.upgrade_lock)
+        .await;
 
     if let Err(e) = task_state
         .storage