Sishan/merge sig agg to main 0821

Cargo.toml

@@ -221,6 +221,9 @@ tokio = { version = "1", optional = true, features = [
 ] }
 
 tracing = "0.1.37"
+ethereum-types = { version = "0.14.1", features = ["impl-serde"] }
+bitvec = { version = "1.0.1", default-features = false, features = ["alloc", "atomic", "serde"] }
+typenum = { version = "1.16.0" }
 
 [dev-dependencies]
 async-std = { version = "1.12.0", features = ["attributes"] }

consensus/Cargo.toml

@@ -57,4 +57,8 @@ tokio = { version = "1", optional = true, features = [
     "tracing",
 ] }
 tracing = "0.1.37"
-time = "0.3.23"
+time = "0.3.22" # 0.3.23
+bitvec = { version = "1.0.1", default-features = false, features = ["alloc", "atomic", "serde"] }
+hotshot-primitives = { git = "https://github.com/EspressoSystems/hotshot-primitives",  branch = 'hotshot-compat'} # rev = "4aee90c" for 'hotshot-compat'
+jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", branch = 'hotshot-compat'} # rev = "470a833" for branch = 'hotshot-compat'
+blake3 = { version = "1.3.3", features = ["traits-preview"] }

consensus/src/da_member.rs

@@ -120,15 +120,13 @@ impl<
                                 if view_leader_key != sender {
                                     continue;
                                 }
-
                                 let block_commitment = p.data.deltas.commit();
                                 if !view_leader_key
                                     .validate(&p.signature, block_commitment.as_ref())
                                 {
                                     warn!(?p.signature, "Could not verify proposal.");
                                     continue;
                                 }
-
                                 let vote_token = self.exchange.make_vote_token(self.cur_view);
                                 match vote_token {
                                     Err(e) => {

consensus/src/sequencing_leader.rs

@@ -8,11 +8,11 @@ use async_compatibility_layer::{
     async_primitives::subscribable_rwlock::{ReadView, SubscribableRwLock},
 };
 use async_lock::{Mutex, RwLock};
+use bitvec::prelude::*;
 use commit::Commitment;
 use commit::Committable;
 use either::Either;
 use either::{Left, Right};
-use hotshot_types::data::QuorumProposal;
 use hotshot_types::message::Message;
 use hotshot_types::traits::election::CommitteeExchangeType;
 use hotshot_types::traits::election::ConsensusExchange;
@@ -22,8 +22,8 @@ use hotshot_types::traits::node_implementation::{
 };
 use hotshot_types::traits::state::State;
 use hotshot_types::{
-    certificate::{DACertificate, QuorumCertificate, YesNoSignature},
-    data::{DAProposal, SequencingLeaf},
+    certificate::{AssembledSignature, DACertificate, QuorumCertificate},
+    data::{DAProposal, QuorumProposal, SequencingLeaf},
     message::{
         CommitteeConsensusMessage, ConsensusMessageType, GeneralConsensusMessage, InternalTrigger,
         ProcessedCommitteeConsensusMessage, ProcessedGeneralConsensusMessage,
@@ -98,16 +98,22 @@ where
         &self,
         cur_view: TYPES::Time,
         threshold: NonZeroU64,
+        total_nodes_num: usize,
         block_commitment: Commitment<<TYPES as NodeType>::BlockType>,
     ) -> Option<DACertificate<TYPES>> {
         let lock = self.vote_collection_chan.lock().await;
         let mut accumulator = VoteAccumulator {
             total_vote_outcomes: HashMap::new(),
+            da_vote_outcomes: HashMap::new(),
             yes_vote_outcomes: HashMap::new(),
             no_vote_outcomes: HashMap::new(),
             viewsync_precommit_vote_outcomes: HashMap::new(),
+            viewsync_commit_vote_outcomes: HashMap::new(),
+            viewsync_finalize_vote_outcomes: HashMap::new(),
             success_threshold: threshold,
             failure_threshold: threshold,
+            sig_lists: Vec::new(),
+            signers: bitvec![0; total_nodes_num],
         };
 
         while let Ok(msg) = lock.recv().await {
@@ -157,9 +163,8 @@ where
                             }
                             Either::Right(qc) => {
                                 match qc.clone().signatures {
-                                    YesNoSignature::Yes(map) => {
-                                        info!("Number of DA signatures in this QC: {}", map.len());
-                                    }
+                                    AssembledSignature::Yes(_signature) => {}
+                                    AssembledSignature::DA(_signature) => {}
                                     _ => unimplemented!(),
                                 };
                                 return Some(qc);
@@ -297,6 +302,7 @@ where
             .wait_for_votes(
                 self.cur_view,
                 self.committee_exchange.success_threshold(),
+                self.committee_exchange.total_nodes(),
                 block_commitment,
             )
             .await
@@ -385,7 +391,6 @@ where
             dac: Some(self.cert),
             proposer_id: leaf.proposer_id,
         };
-
         let message =
             SequencingMessage::<TYPES, I>(Left(GeneralConsensusMessage::Proposal(Proposal {
                 data: proposal,
@@ -459,14 +464,18 @@ where
     pub async fn run_view(self) -> QuorumCertificate<TYPES, SequencingLeaf<TYPES>> {
         let mut qcs = HashSet::<QuorumCertificate<TYPES, SequencingLeaf<TYPES>>>::new();
         qcs.insert(self.generic_qc.clone());
-
         let mut accumulator = VoteAccumulator {
             total_vote_outcomes: HashMap::new(),
+            da_vote_outcomes: HashMap::new(),
             yes_vote_outcomes: HashMap::new(),
             no_vote_outcomes: HashMap::new(),
             viewsync_precommit_vote_outcomes: HashMap::new(),
+            viewsync_commit_vote_outcomes: HashMap::new(),
+            viewsync_finalize_vote_outcomes: HashMap::new(),
             success_threshold: self.quorum_exchange.success_threshold(),
             failure_threshold: self.quorum_exchange.failure_threshold(),
+            sig_lists: Vec::new(),
+            signers: bitvec![0; self.quorum_exchange.total_nodes()],
         };
 
         let lock = self.vote_collection_chan.lock().await;
@@ -501,10 +510,7 @@ where
                                     }
                                     Either::Right(qc) => {
                                         match qc.clone().signatures {
-                                            YesNoSignature::Yes(map) => info!(
-                                                "Number of qurorum signatures in this QC: {}",
-                                                map.len()
-                                            ),
+                                            AssembledSignature::Yes(_signature) => {}
                                             _ => unimplemented!(),
                                         };
                                         return qc;

examples/infra/mod.rs

@@ -7,7 +7,7 @@ use hotshot_types::traits::node_implementation::NodeType;
 use hotshot_types::traits::signature_key::SignatureKey;
 use libp2p::{
     identity::{
-        ed25519::{Keypair as EdKeypair, SecretKey},
+        bn254::{Keypair as EdKeypair, SecretKey},
         Keypair,
     },
     multiaddr::{self},
@@ -57,6 +57,10 @@ pub fn load_config_from_file<TYPES: NodeType>(
         })
         .collect();
 
+    config.config.known_nodes_with_stake = (0..config.config.total_nodes.get())
+        .map(|node_id| config.config.known_nodes[node_id].get_stake_table_entry(1u64))
+        .collect();
+
     config
 }
 

examples/infra/modDA.rs

@@ -225,12 +225,12 @@ pub trait RunDA<
 
         let config = self.get_config();
 
+        // Get KeyPair for certificate Aggregation
         let (pk, sk) =
             TYPES::SignatureKey::generated_from_seed_indexed(config.seed, config.node_index);
-        let ek = jf_primitives::aead::KeyPair::generate(&mut rand_chacha::ChaChaRng::from_seed(
-            config.seed,
-        ));
         let known_nodes = config.config.known_nodes.clone();
+        let known_nodes_with_stake = config.config.known_nodes_with_stake.clone();
+        let entry = pk.get_stake_table_entry(1u64);
 
         let da_network = self.get_da_network();
         let quorum_network = self.get_quorum_network();
@@ -252,6 +252,7 @@ pub trait RunDA<
         );
 
         let exchanges = NODE::Exchanges::create(
+            known_nodes_with_stake.clone(),
             known_nodes.clone(),
             (quorum_election_config, committee_election_config),
             (
@@ -260,8 +261,8 @@ pub trait RunDA<
                 view_sync_network.clone(),
             ),
             pk.clone(),
+            entry.clone(),
             sk.clone(),
-            ek.clone(),
         );
 
         SystemContext::init(

orchestrator/src/client.rs

@@ -5,7 +5,7 @@ use async_compatibility_layer::art::async_sleep;
 use clap::Parser;
 use futures::{Future, FutureExt};
 
-use hotshot_types::traits::node_implementation::NodeType;
+use hotshot_types::traits::{node_implementation::NodeType, signature_key::SignatureKey};
 use surf_disco::{error::ClientError, Client};
 
 /// Holds the client connection to the orchestrator
@@ -63,11 +63,19 @@ impl OrchestratorClient {
     pub async fn get_config_from_orchestrator<TYPES: NodeType>(
         &self,
         node_index: u16,
-    ) -> NetworkConfig<TYPES::SignatureKey, TYPES::ElectionConfigType> {
+    ) -> NetworkConfig<
+        TYPES::SignatureKey,
+        <TYPES::SignatureKey as SignatureKey>::StakeTableEntry,
+        TYPES::ElectionConfigType,
+    > {
         let f = |client: Client<ClientError>| {
             async move {
                 let config: Result<
-                    NetworkConfig<TYPES::SignatureKey, TYPES::ElectionConfigType>,
+                    NetworkConfig<
+                        TYPES::SignatureKey,
+                        <TYPES::SignatureKey as SignatureKey>::StakeTableEntry,
+                        TYPES::ElectionConfigType,
+                    >,
                     ClientError,
                 > = client
                     .post(&format!("api/config/{node_index}"))

orchestrator/src/config.rs

@@ -1,7 +1,6 @@
 use hotshot_types::{ExecutionType, HotShotConfig};
 use std::net::{Ipv4Addr, SocketAddr};
 use std::{net::IpAddr, num::NonZeroUsize, time::Duration};
-
 #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
 pub struct Libp2pConfig {
     pub bootstrap_nodes: Vec<(SocketAddr, Vec<u8>)>,
@@ -53,7 +52,7 @@ pub struct WebServerConfig {
 }
 
 #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
-pub struct NetworkConfig<KEY, ELECTIONCONFIG> {
+pub struct NetworkConfig<KEY, ENTRY, ELECTIONCONFIG> {
     pub rounds: usize,
     pub transactions_per_round: usize,
     pub node_index: u64,
@@ -63,12 +62,12 @@ pub struct NetworkConfig<KEY, ELECTIONCONFIG> {
     pub key_type_name: String,
     pub election_config_type_name: String,
     pub libp2p_config: Option<Libp2pConfig>,
-    pub config: HotShotConfig<KEY, ELECTIONCONFIG>,
+    pub config: HotShotConfig<KEY, ENTRY, ELECTIONCONFIG>,
     pub web_server_config: Option<WebServerConfig>,
     pub da_web_server_config: Option<WebServerConfig>,
 }
 
-impl<K, E> Default for NetworkConfig<K, E> {
+impl<K, ENTRY, E> Default for NetworkConfig<K, ENTRY, E> {
     fn default() -> Self {
         Self {
             rounds: default_rounds(),
@@ -115,7 +114,7 @@ fn default_web_server_config() -> Option<WebServerConfig> {
     None
 }
 
-impl<K, E> From<NetworkConfigFile> for NetworkConfig<K, E> {
+impl<K, ENTRY, E> From<NetworkConfigFile> for NetworkConfig<K, ENTRY, E> {
     fn from(val: NetworkConfigFile) -> Self {
         NetworkConfig {
             rounds: val.rounds,
@@ -181,14 +180,15 @@ pub struct HotShotConfigFile {
     pub propose_max_round_time: Duration,
 }
 
-impl<K, E> From<HotShotConfigFile> for HotShotConfig<K, E> {
+impl<K, ENTRY, E> From<HotShotConfigFile> for HotShotConfig<K, ENTRY, E> {
     fn from(val: HotShotConfigFile) -> Self {
         HotShotConfig {
             execution_type: ExecutionType::Continuous,
             total_nodes: val.total_nodes,
             max_transactions: val.max_transactions,
             min_transactions: val.min_transactions,
             known_nodes: Vec::new(),
+            known_nodes_with_stake: Vec::new(),
             da_committee_size: val.committee_nodes,
             next_view_timeout: val.next_view_timeout,
             timeout_ratio: val.timeout_ratio,

orchestrator/src/lib.rs

@@ -37,11 +37,11 @@ pub fn libp2p_generate_indexed_identity(seed: [u8; 32], index: u64) -> Keypair {
 }
 
 #[derive(Default, Clone)]
-struct OrchestratorState<KEY, ELECTION> {
+struct OrchestratorState<KEY, ENTRY, ELECTION> {
     /// Tracks the latest node index we have generated a configuration for
     latest_index: u16,
     /// The network configuration
-    config: NetworkConfig<KEY, ELECTION>,
+    config: NetworkConfig<KEY, ENTRY, ELECTION>,
     /// Whether nodes should start their HotShot instances
     /// Will be set to true once all nodes post they are ready to start
     start: bool,
@@ -52,9 +52,9 @@ struct OrchestratorState<KEY, ELECTION> {
 }
 
 impl<KEY: SignatureKey + 'static, ELECTION: ElectionConfig + 'static>
-    OrchestratorState<KEY, ELECTION>
+    OrchestratorState<KEY, KEY::StakeTableEntry, ELECTION>
 {
-    pub fn new(network_config: NetworkConfig<KEY, ELECTION>) -> Self {
+    pub fn new(network_config: NetworkConfig<KEY, KEY::StakeTableEntry, ELECTION>) -> Self {
         let mut web_client = None;
         if network_config.web_server_config.is_some() {
             let base_url = "http://0.0.0.0/9000".to_string().parse().unwrap();
@@ -70,18 +70,19 @@ impl<KEY: SignatureKey + 'static, ELECTION: ElectionConfig + 'static>
     }
 }
 
-pub trait OrchestratorApi<KEY, ELECTION> {
+pub trait OrchestratorApi<KEY, ENTRY, ELECTION> {
     fn post_identity(&mut self, identity: IpAddr) -> Result<u16, ServerError>;
     fn post_getconfig(
         &mut self,
         node_index: u16,
-    ) -> Result<NetworkConfig<KEY, ELECTION>, ServerError>;
+    ) -> Result<NetworkConfig<KEY, ENTRY, ELECTION>, ServerError>;
     fn get_start(&self) -> Result<bool, ServerError>;
     fn post_ready(&mut self) -> Result<(), ServerError>;
     fn post_run_results(&mut self) -> Result<(), ServerError>;
 }
 
-impl<KEY, ELECTION> OrchestratorApi<KEY, ELECTION> for OrchestratorState<KEY, ELECTION>
+impl<KEY, ELECTION> OrchestratorApi<KEY, KEY::StakeTableEntry, ELECTION>
+    for OrchestratorState<KEY, KEY::StakeTableEntry, ELECTION>
 where
     KEY: serde::Serialize + Clone + SignatureKey,
     ELECTION: serde::Serialize + Clone + Send,
@@ -142,7 +143,7 @@ where
     fn post_getconfig(
         &mut self,
         _node_index: u16,
-    ) -> Result<NetworkConfig<KEY, ELECTION>, ServerError> {
+    ) -> Result<NetworkConfig<KEY, KEY::StakeTableEntry, ELECTION>, ServerError> {
         if self.config.libp2p_config.is_some() {
             let libp2p_config = self.config.clone().libp2p_config.unwrap();
             if libp2p_config.bootstrap_nodes.len()
@@ -185,11 +186,12 @@ where
 }
 
 /// Sets up all API routes
-fn define_api<KEY, ELECTION, State>() -> Result<Api<State, ServerError>, ApiError>
+fn define_api<KEY, ELECTION, ENTRY, State>() -> Result<Api<State, ServerError>, ApiError>
 where
     State: 'static + Send + Sync + ReadState + WriteState,
-    <State as ReadState>::State: Send + Sync + OrchestratorApi<KEY, ELECTION>,
+    <State as ReadState>::State: Send + Sync + OrchestratorApi<KEY, ENTRY, ELECTION>,
     KEY: serde::Serialize,
+    ENTRY: serde::Serialize,
     ELECTION: serde::Serialize,
 {
     let api_toml = toml::from_str::<toml::Value>(include_str!(concat!(
@@ -232,7 +234,7 @@ where
 
 /// Runs the orchestrator
 pub async fn run_orchestrator<KEY, ELECTION>(
-    network_config: NetworkConfig<KEY, ELECTION>,
+    network_config: NetworkConfig<KEY, KEY::StakeTableEntry, ELECTION>,
     host: IpAddr,
     port: u16,
 ) -> io::Result<()>
@@ -242,10 +244,10 @@ where
 {
     let api = define_api().map_err(|_e| io::Error::new(ErrorKind::Other, "Failed to define api"));
 
-    let state: RwLock<OrchestratorState<KEY, ELECTION>> =
+    let state: RwLock<OrchestratorState<KEY, KEY::StakeTableEntry, ELECTION>> =
         RwLock::new(OrchestratorState::new(network_config));
 
-    let mut app = App::<RwLock<OrchestratorState<KEY, ELECTION>>, ServerError>::with_state(state);
+    let mut app = App::<RwLock<OrchestratorState<KEY, KEY::StakeTableEntry, ELECTION>>, ServerError>::with_state(state);
     app.register_module("api", api.unwrap())
         .expect("Error registering api");
     tracing::error!("lisening on {:?}:{:?}", host, port);