Phase 2 Integration Fixes (#78)

* chunking works, make this into a cli * wip * mabye works end to end * mabye works end to end * fast deserialization * makes reading params fast * add proof example to enjoy fast deserialization * try to fix non working proof * try to fix * fix finally * remove celo-bls test and fix wasm * fixed compiler errors in phase2/lib.rs * removed swp file * wasm compiles * removed swp files * removed diff.txt * changed dependency commit * fixed zexe commit * changed .toml files * changed default features in phase2 * added phase2-cli to workspace * changed epoch-snark import branch * made groth16 utils public * removed swp file * added .swp to gitignore * now compiles * prepare_phase2 * loading circuit seems to work * changed phase2 compression * changed prepare_phase2 compression * info statements * clean up circuit loading * more info statements in celo bls * changed celo bls commit * lock file changed * changed depdendency * .toml * added prints * more println * more println * changed new challenge compression * removed info statements * phase 2 new challenge returns number of chunks * no compression in phase 2 contribute * no compression contribute or verify phase 2 * no compression phase 2 verify contribute combine * print statements read groth16 fast * compiler error fixed * full correctness checks combine * compiler error * no correctness check full file * no correctness check query file * print in read element * correctness check new challenge * removed print in read element * print a_g1 in eval * compiler error * print coeffs_g1 * print at in eval * print ret in dor product * print a_g1 in eval * added println to new challenge phase 2 * print dot product before sum * panic after a_g1 computed * print dot product input * Print at matrices * print b_g1 in eval * print c_g1 in eval * print b_g2 in eval * print gamma_abc_g1 in eval * never check correctness for a_g1+b_g1+b_g2 queries * remove print statements groth16 utils * println for print_hash * no print statements * fixed a,b query correctness checks * no compression * removed diff.txt * compression enabled phase 2 matching phase 1 * phase everything compressed * phase 2 verify computes new challenge file * changed order of parameters phase 2 verify * phase 2 compression matches phase 1 * compiler warnings fixed * no compression used * cleanup * initial challenge parameter added to phase 2 combine * cleanup * format * no compression phase 2 * asymmetric compression phase 2 * removed diff.txt * branch on compressed parameters * branch full contribution verify * phase 2 new challenge writes challenge list * new line in challenge list file * removed phase 1 wasm and phase 2 cli readme * minor edits * consolidated phase 2 compression options * minor memory optimization * tests compile * commented out generate file * tests pass * tests work * removed comment * no compiler warnings tests * removed incomplete circuit generation file Co-authored-by: Kobi Gurkan <[email protected]> Co-authored-by: Sami Mäkelä <[email protected]>
celo-org · May 28, 2021 · 4d76416 · 4d76416
1 parent efa4c5f
commit 4d76416
Show file tree

Hide file tree

Showing 14 changed files with 569 additions and 146 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
 
 [patch.'https://github.com/scipr-lab/zexe']
 algebra_core = { git = "https://github.com/celo-org/zexe", package = "algebra-core", default-features = false, features = ["derive"] }
+algebra_core_derive = { git = "https://github.com/celo-org/zexe", package = "algebra-core-derive", default-features = false, features = [] }
 algebra = { git = "https://github.com/celo-org/zexe", package = "algebra", default-features = false, features = ["ed_on_bls12_381", "bls12_381", "bls12_377"] }
 crypto-primitives = { git = "https://github.com/celo-org/zexe", package = "crypto-primitives", default-features = false, features = ["ed_on_bls12_381", "bls12_381", "bls12_377"] }
 fft = { git = "https://github.com/celo-org/zexe", package = "ff-fft", default-features = false, features = ["ed_on_bls12_381", "bls12_381", "bls12_377"] }

diff --git a/phase2-cli/Cargo.toml b/phase2-cli/Cargo.toml
@@ -12,6 +12,10 @@ edition = "2018"
 phase2 = { path = "../phase2", default-features = false }
 setup-utils = { path = "../setup-utils", default-features = false }
 
+ark-bls12-377 = { git = "https://github.com/arkworks-rs/curves", features = [ "r1cs" ], rev = "6ed2450b5505de5d451f629bb6642e3977bf66e2" }
+ark-relations = { git = "https://github.com/arkworks-rs/snark", features = [ "std" ], rev = "8d9055d5397b510716ad2951ce1f18675aebe7c8" }
+ark-serialize = { git = "https://github.com/arkworks-rs/algebra", features = [ "derive" ], rev = "8d76d181de0079b7e5a92f6e1133e7be635efdd3" }
+
 algebra = { git = "https://github.com/scipr-lab/zexe", version = "0.1.1-alpha.0", package = "algebra", features = ["bls12_377", "bw6_761", "derive"] }
 r1cs_core = { git = "https://github.com/scipr-lab/zexe", version = "0.1.1-alpha.0", package = "r1cs-core" }
 groth16 = { git = "https://github.com/scipr-lab/zexe", version = "0.1.1-alpha.0", package = "groth16", features = [] }
@@ -27,6 +31,8 @@ phase1 = { path = "../phase1" }
 #epoch-snark = { git = "https://github.com/celo-org/celo-bls-snark-rs", branch = "straka/integration_info" }
 epoch-snark = { git = "https://github.com/celo-org/celo-bls-snark-rs", rev = "0995862883b7f88da5ef8e9102f4173a3f3bf20d" }
 
+anyhow = "1"
+
 [dev-dependencies]
 rand_xorshift = { version = "0.2" }
 wasm-bindgen-test = { version = "0.3.15" }

diff --git a/phase2-cli/src/bin/phase2.rs b/phase2-cli/src/bin/phase2.rs
@@ -30,11 +30,11 @@ fn execute_cmd<E: Engine>(opts: Phase2Opts) {
             new_challenge(
                 &opt.challenge_fname,
                 &opt.challenge_hash_fname,
+                &opt.challenge_list_fname,
                 opts.chunk_size,
                 &opt.phase1_fname,
                 opt.phase1_powers,
-                opt.num_validators,
-                opt.num_epochs,
+                &opt.circuit_fname,
             );
         }
         Command::Contribute(opt) => {
@@ -65,6 +65,7 @@ fn execute_cmd<E: Engine>(opts: Phase2Opts) {
                 &opt.new_challenge_fname,
                 &opt.new_challenge_hash_fname,
                 opts.subgroup_check_mode,
+                false,
             );
         }
         Command::Combine(opt) => {
@@ -73,6 +74,7 @@ fn execute_cmd<E: Engine>(opts: Phase2Opts) {
                 &opt.initial_full_fname,
                 &opt.response_list_fname,
                 &opt.combined_fname,
+                false,
             );
         }
     };

diff --git a/phase2-cli/src/combine.rs b/phase2-cli/src/combine.rs
@@ -1,4 +1,4 @@
-use phase2::parameters::MPCParameters;
+use phase2::parameters::{verify_transcript, MPCParameters};
 use setup_utils::{print_hash, CheckForCorrectness, SubgroupCheckMode, UseCompression};
 
 use algebra::{CanonicalSerialize, BW6_761};
@@ -7,15 +7,14 @@ use std::fs::File;
 use std::io::{BufRead, BufReader};
 use tracing::info;
 
-const INITIAL_IS_COMPRESSED: UseCompression = UseCompression::No;
-const CONTRIBUTION_IS_COMPRESSED: UseCompression = UseCompression::Yes;
-const COMBINED_IS_COMPRESSED: UseCompression = UseCompression::No;
+use crate::{COMBINED_IS_COMPRESSED, COMPRESS_CONTRIBUTE_INPUT, COMPRESS_CONTRIBUTE_OUTPUT};
 
 pub fn combine(
     initial_query_filename: &str,
     initial_full_filename: &str,
     response_list_filename: &str,
     combined_filename: &str,
+    combine_initial: bool,
 ) {
     info!("Combining phase 2");
 
@@ -25,7 +24,7 @@ pub fn combine(
     let full_contents = std::fs::read(initial_full_filename).expect("should have initial full parameters");
     let full_parameters = MPCParameters::<BW6_761>::read_fast(
         full_contents.as_slice(),
-        INITIAL_IS_COMPRESSED,
+        UseCompression::No,
         CheckForCorrectness::No,
         false,
         SubgroupCheckMode::Auto,
@@ -36,20 +35,25 @@ pub fn combine(
         std::io::Cursor::new(std::fs::read(initial_query_filename).expect("should have read initial query"));
     let query_parameters = MPCParameters::<BW6_761>::read_groth16_fast(
         &mut query_contents,
-        INITIAL_IS_COMPRESSED,
+        UseCompression::No,
         CheckForCorrectness::No,
         false,
         SubgroupCheckMode::Auto,
     )
     .expect("should have deserialized initial query params");
 
+    let parameters_compressed = if combine_initial {
+        COMPRESS_CONTRIBUTE_INPUT
+    } else {
+        COMPRESS_CONTRIBUTE_OUTPUT
+    };
     let mut all_parameters = vec![];
     for line in response_list_reader.lines() {
         let line = line.expect("should have read line");
         let contents = std::fs::read(line).expect("should have read response");
         let parameters = MPCParameters::<BW6_761>::read_fast(
             contents.as_slice(),
-            CONTRIBUTION_IS_COMPRESSED,
+            parameters_compressed,
             CheckForCorrectness::No,
             false,
             SubgroupCheckMode::Auto,
@@ -61,9 +65,14 @@ pub fn combine(
     let combined =
         MPCParameters::<BW6_761>::combine(&query_parameters, &all_parameters).expect("should have combined parameters");
 
-    let contributions_hash = full_parameters
-        .verify(&combined)
-        .expect("should have verified successfully");
+    let contributions_hash = if combine_initial {
+        verify_transcript(full_parameters.cs_hash, &combined.contributions).expect("should have verified successfully")
+    } else {
+        full_parameters
+            .verify(&combined)
+            .expect("should have verified successfully")
+    };
+
     info!("Contributions hashes:");
     for contribution_hash in contributions_hash {
         print_hash(&contribution_hash[..]);

diff --git a/phase2-cli/src/contribute.rs b/phase2-cli/src/contribute.rs
@@ -1,15 +1,13 @@
 use phase2::parameters::MPCParameters;
-use setup_utils::{calculate_hash, print_hash, BatchExpMode, CheckForCorrectness, SubgroupCheckMode, UseCompression};
+use setup_utils::{calculate_hash, print_hash, BatchExpMode, CheckForCorrectness, SubgroupCheckMode};
 
 use algebra::BW6_761;
 
+use crate::{COMPRESS_CONTRIBUTE_INPUT, COMPRESS_CONTRIBUTE_OUTPUT};
 use rand::Rng;
 use std::io::Write;
 use tracing::info;
 
-const COMPRESSED_INPUT: UseCompression = UseCompression::No;
-const COMPRESSED_OUTPUT: UseCompression = UseCompression::Yes;
-
 pub fn contribute(
     challenge_filename: &str,
     challenge_hash_filename: &str,
@@ -33,7 +31,7 @@ pub fn contribute(
 
     let mut parameters = MPCParameters::<BW6_761>::read_fast(
         challenge_contents.as_slice(),
-        COMPRESSED_INPUT,
+        COMPRESS_CONTRIBUTE_INPUT,
         check_input_correctness,
         false,
         SubgroupCheckMode::Auto,
@@ -44,7 +42,7 @@ pub fn contribute(
         .expect("should have successfully contributed");
     let mut serialized_response = vec![];
     parameters
-        .write(&mut serialized_response, COMPRESSED_OUTPUT)
+        .write(&mut serialized_response, COMPRESS_CONTRIBUTE_OUTPUT)
         .expect("should have written input");
     std::fs::File::create(response_filename)
         .expect("unable to create response")

diff --git a/phase2-cli/src/lib.rs b/phase2-cli/src/lib.rs
@@ -22,10 +22,14 @@ use setup_utils::{
         batch_exp_mode_from_str, contribution_mode_from_str, curve_from_str, proving_system_from_str,
         subgroup_check_mode_from_str,
     },
-    BatchExpMode, SubgroupCheckMode,
+    BatchExpMode, SubgroupCheckMode, UseCompression,
 };
 use std::default::Default;
 
+pub const COMPRESS_CONTRIBUTE_INPUT: UseCompression = UseCompression::No;
+pub const COMPRESS_CONTRIBUTE_OUTPUT: UseCompression = UseCompression::Yes;
+pub const COMBINED_IS_COMPRESSED: UseCompression = UseCompression::No;
+
 #[derive(Debug, Options, Clone)]
 pub struct Phase2Opts {
     help: bool,
@@ -98,6 +102,8 @@ pub struct NewOpts {
     pub challenge_fname: String,
     #[options(help = "the new challenge file hash", default = "challenge.verified.hash")]
     pub challenge_hash_fname: String,
+    #[options(help = "the list of challenge files", default = "new_challenge_list")]
+    pub challenge_list_fname: String,
     #[options(help = "phase 1 file name", default = "phase1")]
     pub phase1_fname: String,
     #[options(help = "phase 1 powers")]
@@ -106,6 +112,8 @@ pub struct NewOpts {
     pub num_validators: usize,
     #[options(help = "number of epochs")]
     pub num_epochs: usize,
+    #[options(help = "circuit file name", default = "circuit.constraints")]
+    pub circuit_fname: String,
 }
 
 // Options for the Contribute command

diff --git a/phase2-cli/src/new_challenge.rs b/phase2-cli/src/new_challenge.rs
@@ -1,28 +1,34 @@
+use phase2::load_circuit::Matrices;
 use phase2::parameters::MPCParameters;
 use setup_utils::{calculate_hash, print_hash, CheckForCorrectness, UseCompression};
 
-use algebra::{bw6_761::Fr, CanonicalSerialize, BW6_761};
-use r1cs_core::ConstraintSynthesizer;
-use r1cs_core::{ConstraintSystem, SynthesisMode};
-
-use epoch_snark::ValidatorSetUpdate;
+use crate::COMPRESS_CONTRIBUTE_INPUT;
+use algebra::{CanonicalDeserialize, CanonicalSerialize, BW6_761};
 use memmap::*;
-use std::{fs::OpenOptions, io::Write};
+use std::{fs::File, fs::OpenOptions, io::Read, io::Write};
 use tracing::info;
 
-const COMPRESS_NEW_CHALLENGE: UseCompression = UseCompression::No;
-
 pub fn new_challenge(
     challenge_filename: &str,
     challenge_hash_filename: &str,
+    challenge_list_filename: &str,
     chunk_size: usize,
     phase1_filename: &str,
     phase1_powers: usize,
-    num_validators: usize,
-    num_epochs: usize,
+    circuit_filename: &str,
 ) -> usize {
     info!("Generating phase 2");
 
+    let mut file = File::open(circuit_filename).unwrap();
+    let mut buffer = Vec::<u8>::new();
+    file.read_to_end(&mut buffer).unwrap();
+    let m = Matrices::<BW6_761>::deserialize(&*buffer).unwrap();
+
+    info!("Loaded circuit with {} constraints", m.num_constraints);
+
+    let phase2_size =
+        std::cmp::max(m.num_constraints, m.num_witness_variables + m.num_instance_variables).next_power_of_two();
+
     let reader = OpenOptions::new()
         .read(true)
         .write(true)
@@ -34,21 +40,9 @@ pub fn new_challenge(
             .expect("unable to create a memory map for input")
     };
 
-    let c = ValidatorSetUpdate::empty(num_validators, num_epochs, 0, None);
-    let counter = ConstraintSystem::<Fr>::new_ref();
-    counter.set_mode(SynthesisMode::Setup);
-    info!("About to generate constraints");
-    c.clone().generate_constraints(counter.clone()).unwrap();
-    info!("Finished generating constraints");
-    let phase2_size = std::cmp::max(
-        counter.num_constraints(),
-        counter.num_witness_variables() + counter.num_instance_variables(),
-    )
-    .next_power_of_two();
-
     let (full_mpc_parameters, query_parameters, all_mpc_parameters) =
         MPCParameters::<BW6_761>::new_from_buffer_chunked(
-            c,
+            m,
             &mut phase1_readable_map,
             UseCompression::No,
             CheckForCorrectness::No,
@@ -60,11 +54,11 @@ pub fn new_challenge(
 
     let mut serialized_mpc_parameters = vec![];
     full_mpc_parameters
-        .write(&mut serialized_mpc_parameters, COMPRESS_NEW_CHALLENGE)
+        .write(&mut serialized_mpc_parameters, COMPRESS_CONTRIBUTE_INPUT)
         .unwrap();
 
     let mut serialized_query_parameters = vec![];
-    match COMPRESS_NEW_CHALLENGE {
+    match COMPRESS_CONTRIBUTE_INPUT {
         UseCompression::No => query_parameters.serialize_uncompressed(&mut serialized_query_parameters),
         UseCompression::Yes => query_parameters.serialize(&mut serialized_query_parameters),
     }
@@ -84,15 +78,21 @@ pub fn new_challenge(
         .write_all(&serialized_query_parameters)
         .expect("unable to write serialized mpc parameters");
 
+    let mut challenge_list_file =
+        std::fs::File::create(challenge_list_filename).expect("unable to open new challenge list file");
+
     for (i, chunk) in all_mpc_parameters.iter().enumerate() {
         let mut serialized_chunk = vec![];
         chunk
-            .write(&mut serialized_chunk, COMPRESS_NEW_CHALLENGE)
+            .write(&mut serialized_chunk, COMPRESS_CONTRIBUTE_INPUT)
             .expect("unable to write chunk");
         std::fs::File::create(format!("{}.{}", challenge_filename, i))
             .expect("unable to open new challenge hash file")
             .write_all(&serialized_chunk)
             .expect("unable to write serialized mpc parameters");
+        challenge_list_file
+            .write(format!("{}.{}\n", challenge_filename, i).as_bytes())
+            .expect("unable to write challenge list");
     }
 
     std::fs::File::create(challenge_hash_filename)

diff --git a/phase2-cli/src/verify.rs b/phase2-cli/src/verify.rs
@@ -1,17 +1,14 @@
 use phase2::parameters::MPCParameters;
-use setup_utils::{calculate_hash, print_hash, CheckForCorrectness, SubgroupCheckMode, UseCompression};
+use setup_utils::{calculate_hash, print_hash, CheckForCorrectness, SubgroupCheckMode};
 
 use algebra::BW6_761;
 
+use crate::{COMBINED_IS_COMPRESSED, COMPRESS_CONTRIBUTE_INPUT, COMPRESS_CONTRIBUTE_OUTPUT};
 use memmap::MmapOptions;
 use std::fs::OpenOptions;
 use std::io::Write;
 use tracing::info;
 
-const PREVIOUS_CHALLENGE_IS_COMPRESSED: UseCompression = UseCompression::No;
-const CONTRIBUTION_IS_COMPRESSED: UseCompression = UseCompression::Yes;
-const NEW_CHALLENGE_IS_COMPRESSED: UseCompression = UseCompression::No;
-
 pub fn verify(
     challenge_filename: &str,
     challenge_hash_filename: &str,
@@ -22,6 +19,7 @@ pub fn verify(
     new_challenge_filename: &str,
     new_challenge_hash_filename: &str,
     subgroup_check_mode: SubgroupCheckMode,
+    verifying_full_contribution: bool,
 ) {
     info!("Verifying phase 2");
 
@@ -37,7 +35,7 @@ pub fn verify(
 
     let parameters_before = MPCParameters::<BW6_761>::read_fast(
         challenge_contents.as_slice(),
-        PREVIOUS_CHALLENGE_IS_COMPRESSED,
+        COMPRESS_CONTRIBUTE_INPUT,
         check_input_correctness,
         true,
         subgroup_check_mode,
@@ -54,9 +52,14 @@ pub fn verify(
     info!("`response` file contains decompressed points and has a hash:");
     print_hash(&response_hash);
 
+    let after_compressed = if verifying_full_contribution {
+        COMBINED_IS_COMPRESSED
+    } else {
+        COMPRESS_CONTRIBUTE_OUTPUT
+    };
     let parameters_after = MPCParameters::<BW6_761>::read_fast(
         response_contents.as_slice(),
-        CONTRIBUTION_IS_COMPRESSED,
+        after_compressed,
         check_output_correctness,
         true,
         subgroup_check_mode,
@@ -70,7 +73,7 @@ pub fn verify(
         .open(new_challenge_filename)
         .expect("unable to create new challenge file in this directory");
     parameters_after
-        .write(writer, NEW_CHALLENGE_IS_COMPRESSED)
+        .write(writer, COMPRESS_CONTRIBUTE_INPUT)
         .expect("unable to write new challenge file");
 
     // Read new challenge to create hash

diff --git a/phase2/src/lib.rs b/phase2/src/lib.rs
@@ -8,6 +8,8 @@ pub mod parameters;
 #[cfg(not(feature = "wasm"))]
 mod polynomial;
 
+pub mod load_circuit;
+
 pub mod chunked_groth16;
 
 cfg_if! {