Ec2/spec tests (#9)

* Script to download spec tests

* rip test-utils from zipline

* Update toolchain and alsoimport zipline

* can read tests

* TODO: Remove the logic w.r.t. execution because altair doesnt have it

* Remove eth1 execution checks

* Some tests work!

* prints

* Capella works

* cleanup

* Point to remote zipline

* spec ttest full prover

* k=22 for evm

* Convert zipline LightClientHeader to ours

* fix tests

* clippy

* fix import

* allow pull from sygmaprotocol/zipline private repo in gh actions:

* fixes

.github/workflows/tests.yml

@@ -19,6 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
+    - name: Give GitHub Actions access to sygmaprotocol/zipline
+      uses: webfactory/[email protected]
+      with:
+        ssh-private-key: ${{ secrets.SYGMA_REPO_PULL_KEY }}
     - uses: actions/checkout@v3
     - name: Install latest nightly-2022-10-28
       uses: dtolnay/rust-toolchain@stable

.gitignore

@@ -13,6 +13,11 @@ Cargo.lock
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
 
+# ignore downloaded spec tests
+consensus-spec-tests
+general.tar.gz
+minimal.tar.gz
+
 
 # Added by cargo
 

Cargo.toml

@@ -24,14 +24,16 @@ incremental = true
 [patch."https://github.com/privacy-scaling-explorations/halo2curves"]
 halo2curves = { git = "https://github.com/timoftime/halo2curves", branch = "dev/bls12_381" }
 # halo2curves = { path = "../halo2curves" }
+# [patch."https://github.com/timoftime/halo2curves"]
+# halo2curves = { path = "../halo2curves" }
 
 [patch."https://github.com/axiom-crypto/halo2-lib"]
-halo2-base = { git = "https://github.com/timoftime/halo2-lib", rev = "03c7baedeae208b21359e542b8683644b3eb1ac2", default-features = false, features = [
+halo2-base = { git = "https://github.com/timoftime/halo2-lib", rev = "1a2f896a5cd9ed4039a497697b83eafabec62519", default-features = false, features = [
     "halo2-pse",
     "display",
 ] }
-halo2-ecc = { git = "https://github.com/timoftime/halo2-lib", rev = "03c7baedeae208b21359e542b8683644b3eb1ac2", default-features = false }
-poseidon = { git = "https://github.com/timoftime/halo2-lib", rev = "03c7baedeae208b21359e542b8683644b3eb1ac2", default-features = false }
+halo2-ecc = { git = "https://github.com/timoftime/halo2-lib", rev = "1a2f896a5cd9ed4039a497697b83eafabec62519", default-features = false }
+poseidon = { git = "https://github.com/timoftime/halo2-lib", rev = "1a2f896a5cd9ed4039a497697b83eafabec62519", default-features = false }
 
 # halo2-base = { path = "../halo2-lib/halo2-base", default-features = false, features = [
 #     "halo2-pse",

justfile

@@ -1,15 +1,36 @@
 test:
     cargo test --workspace
+
 fmt:
     cargo fmt --all
+
 check:
     cargo check --all
+
 lint: fmt
     cargo clippy --all-targets --all-features --workspace
+
 setup-circuits:
     cargo run -r -- sync-step -o artifacts -k 22
     cargo run -r -- committee-update -o artifacts -k 18
+
 gen-step-evm-verifier:
     cargo run -r -- sync-step -c ./lightclient-circuits/config/sync_step.json -o evm-verifier ./contracts/snark-verifiers/sync_step.yul
+
 gen-rotation-evm-verifier:
     cargo run -r -- aggregation -c ./lightclient-circuits/config/aggregation.json --app-pk-path ./build/committee_update.pkey --app-config-path ./lightclient-circuits/config/committee_update.json -i ./rotation-snark -o evm-verifier ./contracts/snark-verifiers/committee_update_compressed.yul 
+
+# downloads spec tests and copies them to the right locations.
+download-spec-tests: clean-spec-tests
+    #!/usr/bin/env bash
+    if [[ ! -d 'consensus-spec-tests' ]]; then
+        echo "Downloading test data."
+        scripts/download_consensus_specs.sh
+    fi
+
+# deletes all the downloaded spec tests
+clean-spec-tests:
+    echo "Cleaning up downloaded tests"
+    rm -rf *.profraw
+    rm -rf *.tar.gz.1
+    rm -rf consensus-spec-tests

lightclient-circuits/Cargo.toml

@@ -60,7 +60,11 @@ rayon = "1.7.0"
 ark-std = { version = "0.4.0", features = ["print-trace"] }
 
 poseidon_native = { git = "https://github.com/axiom-crypto/halo2.git", branch = "axiom/dev", package = "poseidon" }
-# [dev-dependencies]
+[dev-dependencies]
+rstest = "0.18.2"
+test-utils = { git = "ssh://[email protected]/sygmaprotocol/Zipline.git", rev = "27e8a01" }
+ethereum-consensus-types = { git = "ssh://[email protected]/sygmaprotocol/Zipline.git", rev = "27e8a01" }
+light-client-verifier = { git = "ssh://[email protected]/sygmaprotocol/Zipline.git", rev = "27e8a01" }
 
 [features]
 default = []

lightclient-circuits/config/sync_step.json

@@ -1,31 +1,36 @@
 {
   "params": {
     "strategy": "Vertical",
-    "k": 21,
+    "k": 20,
     "num_advice_per_phase": [
-      11,
+      16,
       0,
       0
     ],
     "num_lookup_advice_per_phase": [
-      2,
+      3,
       0,
       0
     ],
     "num_fixed": 1
   },
   "break_points": [
     [
-      2097141,
-      2097141,
-      2097141,
-      2097140,
-      2097142,
-      2097140,
-      2097141,
-      2097141,
-      2097140,
-      2097141
+      1048566,
+      1048566,
+      1048565,
+      1048566,
+      1048566,
+      1048565,
+      1048564,
+      1048566,
+      1048566,
+      1048565,
+      1048566,
+      1048566,
+      1048566,
+      1048565,
+      1048564
     ],
     [],
     []

lightclient-circuits/src/committee_update_circuit.rs

@@ -104,9 +104,9 @@ impl<S: Spec, F: Field> CommitteeUpdateCircuit<S, F> {
         vec![vec![poseidon_commitment]]
     }
 
-    fn decode_pubkeys_x<'a, I: IntoIterator<Item = Vec<AssignedValue<F>>>>(
+    fn decode_pubkeys_x<I: IntoIterator<Item = Vec<AssignedValue<F>>>>(
         ctx: &mut Context<F>,
-        fp_chip: &FpChip<'a, F>,
+        fp_chip: &FpChip<'_, F>,
         compressed_encodings: I,
     ) -> Vec<ProperCrtUint<F>> {
         let range = fp_chip.range();

lightclient-circuits/src/gadget/crypto/hash2curve.rs

@@ -4,12 +4,18 @@
 use std::ops::Deref;
 use std::{cell::RefCell, iter, marker::PhantomData};
 
+use super::{
+    util::{fp2_sgn0, i2osp, strxor},
+    Fp2Point, G1Point, G2Point, HashInstructions,
+};
+use super::{AssignedHashResult, ShaContexts, ShaThreadBuilder};
 use crate::util::{AsBits, ThreadBuilderBase};
 use crate::{
     util::{bigint_to_le_bytes, decode_into_field, decode_into_field_be},
     witness::HashInput,
 };
 use eth_types::{AppCurveExt, Field, HashCurveExt, Spec};
+use ff::Field as _;
 use halo2_base::{
     safe_types::{GateInstructions, RangeInstructions, SafeBytes32, SafeTypeChip},
     utils::ScalarField,
@@ -29,12 +35,6 @@ use itertools::Itertools;
 use num_bigint::{BigInt, BigUint};
 use pasta_curves::arithmetic::SqrtRatio;
 
-use super::{
-    util::{fp2_sgn0, i2osp, strxor},
-    Fp2Point, G1Point, G2Point, HashInstructions,
-};
-use super::{AssignedHashResult, ShaContexts, ShaThreadBuilder};
-
 const G2_EXT_DEGREE: usize = 2;
 
 // L = ceil((ceil(log2(p)) + k) / 8) (see section 5 of ietf draft link above)

lightclient-circuits/src/lib.rs

@@ -17,7 +17,7 @@ pub mod aggregation;
 pub mod committee_update_circuit;
 pub mod sync_step_circuit;
 
-mod builder;
+pub mod builder;
 mod poseidon;
 mod ssz_merkle;
 

lightclient-circuits/src/sync_step_circuit.rs

@@ -64,7 +64,7 @@ use pasta_curves::group::{ff, GroupEncoding};
 use poseidon::PoseidonChip;
 use sha2::{Digest, Sha256};
 use snark_verifier_sdk::{evm::gen_evm_verifier_shplonk, CircuitExt};
-use ssz_rs::{GeneralizedIndex, Merkleized, Node};
+use ssz_rs::{Merkleized, Node};
 
 #[allow(type_alias_bounds)]
 #[derive(Clone, Debug, Default)]
@@ -95,12 +95,6 @@ impl<S: Spec, F: Field> SyncStepCircuit<S, F> {
         let bls_chip = bls_signature::BlsSignatureChip::new(&fp_chip, pairing_chip);
         let h2c_chip = HashToCurveChip::<S, F, _>::new(&sha256_chip);
 
-        let beacon_state_root = args
-            .beacon_state_root
-            .iter()
-            .map(|&b| thread_pool.main().load_witness(F::from(b as u64)))
-            .collect_vec();
-
         let execution_payload_root: HashInputChunk<QuantumCell<F>> =
             args.execution_payload_root.clone().into_witness();
 
@@ -161,14 +155,22 @@ impl<S: Spec, F: Field> SyncStepCircuit<S, F> {
             .collect_vec();
 
         let finalized_slot: HashInputChunk<_> = args.finalized_header.slot.into_witness();
+        let attested_header_state_root = args
+            .attested_header
+            .state_root
+            .as_ref()
+            .iter()
+            .map(|v| thread_pool.main().load_witness(F::from(*v as u64)))
+            .collect_vec();
+
         let finalized_header_root = ssz_merkleize_chunks(
             thread_pool,
             &sha256_chip,
             [
                 finalized_slot.clone(),
                 args.finalized_header.proposer_index.into_witness(),
                 args.finalized_header.parent_root.as_ref().into_witness(),
-                args.finalized_header.state_root.as_ref().into_witness(),
+                attested_header_state_root.clone().into(),
                 finalized_block_body_root.clone().into(),
             ],
         )?;
@@ -200,15 +202,15 @@ impl<S: Spec, F: Field> SyncStepCircuit<S, F> {
             bls_chip.verify_pairing(thread_pool.main(), signature, msghash, agg_pubkey, g1_neg);
         fp12_chip.assert_equal(thread_pool.main(), res, fp12_one);
 
-        // verify finilized block header against current beacon state merkle proof
+        // verify finalized block header against current beacon state merkle proof
         verify_merkle_proof(
             thread_pool,
             &sha256_chip,
             args.finality_branch
                 .iter()
                 .map(|w| w.clone().into_witness()),
             finalized_header_root.into(),
-            &beacon_state_root,
+            &attested_header_state_root,
             S::FINALIZED_HEADER_INDEX,
         )?;
 
@@ -317,7 +319,7 @@ impl<S: Spec, F: Field> SyncStepCircuit<S, F> {
         Ok(vec![pi_field])
     }
 
-    fn instances(args: SyncStepArgs<S>) -> Vec<Vec<bn256::Fr>> {
+    pub fn instance(args: SyncStepArgs<S>) -> bn256::Fr {
         let mut input: [u8; 64] = [0; 64];
 
         let mut attested_slot = args.attested_header.slot.to_le_bytes().to_vec();
@@ -377,8 +379,7 @@ impl<S: Spec, F: Field> SyncStepCircuit<S, F> {
         let mut public_input_commitment = sha2::Sha256::digest(input).to_vec();
         // Truncate to 253 bits
         public_input_commitment[31] &= 0b00011111;
-        let pi_field = bn256::Fr::from_bytes_le(&public_input_commitment);
-        vec![vec![pi_field]]
+        bn256::Fr::from_bytes_le(&public_input_commitment)
     }
 }
 
@@ -412,9 +413,9 @@ impl<S: Spec, F: Field> SyncStepCircuit<S, F> {
     }
 
     /// Takes a list of pubkeys and aggregates them.
-    fn aggregate_pubkeys<'a>(
+    fn aggregate_pubkeys(
         ctx: &mut Context<F>,
-        fp_chip: &FpChip<'a, F>,
+        fp_chip: &FpChip<'_, F>,
         pubkey_affines: &[G1Affine],
         pariticipation_bits: &[bool],
         assigned_affines: &mut Vec<G1Point<F>>,

lightclient-circuits/src/util/circuit.rs

@@ -9,7 +9,9 @@ use halo2_proofs::plonk::{Circuit, Error, VerifyingKey};
 use halo2_proofs::{plonk::ProvingKey, poly::kzg::commitment::ParamsKZG};
 use halo2curves::bn256::{Bn256, Fr, G1Affine};
 use serde::{Deserialize, Serialize};
-use snark_verifier_sdk::evm::{gen_evm_proof, gen_evm_proof_shplonk, gen_evm_verifier_shplonk, encode_calldata, evm_verify};
+use snark_verifier_sdk::evm::{
+    encode_calldata, evm_verify, gen_evm_proof, gen_evm_proof_shplonk, gen_evm_verifier_shplonk,
+};
 use snark_verifier_sdk::halo2::aggregation::AggregationCircuit;
 use snark_verifier_sdk::{gen_pk, halo2::gen_snark_shplonk, read_pk};
 use snark_verifier_sdk::{CircuitExt, Snark};

lightclient-circuits/src/util/common.rs

@@ -114,7 +114,7 @@ impl CellType {
     }
 
     /// Return the storage cell of the advice column
-    pub(crate) fn storage_for_column<F: Field>(col: &Column<Advice>) -> CellType {
+    pub(crate) fn storage_for_column(col: &Column<Advice>) -> CellType {
         Self::storage_for_phase(col.column_type().phase())
     }
 }

lightclient-circuits/src/witness/sync.rs

@@ -62,11 +62,11 @@ impl<S: Spec> Default for SyncStepArgs<S> {
             body_root: Node::from_bytes(beacon_block_body_root.try_into().unwrap()),
             ..Default::default()
         };
-        let finilized_header = finalized_block.hash_tree_root().unwrap().as_ref().to_vec();
+        let finalized_header = finalized_block.hash_tree_root().unwrap().as_ref().to_vec();
 
         let finality_merkle_branch = vec![vec![0; 32]; S::FINALIZED_HEADER_DEPTH];
 
-        let beacon_state_root = compute_root(finilized_header, &state_merkle_branch);
+        let beacon_state_root = compute_root(finalized_header, &state_merkle_branch);
 
         Self {
             signature_compressed: hex::decode("462c5acb68722355eaa568a166e6da4c46702a496586aa94c681e0b03a200394b8f4adc98d6b5a68e3caf9dae31ff7035a402aad93bdd4752e521b3b536b47dee55d129b6374177f2be8c99b6ea6618abae84b389affc5a50ad8d991f763beaa").unwrap(),