From 0ecd781828f61c257d8b22fc879f2a98c593d8ab Mon Sep 17 00:00:00 2001 From: Eduard S Date: Tue, 20 Jun 2023 09:50:03 +0200 Subject: [PATCH 1/3] Introduce the Root Circuit to the integration tests (#1471) ### Description Introduce two new sets of tests in the Integration tests with the root circuit: one where the root circuit is verified with the MockProver and the other where a real proof is generated. The input proof to the root circuit is the output of each subcircuit test we had before. ### Issue Link Resolve https://github.com/privacy-scaling-explorations/zkevm-circuits/issues/1334 Resolve https://github.com/privacy-scaling-explorations/zkevm-circuits/issues/1460 ### Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) ### Contents Issue https://github.com/privacy-scaling-explorations/zkevm-circuits/issues/1335 has been resolved by assigning the fixed columns in the ExpTable and CopyTable in their `load` methods. Previously they were either not assigned, or assigned dynamically. This was not affecting the SuperCircuit or the Exp/Copy Circuit because in those cases, the assignment is done with a different method. Only for SubCircuit tests where the SubCircuit requires the external table the `load` method is used. ### Rationale Testing a Root Circuit (no mater if it's with the MockProver or with a real proof) requires a real proof of the underlying circuit. This means that now for each SubCircuit we have 3 tests that require the SubCircuit proof: - normal real prover test - root mock prover test - root real prover test I implemented a cache to keep the SubCircuit proofs for each SubCircuit and block so that we have the possibility of running all the tests in a single run while reusing the SubCircuit proofs. ### How Has This Been Tested? I have only tested the SubCircuits (minus SuperCircuit) + Root Circuit via MockProver. - The machine I use has 64 GiB of memory, and that's not enough to generate a SuperCircuit proof - I have as a pending task to try the Root Circuit real prover But now we can run these tests under the github CI, so we can defer full testing after the PR is merged. --- .github/workflows/integration.yml | 30 +- Cargo.lock | 87 +++- bus-mapping/src/circuit_input_builder.rs | 2 +- integration-tests/run.sh | 2 +- .../src/integration_test_circuits.rs | 371 +++++++++++++----- integration-tests/tests/circuits.rs | 53 ++- zkevm-circuits/Cargo.toml | 2 +- .../src/evm_circuit/execution/end_block.rs | 2 +- zkevm-circuits/src/exp_circuit/test.rs | 22 +- zkevm-circuits/src/root_circuit.rs | 7 +- zkevm-circuits/src/table/copy_table.rs | 14 + zkevm-circuits/src/table/exp_table.rs | 27 +- 12 files changed, 464 insertions(+), 155 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6839d3cb76..e2183badad 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,16 +9,18 @@ on: push: branches: - main - workflow_dispatch: + workflow_dispatch: inputs: provertype: - description: 'invoke real vs mock prover' + description: 'invoke real vs mock prover (with or without root)' required: true - default: 'mock_prover' + default: 'sub_mock_prover' type: choice options: - - real_prover - - mock_prover + - sub_real_prover + - sub_mock_prover + - root_mock_prover + - root_real_prover concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -36,12 +38,18 @@ jobs: - id: set-outputs name: Select instance and prover type run: | - if [ "${{ github.event.inputs.provertype }}" = "real_prover" ] || [ "${{ github.event_name }}" = "schedule" ]; then + if [ "${{ github.event.inputs.provertype }}" = "root_real_prover" ] || [ "${{ github.event_name }}" = "schedule" ]; then echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT" - echo "provertype=real_prover" >> "$GITHUB_OUTPUT" - elif [ "${{ github.event.inputs.provertype }}" = "mock_prover" ] || [ -z ${{ github.event.inputs.provertype }} ]; then + echo "provertype=root_real_prover" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event.inputs.provertype }}" = "root_mock_prover" ]; then + echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT" + echo "provertype=root_mock_prover" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event.inputs.provertype }}" = "sub_real_prover" ]; then + echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT" + echo "provertype=sub_real_prover" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event.inputs.provertype }}" = "sub_mock_prover" ] || [ -z ${{ github.event.inputs.provertype }} ]; then echo "instancetype=c5.9xlarge" >> "$GITHUB_OUTPUT" - echo "provertype=mock_prover" >> "$GITHUB_OUTPUT" + echo "provertype=sub_mock_prover" >> "$GITHUB_OUTPUT" else exit 1 fi @@ -73,7 +81,7 @@ jobs: restore-keys: | ${{ runner.os }}-go- - name: Cargo cache - uses: actions/cache@v3 + uses: actions/cache@v3 with: path: | ~/.cargo/bin/ @@ -82,7 +90,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # Run an initial build in a separate step to split the build time from execution time + # Run an initial build in a separate step to split the build time from execution time - name: Build bins run: cargo build --bin gen_blockchain_data - name: Build tests diff --git a/Cargo.lock b/Cargo.lock index 54417fccd6..a9516c3df2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1496,6 +1496,8 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ + "byteorder", + "rand", "rustc-hex", "static_assertions", ] @@ -2508,6 +2510,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate 1.3.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "once_cell" version = "1.17.0" @@ -2914,6 +2937,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f3486ccba82358b11a77516035647c34ba167dfa53312630de83b12bd4f3d66" dependencies = [ "fixed-hash 0.8.0", + "impl-codec", + "impl-rlp", "uint", ] @@ -3168,6 +3193,23 @@ dependencies = [ "winreg", ] +[[package]] +name = "revm" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73d84c8f9836efb0f5f5f8de4700a953c4e1f3119e5cfcb0aad8e5be73daf991" +dependencies = [ + "arrayref", + "auto_impl", + "bytes", + "hashbrown 0.13.2", + "num_enum", + "primitive-types 0.12.1", + "revm_precompiles", + "rlp", + "sha3 0.10.7", +] + [[package]] name = "revm-precompile" version = "2.0.2" @@ -3179,7 +3221,7 @@ dependencies = [ "once_cell", "revm-primitives", "ripemd", - "secp256k1", + "secp256k1 0.27.0", "sha2 0.10.6", "sha3 0.10.7", "substrate-bn", @@ -3206,6 +3248,24 @@ dependencies = [ "sha3 0.10.7", ] +[[package]] +name = "revm_precompiles" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0353d456ef3e989dc9190f42c6020f09bc2025930c37895826029304413204b5" +dependencies = [ + "bytes", + "hashbrown 0.13.2", + "num", + "once_cell", + "primitive-types 0.12.1", + "ripemd", + "secp256k1 0.24.3", + "sha2 0.10.6", + "sha3 0.10.7", + "substrate-bn", +] + [[package]] name = "rfc6979" version = "0.3.1" @@ -3477,13 +3537,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secp256k1" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" +dependencies = [ + "secp256k1-sys 0.6.1", +] + [[package]] name = "secp256k1" version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" dependencies = [ - "secp256k1-sys", + "secp256k1-sys 0.8.1", +] + +[[package]] +name = "secp256k1-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" +dependencies = [ + "cc", ] [[package]] @@ -3712,6 +3790,7 @@ name = "snark-verifier" version = "0.1.0" source = "git+https://github.com/privacy-scaling-explorations/snark-verifier?tag=v2023_04_20#e5d5e4a6ccff2bba71baf77ab7a12b124d6364a1" dependencies = [ + "bytes", "ecc", "halo2_proofs", "halo2curves", @@ -3722,7 +3801,11 @@ dependencies = [ "num-integer", "num-traits", "poseidon", + "primitive-types 0.12.1", "rand", + "revm", + "rlp", + "sha3 0.10.7", ] [[package]] diff --git a/bus-mapping/src/circuit_input_builder.rs b/bus-mapping/src/circuit_input_builder.rs index 11e676d561..b1e470763d 100644 --- a/bus-mapping/src/circuit_input_builder.rs +++ b/bus-mapping/src/circuit_input_builder.rs @@ -80,7 +80,7 @@ impl Default for CircuitsParams { // TODO: Check whether this value is correct or we should increase/decrease based on // this lib tests max_copy_rows: 1000, - max_exp_steps: 1000, + max_exp_steps: 1000 / 7, // exp_circuit::OFFSET_INCREMENT = 7 max_bytecode: 512, max_evm_rows: 0, max_keccak_rows: 0, diff --git a/integration-tests/run.sh b/integration-tests/run.sh index a67bdb4902..a1f1a8dda9 100755 --- a/integration-tests/run.sh +++ b/integration-tests/run.sh @@ -3,7 +3,7 @@ set -e ARG_DEFAULT_SUDO= ARG_DEFAULT_STEPS="setup gendata tests cleanup" -ARG_DEFAULT_TESTS="rpc circuit_input_builder circuits::mock_prover" +ARG_DEFAULT_TESTS="rpc circuit_input_builder circuits::sub_mock_prover" usage() { cat >&2 << EOF diff --git a/integration-tests/src/integration_test_circuits.rs b/integration-tests/src/integration_test_circuits.rs index c9bfd37a09..c43d2026bf 100644 --- a/integration-tests/src/integration_test_circuits.rs +++ b/integration-tests/src/integration_test_circuits.rs @@ -5,11 +5,13 @@ use bus_mapping::{ }; use eth_types::geth_types::GethData; use halo2_proofs::{ + self, + circuit::Value, dev::{CellValue, MockProver}, halo2curves::bn256::{Bn256, Fr, G1Affine}, plonk::{ create_proof, keygen_pk, keygen_vk, permutation::Assembly, verify_proof, Circuit, - ProvingKey, VerifyingKey, + ProvingKey, }, poly::{ commitment::ParamsProver, @@ -19,14 +21,10 @@ use halo2_proofs::{ strategy::SingleStrategy, }, }, - transcript::{ - Blake2bRead, Blake2bWrite, Challenge255, TranscriptReadBuffer, TranscriptWriterBuffer, - }, }; use lazy_static::lazy_static; use mock::TestContext; use rand_chacha::rand_core::SeedableRng; -use rand_core::RngCore; use rand_xorshift::XorShiftRng; use std::{collections::HashMap, marker::PhantomData, sync::Mutex}; use tokio::sync::Mutex as TokioMutex; @@ -37,6 +35,9 @@ use zkevm_circuits::{ exp_circuit::TestExpCircuit, keccak_circuit::TestKeccakCircuit, pi_circuit::TestPiCircuit, + root_circuit::{ + compile, Config, EvmTranscript, NativeLoader, PoseidonTranscript, RootCircuit, Shplonk, + }, state_circuit::TestStateCircuit, super_circuit::SuperCircuit, tx_circuit::TestTxCircuit, @@ -84,6 +85,9 @@ const KECCAK_CIRCUIT_DEGREE: u32 = 16; const SUPER_CIRCUIT_DEGREE: u32 = 20; const EXP_CIRCUIT_DEGREE: u32 = 16; const PI_CIRCUIT_DEGREE: u32 = 17; +const ROOT_CIRCUIT_SMALL_DEGREE: u32 = 24; +// Big is for SuperCircuit only +const ROOT_CIRCUIT_BIG_DEGREE: u32 = 26; lazy_static! { /// Data generation. @@ -101,63 +105,177 @@ lazy_static! { lazy_static! { /// Integration test for EVM circuit pub static ref EVM_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("EVM", EVM_CIRCUIT_DEGREE)); + TokioMutex::new(IntegrationTest::new("EVM", EVM_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); /// Integration test for State circuit pub static ref STATE_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("State", STATE_CIRCUIT_DEGREE)); + TokioMutex::new(IntegrationTest::new("State", STATE_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); /// Integration test for State circuit pub static ref TX_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("Tx", TX_CIRCUIT_DEGREE)); + TokioMutex::new(IntegrationTest::new("Tx", TX_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); /// Integration test for Bytecode circuit pub static ref BYTECODE_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("Bytecode", BYTECODE_CIRCUIT_DEGREE)); + TokioMutex::new(IntegrationTest::new("Bytecode", BYTECODE_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); /// Integration test for Copy circuit pub static ref COPY_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("Copy", COPY_CIRCUIT_DEGREE)); + TokioMutex::new(IntegrationTest::new("Copy", COPY_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); /// Integration test for Keccak circuit pub static ref KECCAK_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("Keccak", KECCAK_CIRCUIT_DEGREE)); + TokioMutex::new(IntegrationTest::new("Keccak", KECCAK_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); /// Integration test for Copy circuit pub static ref SUPER_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("Super", SUPER_CIRCUIT_DEGREE)); + TokioMutex::new(IntegrationTest::new("Super", SUPER_CIRCUIT_DEGREE, ROOT_CIRCUIT_BIG_DEGREE)); + + /// Integration test for Exp circuit + pub static ref EXP_CIRCUIT_TEST: TokioMutex>> = + TokioMutex::new(IntegrationTest::new("Exp", EXP_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); + + /// Integration test for Pi circuit + pub static ref PI_CIRCUIT_TEST: TokioMutex>> = + TokioMutex::new(IntegrationTest::new("Pi", PI_CIRCUIT_DEGREE, ROOT_CIRCUIT_SMALL_DEGREE)); +} + +lazy_static! { + /// Cache of real proofs from each block to be reused with the Root circuit tests + static ref PROOF_CACHE: TokioMutex>> = TokioMutex::new(HashMap::new()); +} - /// Integration test for Exp circuit - pub static ref EXP_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("Exp", EXP_CIRCUIT_DEGREE)); +/// Generate a real proof of a Circuit with Poseidon transcript and Shplonk accumulation scheme. +/// Verify the proof and return it. The proof is suitable to be verified by the Root Circuit. +fn test_actual_circuit>( + circuit: C, + degree: u32, + instance: Vec>, + proving_key: ProvingKey, +) -> Vec { + let general_params = get_general_params(degree); + let verifier_params: ParamsVerifierKZG = general_params.verifier_params().clone(); + + let mut transcript = PoseidonTranscript::new(Vec::new()); + + // change instace to slice + let instance: Vec<&[Fr]> = instance.iter().map(|v| v.as_slice()).collect(); + + log::info!("gen circuit proof"); + create_proof::, ProverSHPLONK<'_, Bn256>, _, _, _, _>( + &general_params, + &proving_key, + &[circuit], + &[&instance], + RNG.clone(), + &mut transcript, + ) + .expect("proof generation should not fail"); + let proof = transcript.finalize(); + + log::info!("verify circuit proof"); + let verifying_key = proving_key.get_vk(); + let mut verifier_transcript = PoseidonTranscript::new(proof.as_slice()); + let strategy = SingleStrategy::new(&general_params); + + verify_proof::, VerifierSHPLONK<'_, Bn256>, _, _, _>( + &verifier_params, + verifying_key, + strategy, + &[&instance], + &mut verifier_transcript, + ) + .expect("failed to verify circuit"); + + proof +} - /// Integration test for Pi circuit - pub static ref PI_CIRCUIT_TEST: TokioMutex>> = - TokioMutex::new(IntegrationTest::new("Pi", PI_CIRCUIT_DEGREE)); +/// Generate a real proof of the RootCircuit with Keccak transcript and Shplonk accumulation +/// scheme. Verify the proof and return it. By using the Keccak transcript (via EvmTranscript) +/// the resulting proof is suitable for verification by the EVM. +/// +/// NOTE: MockProver Root Circuit with 64 GiB RAM (2023-06-12): +/// - degree=26 -> OOM +/// - degree=25 -> OK (peak ~35 GiB) +fn test_actual_root_circuit>( + circuit: C, + degree: u32, + instance: Vec>, + proving_key: ProvingKey, +) -> Vec { + let general_params = get_general_params(degree); + let verifier_params: ParamsVerifierKZG = general_params.verifier_params().clone(); + + let mut transcript = EvmTranscript::<_, NativeLoader, _, _>::new(vec![]); + + // change instace to slice + let instance: Vec<&[Fr]> = instance.iter().map(|v| v.as_slice()).collect(); + + log::info!("gen root circuit proof"); + create_proof::, ProverSHPLONK<'_, Bn256>, _, _, _, _>( + &general_params, + &proving_key, + &[circuit], + &[&instance], + RNG.clone(), + &mut transcript, + ) + .expect("proof generation should not fail"); + let proof = transcript.finalize(); + + log::info!("verify root circuit proof"); + let verifying_key = proving_key.get_vk(); + let mut verifier_transcript = EvmTranscript::<_, NativeLoader, _, _>::new(proof.as_slice()); + let strategy = SingleStrategy::new(&general_params); + + verify_proof::, VerifierSHPLONK<'_, Bn256>, _, _, _>( + &verifier_params, + verifying_key, + strategy, + &[&instance], + &mut verifier_transcript, + ) + .expect("failed to verify circuit"); + + proof } /// Generic implementation for integration tests pub struct IntegrationTest + Circuit> { name: &'static str, degree: u32, + root_degree: u32, key: Option>, + root_key: Option>, fixed: Option>>>, permutation: Option, + // The RootCircuit changes depending on the underlying circuit, so we keep a copy of its fixed + // columns and permutation here to have a unique version for each SubCircuit. + root_fixed: Option>>>, + root_permutation: Option, _marker: PhantomData, } impl + Circuit> IntegrationTest { - fn new(name: &'static str, degree: u32) -> Self { + fn new(name: &'static str, degree: u32, root_degree: u32) -> Self { Self { name, degree, + root_degree, key: None, + root_key: None, fixed: None, permutation: None, + root_fixed: None, + root_permutation: None, _marker: PhantomData, } } + fn proof_name(&self, block_tag: &str) -> String { + format!("{}_{}", self.name, block_tag) + } + fn get_key(&mut self) -> ProvingKey { match self.key.clone() { Some(key) => key, @@ -176,86 +294,41 @@ impl + Circuit> IntegrationTest { } } - fn test_actual(&self, circuit: C, instance: Vec>, proving_key: ProvingKey) { - fn test_gen_proof, R: RngCore>( - rng: R, - circuit: C, - general_params: &ParamsKZG, - proving_key: &ProvingKey, - mut transcript: Blake2bWrite, G1Affine, Challenge255>, - instances: &[&[Fr]], - ) -> Vec { - create_proof::< - KZGCommitmentScheme, - ProverSHPLONK<'_, Bn256>, - Challenge255, - R, - Blake2bWrite, G1Affine, Challenge255>, - C, - >( - general_params, - proving_key, - &[circuit], - &[instances], - rng, - &mut transcript, - ) - .expect("proof generation should not fail"); - - transcript.finalize() - } + fn get_root_key(&mut self) -> ProvingKey { + match self.root_key.clone() { + Some(key) => key, + None => { + let params = get_general_params(self.degree); + let pk = self.get_key(); - fn test_verify( - general_params: &ParamsKZG, - verifier_params: &ParamsKZG, - verifying_key: &VerifyingKey, - proof: &[u8], - instances: &[&[Fr]], - ) { - let mut verifier_transcript = Blake2bRead::<_, G1Affine, Challenge255<_>>::init(proof); - let strategy = SingleStrategy::new(general_params); - - verify_proof::< - KZGCommitmentScheme, - VerifierSHPLONK<'_, Bn256>, - Challenge255, - Blake2bRead<&[u8], G1Affine, Challenge255>, - SingleStrategy<'_, Bn256>, - >( - verifier_params, - verifying_key, - strategy, - &[instances], - &mut verifier_transcript, - ) - .expect("failed to verify circuit"); + let block = new_empty_block(); + let circuit = C::new_from_block(&block); + let instance = circuit.instance(); + + let protocol = compile( + ¶ms, + pk.get_vk(), + Config::kzg().with_num_instance( + instance.iter().map(|instance| instance.len()).collect(), + ), + ); + let circuit = RootCircuit::>::new( + ¶ms, + &protocol, + Value::unknown(), + Value::unknown(), + ) + .unwrap(); + + let general_params = get_general_params(self.root_degree); + let verifying_key = + keygen_vk(&general_params, &circuit).expect("keygen_vk should not fail"); + let key = keygen_pk(&general_params, verifying_key, &circuit) + .expect("keygen_pk should not fail"); + self.root_key = Some(key.clone()); + key + } } - - let general_params = get_general_params(self.degree); - let verifier_params: ParamsVerifierKZG = general_params.verifier_params().clone(); - - let transcript = Blake2bWrite::<_, G1Affine, Challenge255<_>>::init(vec![]); - - // change instace to slice - let instance: Vec<&[Fr]> = instance.iter().map(|v| v.as_slice()).collect(); - - let proof = test_gen_proof( - RNG.clone(), - circuit, - &general_params, - &proving_key, - transcript, - &instance, - ); - - let verifying_key = proving_key.get_vk(); - test_verify( - &general_params, - &verifier_params, - verifying_key, - &proof, - &instance, - ); } fn test_mock(&mut self, circuit: &C, instance: Vec>) { @@ -292,27 +365,117 @@ impl + Circuit> IntegrationTest { } } + fn test_root_variadic(&mut self, mock_prover: &MockProver) { + let fixed = mock_prover.fixed(); + + match self.root_fixed.clone() { + Some(prev_fixed) => { + assert!( + fixed.eq(&prev_fixed), + "root circuit fixed columns are not constant for different witnesses" + ); + } + None => { + self.root_fixed = Some(fixed.clone()); + } + }; + + let permutation = mock_prover.permutation(); + + if let Some(prev_permutation) = self.root_permutation.clone() { + assert!( + permutation.eq(&prev_permutation), + "root circuit permutations are not constant for different witnesses" + ); + } else { + self.root_permutation = Some(permutation.clone()); + } + } + /// Run integration test at a block identified by a tag. - pub async fn test_at_block_tag(&mut self, block_tag: &str, actual: bool) { + pub async fn test_at_block_tag(&mut self, block_tag: &str, root: bool, actual: bool) { let block_num = *GEN_DATA.blocks.get(block_tag).unwrap(); + let proof_name = self.proof_name(block_tag); let (builder, _) = gen_inputs(block_num).await; log::info!( - "test {} circuit, block: #{} - {}", + "test {} circuit{}, {} prover, block: #{} - {}", self.name, + if root { + " with aggregation (root circuit)" + } else { + "" + }, + if actual { "real" } else { "mock" }, block_num, - block_tag + block_tag, ); let mut block = block_convert(&builder.block, &builder.code_db).unwrap(); block.randomness = Fr::from(TEST_MOCK_RANDOMNESS); let circuit = C::new_from_block(&block); let instance = circuit.instance(); - if actual { - let key = self.get_key(); - self.test_actual(circuit, instance, key); + #[allow(clippy::collapsible_else_if)] + if root { + let params = get_general_params(self.degree); + let pk = self.get_key(); + let protocol = compile( + ¶ms, + pk.get_vk(), + Config::kzg() + .with_num_instance(instance.iter().map(|instance| instance.len()).collect()), + ); + + let proof = { + let mut proof_cache = PROOF_CACHE.lock().await; + if let Some(proof) = proof_cache.get(&proof_name) { + log::info!("using circuit cached proof"); + proof.clone() + } else { + let key = self.get_key(); + log::info!("circuit proof generation (no proof in the cache)"); + let proof = test_actual_circuit(circuit, self.degree, instance.clone(), key); + proof_cache.insert(proof_name, proof.clone()); + proof + } + }; + + log::info!("root circuit new"); + let root_circuit = RootCircuit::>::new( + ¶ms, + &protocol, + Value::known(&instance), + Value::known(&proof), + ) + .unwrap(); + + if actual { + let root_key = self.get_root_key(); + let instance = root_circuit.instance(); + log::info!("root circuit proof generation"); + test_actual_root_circuit(root_circuit, self.root_degree, instance, root_key); + } else { + log::info!("root circuit mock prover verification"); + // Mock + let mock_prover = + MockProver::::run(self.root_degree, &root_circuit, root_circuit.instance()) + .unwrap(); + self.test_root_variadic(&mock_prover); + mock_prover + .verify_par() + .expect("mock prover verification failed"); + } } else { - self.test_mock(&circuit, instance); + if actual { + let key = self.get_key(); + log::info!("circuit proof generation"); + let proof = test_actual_circuit(circuit, self.degree, instance, key); + let mut proof_cache = PROOF_CACHE.lock().await; + proof_cache.insert(proof_name, proof); + } else { + log::info!("circuit mock prover verification"); + self.test_mock(&circuit, instance); + } } } } diff --git a/integration-tests/tests/circuits.rs b/integration-tests/tests/circuits.rs index 1582bd196b..5169b4977d 100644 --- a/integration-tests/tests/circuits.rs +++ b/integration-tests/tests/circuits.rs @@ -1,58 +1,59 @@ macro_rules! run_test { - ($test_instance:expr, $block_tag:expr, $real_prover:expr) => { + ($test_instance:expr, $block_tag:expr, $root:expr, $real_prover:expr) => { log_init(); let mut test = $test_instance.lock().await; - test.test_at_block_tag($block_tag, $real_prover).await; + test.test_at_block_tag($block_tag, $root, $real_prover) + .await; }; } macro_rules! declare_tests { - (($name:ident, $block_tag:expr),$real_prover:expr) => { + (($name:ident, $block_tag:expr),$root:expr,$real_prover:expr) => { paste! { #[tokio::test] async fn []() { - run_test! (EVM_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (EVM_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (STATE_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (STATE_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (TX_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (TX_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (BYTECODE_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (BYTECODE_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (COPY_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (COPY_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (KECCAK_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (KECCAK_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (SUPER_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (SUPER_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (EXP_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (EXP_CIRCUIT_TEST, $block_tag, $root, $real_prover); } #[tokio::test] async fn []() { - run_test! (PI_CIRCUIT_TEST, $block_tag, $real_prover); + run_test! (PI_CIRCUIT_TEST, $block_tag, $root, $real_prover); } } }; @@ -73,17 +74,37 @@ macro_rules! unroll_tests { PI_CIRCUIT_TEST, }; use integration_tests::log_init; - mod real_prover { + // NOTE: The SubCircuits include all well known SubCircuits and the SuperCircuit. + + // SubCircuit tests with real prover + mod sub_real_prover { + use super::*; + $( + declare_tests! ($arg, false, true) ; + )* + } + + // SubCircuit tests with mock prover + mod sub_mock_prover { + use super::*; + $( + declare_tests! ($arg, false, false) ; + )* + } + + // Root Circuit (aggregation) tests with real prover. Needs real proof of each SubCircuit. + mod root_real_prover { use super::*; $( - declare_tests! ($arg, true) ; + declare_tests! ($arg, true, true) ; )* } - mod mock_prover { + // Root Circuit (aggregation) tests with mock prover. Needs real proof of each SubCircuit. + mod root_mock_prover { use super::*; $( - declare_tests! ($arg, false) ; + declare_tests! ($arg, true, false) ; )* } } diff --git a/zkevm-circuits/Cargo.toml b/zkevm-circuits/Cargo.toml index b20410557a..4941c9ace5 100644 --- a/zkevm-circuits/Cargo.toml +++ b/zkevm-circuits/Cargo.toml @@ -33,7 +33,7 @@ integer = { git = "https://github.com/privacy-scaling-explorations/halo2wrong" libsecp256k1 = "0.7" num-bigint = { version = "0.4" } rand_chacha = "0.3" -snark-verifier = { git = "https://github.com/privacy-scaling-explorations/snark-verifier", tag = "v2023_04_20", default-features = false, features = ["loader_halo2", "system_halo2"] } +snark-verifier = { git = "https://github.com/privacy-scaling-explorations/snark-verifier", tag = "v2023_04_20", default-features = false, features = ["loader_halo2", "system_halo2", "loader_evm"] } cli-table = { version = "0.4", optional = true } [dev-dependencies] diff --git a/zkevm-circuits/src/evm_circuit/execution/end_block.rs b/zkevm-circuits/src/evm_circuit/execution/end_block.rs index 0b3eaee2a4..4ed697017e 100644 --- a/zkevm-circuits/src/evm_circuit/execution/end_block.rs +++ b/zkevm-circuits/src/evm_circuit/execution/end_block.rs @@ -133,7 +133,7 @@ impl ExecutionGadget for EndBlockGadget { let max_txs_assigned = self.max_txs.assign(region, offset, Value::known(max_txs))?; // When rw_indices is not empty, we're at the last row (at a fixed offset), // where we need to access the max_rws and max_txs constant. - if !step.rw_indices_len() == 0 { + if step.rw_indices_len() != 0 { region.constrain_constant(max_rws_assigned, max_rws)?; region.constrain_constant(max_txs_assigned, max_txs)?; } diff --git a/zkevm-circuits/src/exp_circuit/test.rs b/zkevm-circuits/src/exp_circuit/test.rs index 2b5012458d..a4a8e0da57 100644 --- a/zkevm-circuits/src/exp_circuit/test.rs +++ b/zkevm-circuits/src/exp_circuit/test.rs @@ -54,7 +54,14 @@ fn gen_code_multiple(args: Vec<(Word, Word)>) -> Bytecode { fn gen_data(code: Bytecode) -> CircuitInputBuilder { let test_ctx = TestContext::<2, 1>::simple_ctx_with_bytecode(code).unwrap(); let block: GethData = test_ctx.into(); - let mut builder = BlockData::new_from_geth_data(block.clone()).new_circuit_input_builder(); + let mut builder = BlockData::new_from_geth_data_with_params( + block.clone(), + CircuitsParams { + max_exp_steps: 1000, + ..CircuitsParams::default() + }, + ) + .new_circuit_input_builder(); builder .handle_block(&block.eth_block, &block.geth_traces) .unwrap(); @@ -111,14 +118,19 @@ fn exp_circuit_multiple() { #[test] fn variadic_size_check() { - let k = 20; + let k = 13; // Empty let block: GethData = TestContext::<0, 0>::new(None, |_| {}, |_, _| {}, |b, _| b) .unwrap() .into(); - let mut builder = - BlockData::new_from_geth_data_with_params(block.clone(), CircuitsParams::default()) - .new_circuit_input_builder(); + let mut builder = BlockData::new_from_geth_data_with_params( + block.clone(), + CircuitsParams { + max_exp_steps: 1000, + ..CircuitsParams::default() + }, + ) + .new_circuit_input_builder(); builder .handle_block(&block.eth_block, &block.geth_traces) .unwrap(); diff --git a/zkevm-circuits/src/root_circuit.rs b/zkevm-circuits/src/root_circuit.rs index c5c38feb50..ec51a01ccd 100644 --- a/zkevm-circuits/src/root_circuit.rs +++ b/zkevm-circuits/src/root_circuit.rs @@ -9,8 +9,8 @@ use halo2_proofs::{ }; use itertools::Itertools; use maingate::MainGateInstructions; + use snark_verifier::{ - loader::native::NativeLoader, pcs::{ kzg::{KzgAccumulator, KzgAsProvingKey, KzgAsVerifyingKey, KzgDecidingKey}, AccumulationDecider, AccumulationScheme, AccumulationSchemeProver, @@ -32,7 +32,10 @@ pub use aggregation::{ aggregate, AggregationConfig, EccChip, Gwc, Halo2Loader, KzgDk, KzgSvk, PlonkSuccinctVerifier, PlonkVerifier, PoseidonTranscript, Shplonk, Snark, SnarkWitness, BITS, LIMBS, }; -pub use snark_verifier::system::halo2::{compile, Config}; +pub use snark_verifier::{ + loader::native::NativeLoader, + system::halo2::{compile, transcript::evm::EvmTranscript, Config}, +}; #[cfg(any(feature = "test", test))] pub use aggregation::TestAggregationCircuit; diff --git a/zkevm-circuits/src/table/copy_table.rs b/zkevm-circuits/src/table/copy_table.rs index a48761229d..a51d48a146 100644 --- a/zkevm-circuits/src/table/copy_table.rs +++ b/zkevm-circuits/src/table/copy_table.rs @@ -32,6 +32,8 @@ pub struct CopyTable { pub rw_counter: Column, /// Decrementing counter denoting reverse read-write counter. pub rwc_inc_left: Column, + /// Selector for the tag BinaryNumberChip + pub q_enable: Column, /// Binary chip to constrain the copy table conditionally depending on the /// current row's tag, whether it is Bytecode, Memory, TxCalldata or /// TxLog. @@ -44,6 +46,7 @@ impl CopyTable { Self { is_first: meta.advice_column(), id: meta.advice_column_in(SecondPhase), + q_enable, tag: BinaryNumberChip::configure(meta, q_enable, None), addr: meta.advice_column(), src_addr_end: meta.advice_column(), @@ -235,6 +238,17 @@ impl CopyTable { } } + // Enable selector at all rows + let max_copy_rows = block.circuits_params.max_copy_rows; + for offset in 0..max_copy_rows { + region.assign_fixed( + || "q_enable", + self.q_enable, + offset, + || Value::known(F::ONE), + )?; + } + Ok(()) }, ) diff --git a/zkevm-circuits/src/table/exp_table.rs b/zkevm-circuits/src/table/exp_table.rs index c2ee645e53..628a4c30ff 100644 --- a/zkevm-circuits/src/table/exp_table.rs +++ b/zkevm-circuits/src/table/exp_table.rs @@ -134,17 +134,6 @@ impl ExpTable { || Value::known(value), )?; } - let is_step = if offset % OFFSET_INCREMENT == 0 { - F::ONE - } else { - F::ZERO - }; - region.assign_fixed( - || format!("exponentiation table row {}", offset), - self.is_step, - offset, - || Value::known(is_step), - )?; offset += 1; } } @@ -160,6 +149,22 @@ impl ExpTable { )?; } + // Enable selector at all rows + let max_exp_steps = block.circuits_params.max_exp_steps; + for offset in 0..max_exp_steps * OFFSET_INCREMENT { + let is_step = if offset % OFFSET_INCREMENT == 0 { + F::ONE + } else { + F::ZERO + }; + region.assign_fixed( + || format!("exponentiation table row {}", offset), + self.is_step, + offset, + || Value::known(is_step), + )?; + } + Ok(()) }, ) From 1e1d104ccce7e585a81f4d8646413f3c9e5949b3 Mon Sep 17 00:00:00 2001 From: Steven Date: Tue, 20 Jun 2023 18:10:10 +0800 Subject: [PATCH 2/3] Refactor: move `Sha3CodeGen` struct and `rand_bytes` function to mock crate (#1481) ### Description 1. Move `Sha3CodeGen` and `rand_bytes` from bus-mapping to mock crate. Since they are only used for test. 2. Delete `rand` dependency for bus-mapping. ### Issue Link `cargo build` fails in `bus-mapping` folder. ### Type of change - [x] Bug fix (non-breaking change which fixes an issue) --- Cargo.lock | 1 - bus-mapping/Cargo.toml | 4 +- bus-mapping/src/evm.rs | 2 - bus-mapping/src/evm/opcodes.rs | 2 - bus-mapping/src/evm/opcodes/calldataload.rs | 20 ++-- bus-mapping/src/evm/opcodes/sha3.rs | 113 +----------------- mock/src/lib.rs | 9 +- mock/src/sha3.rs | 101 ++++++++++++++++ zkevm-circuits/src/copy_circuit/test.rs | 5 +- .../src/evm_circuit/execution/sha3.rs | 4 +- 10 files changed, 129 insertions(+), 132 deletions(-) create mode 100644 mock/src/sha3.rs diff --git a/Cargo.lock b/Cargo.lock index a9516c3df2..fa16a58db7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -410,7 +410,6 @@ dependencies = [ "log", "mock", "pretty_assertions", - "rand", "revm-precompile", "serde", "serde_json", diff --git a/bus-mapping/Cargo.toml b/bus-mapping/Cargo.toml index 3632275fa6..a05855fda0 100644 --- a/bus-mapping/Cargo.toml +++ b/bus-mapping/Cargo.toml @@ -16,7 +16,6 @@ halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.gi itertools = "0.10" lazy_static = "1.4" log = "0.4.14" -rand = { version = "0.8", optional = true } serde = {version = "1.0.130", features = ["derive"] } serde_json = "1.0.66" strum = "0.24" @@ -30,7 +29,6 @@ pretty_assertions = "1.0.0" tokio = { version = "1.13", features = ["macros"] } url = "2.2.2" mock = { path = "../mock" } -rand = "0.8" [features] -test = ["mock", "rand"] +test = ["mock"] diff --git a/bus-mapping/src/evm.rs b/bus-mapping/src/evm.rs index 39cd478c9f..2441287852 100644 --- a/bus-mapping/src/evm.rs +++ b/bus-mapping/src/evm.rs @@ -4,5 +4,3 @@ pub(crate) mod opcodes; pub use eth_types::evm_types::opcode_ids::OpcodeId; pub use opcodes::Opcode; - -pub use opcodes::Sha3CodeGen; diff --git a/bus-mapping/src/evm/opcodes.rs b/bus-mapping/src/evm/opcodes.rs index 60fd50275a..c163d4016f 100644 --- a/bus-mapping/src/evm/opcodes.rs +++ b/bus-mapping/src/evm/opcodes.rs @@ -9,8 +9,6 @@ use crate::{ use core::fmt::Debug; use eth_types::{evm_unimplemented, GethExecStep, ToAddress}; -pub use self::sha3::Sha3CodeGen; - mod address; mod balance; mod begin_end_tx; diff --git a/bus-mapping/src/evm/opcodes/calldataload.rs b/bus-mapping/src/evm/opcodes/calldataload.rs index b28ae5d07c..a6d27fb6f1 100644 --- a/bus-mapping/src/evm/opcodes/calldataload.rs +++ b/bus-mapping/src/evm/opcodes/calldataload.rs @@ -103,7 +103,12 @@ impl Opcode for Calldataload { #[cfg(test)] mod calldataload_tests { - use crate::operation::CallContextOp; + use super::*; + use crate::{ + circuit_input_builder::ExecState, + mock::BlockData, + operation::{CallContextOp, StackOp}, + }; use eth_types::{ bytecode, evm_types::{OpcodeId, StackAddress}, @@ -111,18 +116,9 @@ mod calldataload_tests { Word, }; use mock::{ - generate_mock_call_bytecode, test_ctx::helpers::account_0_code_account_1_no_code, - MockCallBytecodeParams, TestContext, + generate_mock_call_bytecode, rand_bytes, + test_ctx::helpers::account_0_code_account_1_no_code, MockCallBytecodeParams, TestContext, }; - use rand::random; - - use crate::{circuit_input_builder::ExecState, mock::BlockData, operation::StackOp}; - - use super::*; - - fn rand_bytes(size: usize) -> Vec { - (0..size).map(|_| random()).collect::>() - } fn test_internal_ok( call_data_length: usize, diff --git a/bus-mapping/src/evm/opcodes/sha3.rs b/bus-mapping/src/evm/opcodes/sha3.rs index ca2cc1a430..f27af7f48c 100644 --- a/bus-mapping/src/evm/opcodes/sha3.rs +++ b/bus-mapping/src/evm/opcodes/sha3.rs @@ -5,9 +5,8 @@ use crate::{ }, Error, }; -use eth_types::{bytecode, Bytecode, GethExecStep, Word, U256}; +use eth_types::{GethExecStep, U256}; use ethers_core::utils::keccak256; -use rand::{rngs::ThreadRng, Rng}; #[derive(Clone, Copy, Debug)] pub(crate) struct Sha3; @@ -46,7 +45,7 @@ impl Opcode for Sha3 { // keccak-256 hash of the given data in memory. let sha3 = keccak256(&memory); - debug_assert_eq!(Word::from_big_endian(&sha3), expected_sha3); + debug_assert_eq!(U256::from_big_endian(&sha3), expected_sha3); state.stack_write( &mut exec_step, geth_steps[1].stack.last_filled(), @@ -87,113 +86,13 @@ impl Opcode for Sha3 { } } -/// Generate Sha3 opcode -pub struct Sha3CodeGen { - /// The offset - pub offset: usize, - /// The size - pub size: usize, - data_len: usize, - rng: ThreadRng, -} -impl Sha3CodeGen { - /// Construct with memory less than size - pub fn mem_lt_size(offset: usize, size: usize) -> Self { - let mut rng = rand::thread_rng(); - let data_len = offset - + if size.gt(&0) { - rng.gen_range(0..size) - } else { - 0 - }; - Self { - offset, - size, - data_len, - rng, - } - } - /// Construct with memory equal to size - pub fn mem_eq_size(offset: usize, size: usize) -> Self { - let data_len = offset + size; - Self { - offset, - size, - data_len, - rng: rand::thread_rng(), - } - } - /// Construct with memory greater than size - pub fn mem_gt_size(offset: usize, size: usize) -> Self { - let mut rng = rand::thread_rng(); - let data_len = offset - + size - + if size.gt(&0) { - rng.gen_range(0..size) - } else { - 0 - }; - Self { - offset, - size, - data_len, - rng, - } - } - /// Construct with empty memory - pub fn mem_empty(offset: usize, size: usize) -> Self { - Self { - offset, - size, - data_len: 0, - rng: rand::thread_rng(), - } - } - fn rand_bytes(&mut self) -> Vec { - (0..self.data_len) - .map(|_| self.rng.gen()) - .collect::>() - } - /// Generate bytecode for SHA3 opcode after having populated sufficient - /// memory given the offset and size arguments for SHA3. - pub fn gen_sha3_code(&mut self) -> (Bytecode, Vec) { - let data = self.rand_bytes(); - let mut memory = Vec::with_capacity(self.data_len); - - // add opcodes to populate memory in the current context. - let mut code = Bytecode::default(); - for (i, mem_chunk) in data.chunks(32).enumerate() { - let mem_value = if mem_chunk.len() < 32 { - std::iter::repeat(0u8) - .take(32 - mem_chunk.len()) - .chain(mem_chunk.to_vec()) - .collect::>() - } else { - mem_chunk.to_vec() - }; - memory.extend_from_slice(&mem_value); - code.op_mstore(32 * i, Word::from_big_endian(&mem_value)); - } - // append SHA3 related opcodes at the tail end. - let code_tail = bytecode! { - PUSH32(self.size) - PUSH32(self.offset) - SHA3 - STOP - }; - code.append(&code_tail); - (code, memory) - } -} - #[cfg(test)] pub(crate) mod sha3_tests { - use super::Sha3CodeGen; - use eth_types::{evm_types::OpcodeId, geth_types::GethData, Word}; + use eth_types::{evm_types::OpcodeId, geth_types::GethData, U256}; use ethers_core::utils::keccak256; use mock::{ test_ctx::helpers::{account_0_code_account_1_no_code, tx_from_1_to_0}, - TestContext, + Sha3CodeGen, TestContext, }; use crate::{ @@ -253,11 +152,11 @@ pub(crate) mod sha3_tests { [ ( RW::READ, - &StackOp::new(call_id, 1022.into(), Word::from(offset)), + &StackOp::new(call_id, 1022.into(), U256::from(offset)), ), ( RW::READ, - &StackOp::new(call_id, 1023.into(), Word::from(size)), + &StackOp::new(call_id, 1023.into(), U256::from(size)), ), ( RW::WRITE, diff --git a/mock/src/lib.rs b/mock/src/lib.rs index f190e0612c..ebdc84f528 100644 --- a/mock/src/lib.rs +++ b/mock/src/lib.rs @@ -3,15 +3,17 @@ use eth_types::{address, bytecode, bytecode::Bytecode, word, Address, Bytes, Word}; use ethers_signers::LocalWallet; use lazy_static::lazy_static; -use rand::SeedableRng; +use rand::{random, SeedableRng}; use rand_chacha::ChaCha20Rng; mod account; mod block; +mod sha3; pub mod test_ctx; mod transaction; pub(crate) use account::MockAccount; pub(crate) use block::MockBlock; +pub use sha3::Sha3CodeGen; pub use test_ctx::TestContext; pub use transaction::{AddrOrWallet, MockTransaction, CORRECT_MOCK_TXS}; @@ -126,6 +128,11 @@ impl Default for MockCallBytecodeParams { } } +/// Generate random bytes for the specified size. +pub fn rand_bytes(size: usize) -> Vec { + (0..size).map(|_| random()).collect::>() +} + /// Generate mock EVM bytecode that performs a contract call pub fn generate_mock_call_bytecode(params: MockCallBytecodeParams) -> Bytecode { bytecode! { diff --git a/mock/src/sha3.rs b/mock/src/sha3.rs new file mode 100644 index 0000000000..ae247e5043 --- /dev/null +++ b/mock/src/sha3.rs @@ -0,0 +1,101 @@ +use eth_types::{bytecode, Bytecode, U256}; +use rand::{rngs::ThreadRng, Rng}; + +/// Generate Sha3 opcode +pub struct Sha3CodeGen { + /// The offset + pub offset: usize, + /// The size + pub size: usize, + data_len: usize, + rng: ThreadRng, +} +impl Sha3CodeGen { + /// Construct with memory less than size + pub fn mem_lt_size(offset: usize, size: usize) -> Self { + let mut rng = rand::thread_rng(); + let data_len = offset + + if size.gt(&0) { + rng.gen_range(0..size) + } else { + 0 + }; + Self { + offset, + size, + data_len, + rng, + } + } + /// Construct with memory equal to size + pub fn mem_eq_size(offset: usize, size: usize) -> Self { + let data_len = offset + size; + Self { + offset, + size, + data_len, + rng: rand::thread_rng(), + } + } + /// Construct with memory greater than size + pub fn mem_gt_size(offset: usize, size: usize) -> Self { + let mut rng = rand::thread_rng(); + let data_len = offset + + size + + if size.gt(&0) { + rng.gen_range(0..size) + } else { + 0 + }; + Self { + offset, + size, + data_len, + rng, + } + } + /// Construct with empty memory + pub fn mem_empty(offset: usize, size: usize) -> Self { + Self { + offset, + size, + data_len: 0, + rng: rand::thread_rng(), + } + } + fn rand_bytes(&mut self) -> Vec { + (0..self.data_len) + .map(|_| self.rng.gen()) + .collect::>() + } + /// Generate bytecode for SHA3 opcode after having populated sufficient + /// memory given the offset and size arguments for SHA3. + pub fn gen_sha3_code(&mut self) -> (Bytecode, Vec) { + let data = self.rand_bytes(); + let mut memory = Vec::with_capacity(self.data_len); + + // add opcodes to populate memory in the current context. + let mut code = Bytecode::default(); + for (i, mem_chunk) in data.chunks(32).enumerate() { + let mem_value = if mem_chunk.len() < 32 { + std::iter::repeat(0u8) + .take(32 - mem_chunk.len()) + .chain(mem_chunk.to_vec()) + .collect::>() + } else { + mem_chunk.to_vec() + }; + memory.extend_from_slice(&mem_value); + code.op_mstore(32 * i, U256::from_big_endian(&mem_value)); + } + // append SHA3 related opcodes at the tail end. + let code_tail = bytecode! { + PUSH32(self.size) + PUSH32(self.offset) + SHA3 + STOP + }; + code.append(&code_tail); + (code, memory) + } +} diff --git a/zkevm-circuits/src/copy_circuit/test.rs b/zkevm-circuits/src/copy_circuit/test.rs index b60b965c9a..f220d1e8f7 100644 --- a/zkevm-circuits/src/copy_circuit/test.rs +++ b/zkevm-circuits/src/copy_circuit/test.rs @@ -6,7 +6,6 @@ use crate::{ }; use bus_mapping::{ circuit_input_builder::{CircuitInputBuilder, CircuitsParams}, - evm::Sha3CodeGen, mock::BlockData, }; use eth_types::{bytecode, geth_types::GethData, ToWord, Word}; @@ -14,7 +13,9 @@ use halo2_proofs::{ dev::{MockProver, VerifyFailure}, halo2curves::bn256::Fr, }; -use mock::{test_ctx::helpers::account_0_code_account_1_no_code, TestContext, MOCK_ACCOUNTS}; +use mock::{ + test_ctx::helpers::account_0_code_account_1_no_code, Sha3CodeGen, TestContext, MOCK_ACCOUNTS, +}; #[test] fn copy_circuit_unusable_rows() { diff --git a/zkevm-circuits/src/evm_circuit/execution/sha3.rs b/zkevm-circuits/src/evm_circuit/execution/sha3.rs index 736b961dfa..9c59ff10cf 100644 --- a/zkevm-circuits/src/evm_circuit/execution/sha3.rs +++ b/zkevm-circuits/src/evm_circuit/execution/sha3.rs @@ -157,9 +157,9 @@ impl ExecutionGadget for Sha3Gadget { #[cfg(test)] mod tests { use crate::test_util::CircuitTestBuilder; - use bus_mapping::{circuit_input_builder::CircuitsParams, evm::Sha3CodeGen}; + use bus_mapping::circuit_input_builder::CircuitsParams; use eth_types::{bytecode, U256}; - use mock::TestContext; + use mock::{Sha3CodeGen, TestContext}; fn test_ok(mut gen: Sha3CodeGen) { let (code, _) = gen.gen_sha3_code(); From aa5cbb1825e0eb869776678bd70a478fe6f5e37b Mon Sep 17 00:00:00 2001 From: soham <22412996+zemse@users.noreply.github.com> Date: Tue, 20 Jun 2023 14:25:29 +0000 Subject: [PATCH 3/3] Assign correct nonce in TestContext (#1464) ### Description This PR changes the way TestContext sets tx nonces. Before nonce is set serially. Applying this PR causes the tx to use nonce from the accounts config. ### Issue Link https://github.com/privacy-scaling-explorations/zkevm-circuits/issues/1461 ### Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update ### Contents - Assigns correct nonce just after `func_tx` is called. - Adds a test case to ensure that the nonces are assigned as expected. ### Rationale The automatic assignment only works if all txs use same account and its account nonce is zero. Otherwise dev has to manually override the nonce. Inferring the nonce from account state adds convenience such that it automatically works for same accounts as well as different accounts. ### How Has This Been Tested? A test case has been added. --- .../src/circuit_input_builder/tracer_tests.rs | 34 ++++----- mock/src/test_ctx.rs | 76 +++++++++++++++---- mock/src/transaction.rs | 8 +- .../src/evm_circuit/execution/begin_tx.rs | 3 +- .../src/evm_circuit/execution/end_tx.rs | 6 +- 5 files changed, 85 insertions(+), 42 deletions(-) diff --git a/bus-mapping/src/circuit_input_builder/tracer_tests.rs b/bus-mapping/src/circuit_input_builder/tracer_tests.rs index 35366d1033..81f141ab16 100644 --- a/bus-mapping/src/circuit_input_builder/tracer_tests.rs +++ b/bus-mapping/src/circuit_input_builder/tracer_tests.rs @@ -250,7 +250,7 @@ fn tracer_err_insufficient_balance() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -405,7 +405,7 @@ fn tracer_err_address_collision() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -525,7 +525,7 @@ fn tracer_create_collision_free() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -655,7 +655,7 @@ fn tracer_err_code_store_out_of_gas() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -705,7 +705,6 @@ fn tracer_err_code_store_out_of_gas_tx_deploy() { txs[0] .from(accs[1].address) .gas(55000u64.into()) - .nonce(0) .input(code_creator.into()); }, |block, _tx| block.number(0x0264), @@ -804,7 +803,7 @@ fn tracer_err_invalid_code() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -901,7 +900,7 @@ fn tracer_err_max_code_size_exceeded() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -951,7 +950,6 @@ fn tracer_err_max_code_size_exceeded_tx_deploy() { txs[0] .from(accs[1].address) .gas(60000u64.into()) - .nonce(0) .input(code_creator.into()); }, |block, _tx| block.number(0x0264), @@ -1042,7 +1040,7 @@ fn tracer_create_stop() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1160,7 +1158,7 @@ fn tracer_err_invalid_jump() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1260,7 +1258,7 @@ fn tracer_err_execution_reverted() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1319,7 +1317,7 @@ fn tracer_stop() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1387,7 +1385,7 @@ fn tracer_err_return_data_out_of_bounds() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1540,7 +1538,7 @@ fn tracer_err_write_protection(is_call: bool) { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1742,7 +1740,7 @@ fn create2_address() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1840,7 +1838,7 @@ fn create_address() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -1924,7 +1922,7 @@ fn test_gen_access_trace() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), @@ -2146,7 +2144,7 @@ fn test_gen_access_trace_create_push_call_stack() { }, |mut txs, accs| { txs[0].to(accs[0].address).from(accs[2].address); - txs[1].to(accs[1].address).from(accs[2].address).nonce(1); + txs[1].to(accs[1].address).from(accs[2].address); }, |block, _tx| block.number(0xcafeu64), LoggerConfig::enable_memory(), diff --git a/mock/src/test_ctx.rs b/mock/src/test_ctx.rs index 11b8e0c5d2..b6cd453aca 100644 --- a/mock/src/test_ctx.rs +++ b/mock/src/test_ctx.rs @@ -65,8 +65,7 @@ pub use external_tracer::LoggerConfig; /// txs[0].to(accs[0].address).from(accs[2].address); /// txs[1] /// .to(accs[1].address) -/// .from(accs[2].address) -/// .nonce(1); +/// .from(accs[2].address); /// }, /// |block, _tx| block.number(0xcafeu64), /// ) @@ -132,22 +131,26 @@ impl TestContext { .expect("Mismatched acc len"); let mut transactions = vec![MockTransaction::default(); NTX]; - // By default, set the TxIndex and the Nonce values of the multiple transactions - // of the context correlative so that any Ok test passes by default. - // If the user decides to override these values, they'll then be set to whatever - // inputs were provided by the user. - transactions - .iter_mut() - .enumerate() - .skip(1) - .for_each(|(idx, tx)| { - let idx = u64::try_from(idx).expect("Unexpected idx conversion error"); - tx.transaction_idx(idx).nonce(idx); - }); let tx_refs = transactions.iter_mut().collect(); // Build Tx modifiers. func_tx(tx_refs, accounts.clone()); + + // Sets the transaction_idx and nonce after building the tx modifiers. Hence, if user has + // overridden these values above using the tx modifiers, that will be ignored. + let mut acc_tx_count = vec![0u64; NACC]; + transactions.iter_mut().enumerate().for_each(|(idx, tx)| { + let idx = u64::try_from(idx).expect("Unexpected idx conversion error"); + tx.transaction_idx(idx); + if let Some((pos, from_acc)) = accounts + .iter() + .find_position(|acc| acc.address == tx.from.address()) + { + tx.nonce(from_acc.nonce + acc_tx_count[pos]); + acc_tx_count[pos] += 1; + } + }); + let transactions: Vec = transactions.iter_mut().map(|tx| tx.build()).collect(); @@ -279,3 +282,48 @@ pub mod helpers { txs[0].from(accs[1].address).to(accs[0].address); } } + +#[cfg(test)] +mod tests { + use eth_types::{address, U256, U64}; + + use super::{eth, TestContext}; + + #[test] + fn test_nonce() { + let block = TestContext::<2, 5>::new( + None, + |accs| { + accs[0] + .address(address!("0x0000000000000000000000000000000000000000")) + .balance(eth(10)); + accs[1] + .address(address!("0x000000000000000000000000000000000cafe001")) + .balance(eth(10)) + .nonce(100); + }, + |mut txs, accs| { + txs[0].from(accs[0].address); + txs[1].from(accs[0].address); + txs[2].from(accs[1].address); + txs[3].from(accs[1].address); + txs[4].from(accs[1].address).nonce(12345); // set nonce here is ignored + }, + |block, _tx| block.number(0xcafeu64), + ) + .unwrap(); + + // account 0 starts with nonce default 0 + assert_eq!(block.eth_block.transactions[0].nonce, U256::from(0)); + assert_eq!(block.eth_block.transactions[1].nonce, U256::from(1)); + + // account 1 starts with nonce specified 100 + assert_eq!(block.eth_block.transactions[2].nonce, U256::from(100)); + assert_eq!(block.eth_block.transactions[3].nonce, U256::from(101)); + assert_eq!(block.eth_block.transactions[4].nonce, U256::from(102)); // 12345 is ignored + + // nonce in accounts is the nonce before the block processing + assert_eq!(block.accounts[0].nonce, U64::from(0)); + assert_eq!(block.accounts[1].nonce, U64::from(100)); + } +} diff --git a/mock/src/transaction.rs b/mock/src/transaction.rs index 1b6c6ccd1f..fe4b9ad44d 100644 --- a/mock/src/transaction.rs +++ b/mock/src/transaction.rs @@ -206,8 +206,8 @@ impl MockTransaction { self } - /// Set nonce field for the MockTransaction. - pub fn nonce(&mut self, nonce: u64) -> &mut Self { + /// Set nonce field for the MockTransaction. Overridden in TestContext. + pub(crate) fn nonce(&mut self, nonce: u64) -> &mut Self { self.nonce = nonce; self } @@ -224,8 +224,8 @@ impl MockTransaction { self } - /// Set transaction_idx field for the MockTransaction. - pub fn transaction_idx(&mut self, transaction_idx: u64) -> &mut Self { + /// Set transaction_idx field for the MockTransaction. Overridden in TestContext. + pub(crate) fn transaction_idx(&mut self, transaction_idx: u64) -> &mut Self { self.transaction_index = U64::from(transaction_idx); self } diff --git a/zkevm-circuits/src/evm_circuit/execution/begin_tx.rs b/zkevm-circuits/src/evm_circuit/execution/begin_tx.rs index 90db035125..e49e77cd89 100644 --- a/zkevm-circuits/src/evm_circuit/execution/begin_tx.rs +++ b/zkevm-circuits/src/evm_circuit/execution/begin_tx.rs @@ -740,7 +740,7 @@ mod test { accs[1].address(from).balance(eth(1)).nonce(multibyte_nonce); }, |mut txs, _| { - txs[0].to(to).from(from).nonce(multibyte_nonce); + txs[0].to(to).from(from); }, |block, _| block, ) @@ -841,7 +841,6 @@ mod test { |mut txs, _accs| { txs[0] .from(MOCK_ACCOUNTS[0]) - .nonce(nonce) .gas_price(gwei(2)) .gas(Word::from(0x10000)) .value(eth(2)) diff --git a/zkevm-circuits/src/evm_circuit/execution/end_tx.rs b/zkevm-circuits/src/evm_circuit/execution/end_tx.rs index f9bcdc6380..89fe3032f7 100644 --- a/zkevm-circuits/src/evm_circuit/execution/end_tx.rs +++ b/zkevm-circuits/src/evm_circuit/execution/end_tx.rs @@ -385,8 +385,7 @@ mod test { .to(accs[0].address) .from(accs[1].address) .gas(Word::from(30_000)) - .gas_price(gwei(2)) - .nonce(0); + .gas_price(gwei(2)); // txs[1] // .to(accs[2].address) // .from(accs[3].address) @@ -457,8 +456,7 @@ mod test { .to(accs[0].address) .from(accs[1].address) .gas(Word::from(50_000)) - .gas_price(gwei(2)) - .nonce(0); + .gas_price(gwei(2)); }, |block, _tx| block, )