Merge pull request #36 from hyperledger-labs/circuit-gen

Circuit compile and proving key gen enhancements

.github/workflows/e2e.yaml

@@ -52,6 +52,7 @@ jobs:
 
       - name: Build circuits
         env:
+          CIRCUITS_ROOT: ${{ runner.temp }}/zeto-artifacts
           PROVING_KEYS_ROOT: ${{ runner.temp }}/zeto-artifacts
           PTAU_DOWNLOAD_PATH: ${{ runner.temp }}/zeto-artifacts
         working-directory: zkp/circuits
@@ -62,15 +63,15 @@ jobs:
       - name: Run golang e2e tests
         env:
           PROVING_KEYS_ROOT: ${{ runner.temp }}/zeto-artifacts
-          CIRCUITS_ROOT: ${{ github.workspace }}/zkp/js/lib
+          CIRCUITS_ROOT: ${{ runner.temp }}/zeto-artifacts
         working-directory: go-sdk
         run: |
           make e2e
 
       - name: Run js e2e tests
         env:
           PROVING_KEYS_ROOT: ${{ runner.temp }}/zeto-artifacts
-          CIRCUITS_ROOT: ${{ github.workspace }}/zkp/js/lib
+          CIRCUITS_ROOT: ${{ runner.temp }}/zeto-artifacts
         working-directory: zkp/js
         run: |
           npm install
@@ -79,7 +80,7 @@ jobs:
       - name: Run Zeto Tokens hardhat tests
         env:
           PROVING_KEYS_ROOT: ${{ runner.temp }}/zeto-artifacts
-          CIRCUITS_ROOT: ${{ github.workspace }}/zkp/js/lib
+          CIRCUITS_ROOT: ${{ runner.temp }}/zeto-artifacts
         working-directory: solidity
         run: |
           npm install

go-sdk/Makefile

@@ -11,7 +11,7 @@ GOGC=30
 
 all: test go-mod-tidy
 test: deps lint
-		$(VGO) test ./internal/... ./pkg/... -cover -coverprofile=coverage.txt -covermode=atomic -timeout=30s ${TEST_ARGS}
+		$(VGO) test ./internal/... -cover -coverprofile=coverage.txt -covermode=atomic -timeout=30s ${TEST_ARGS}
 coverage.html:
 		$(VGO) tool cover -html=coverage.txt
 coverage: test coverage.html

go.mod

@@ -1,5 +1,3 @@
 module github.com/hyperledger-labs/zeto
 
 go 1.22.5
-
-replace github.com/hyperledger-labs/zeto/go-sdk v0.0.0 => ./zkp/golang

solidity/test/zeto_anon_enc_nullifier.ts

@@ -280,10 +280,6 @@ describe("Zeto based fungible token with anonymity using nullifiers and encrypti
     utxo7 = newUTXO(15, Bob);
 
     await expect(doTransfer(Alice, [nonExisting1, nonExisting2], [nullifier1, nullifier2], [utxo7, _utxo1], root.bigInt(), merkleProofs, [Bob, Charlie])).rejectedWith("UTXORootNotFound");
-
-    // clean up the fake UTXOs from the local SMT
-    await smtAlice.delete(nonExisting1.hash);
-    await smtAlice.delete(nonExisting2.hash);
   }).timeout(600000);
 
   async function doTransfer(signer: User, inputs: UTXO[], _nullifiers: UTXO[], outputs: UTXO[], root: BigInt, merkleProofs: BigInt[][], owners: User[]) {

solidity/test/zeto_anon_nullifier.ts

@@ -281,10 +281,6 @@ describe("Zeto based fungible token with anonymity using nullifiers without encr
     utxo7 = newUTXO(15, Bob);
 
     await expect(doTransfer(Alice, [nonExisting1, nonExisting2], [nullifier1, nullifier2], [utxo7, _utxo1], root.bigInt(), merkleProofs, [Bob, Charlie])).rejectedWith("UTXORootNotFound");
-
-    // clean up the fake UTXOs from the local SMT
-    await smtAlice.delete(nonExisting1.hash);
-    await smtAlice.delete(nonExisting2.hash);
   }).timeout(600000);
 
   async function doTransfer(signer: User, inputs: UTXO[], _nullifiers: UTXO[], outputs: UTXO[], root: BigInt, merkleProofs: BigInt[][], owners: User[]) {

solidity/test/zeto_anon_nullifier_kyc.ts

@@ -327,46 +327,6 @@ describe("Zeto based fungible token with anonymity, KYC, using nullifiers withou
 
     await expect(doTransfer(Bob, [utxo7, utxo7], [nullifier1, nullifier2], [_utxo1, _utxo2], root.bigInt(), merkleProofs, identitiesRoot.bigInt(), identitiesMerkleProofs, [Alice, Bob])).rejectedWith(`UTXODuplicate`);
   }).timeout(600000);
-
-  it("transfer non-existing UTXOs should fail", async function () {
-    const nonExisting1 = newUTXO(25, Alice);
-    const nonExisting2 = newUTXO(20, Alice, nonExisting1.salt);
-
-    // add to our local SMT (but they don't exist on the chain)
-    await smtAlice.add(nonExisting1.hash, nonExisting1.hash);
-    await smtAlice.add(nonExisting2.hash, nonExisting2.hash);
-
-    // generate the nullifiers for the UTXOs to be spent
-    const nullifier1 = newNullifier(nonExisting1, Alice);
-    const nullifier2 = newNullifier(nonExisting2, Alice);
-
-    // generate inclusion proofs for the UTXOs to be spent
-    let root = await smtAlice.root();
-    const proof1 = await smtAlice.generateCircomVerifierProof(nonExisting1.hash, root);
-    const proof2 = await smtAlice.generateCircomVerifierProof(nonExisting2.hash, root);
-    const merkleProofs = [proof1.siblings.map((s) => s.bigInt()), proof2.siblings.map((s) => s.bigInt())];
-
-    // propose the output UTXOs
-    const _utxo1 = newUTXO(30, Charlie);
-    utxo7 = newUTXO(15, Bob);
-
-    const identitiesRoot = await smtKyc.root();
-    const proof3 = await smtKyc.generateCircomVerifierProof(kycHash(Alice.babyJubPublicKey), identitiesRoot);
-    const proof4 = await smtKyc.generateCircomVerifierProof(kycHash(Bob.babyJubPublicKey), identitiesRoot);
-    const proof5 = await smtKyc.generateCircomVerifierProof(kycHash(Charlie.babyJubPublicKey), identitiesRoot);
-    const identitiesMerkleProofs = [
-      proof3.siblings.map((s) => s.bigInt()), // identity proof for the sender (Alice)
-      proof4.siblings.map((s) => s.bigInt()), // identity proof for the 1st owner of the output UTXO (Bob)
-      proof5.siblings.map((s) => s.bigInt())  // identity proof for the 2nd owner of the output UTXO (Charlie)
-    ];
-
-    await expect(doTransfer(Alice, [nonExisting1, nonExisting2], [nullifier1, nullifier2], [utxo7, _utxo1], root.bigInt(), merkleProofs, identitiesRoot.bigInt(), identitiesMerkleProofs, [Bob, Charlie])).rejectedWith("UTXORootNotFound");
-
-    // clean up the fake UTXOs from the local SMT
-    await smtAlice.delete(nonExisting1.hash);
-    await smtAlice.delete(nonExisting2.hash);
-  }).timeout(600000);
-
   it("transfer from an unregistered user should fail", async function () {
     const tx = await erc20.connect(deployer).mint(unregistered.ethAddress, 100);
     await tx.wait();
@@ -422,6 +382,41 @@ describe("Zeto based fungible token with anonymity, KYC, using nullifiers withou
     expect(balance).to.equal(100);
   });
 
+  it("transfer non-existing UTXOs should fail", async function () {
+    const nonExisting1 = newUTXO(25, Alice);
+    const nonExisting2 = newUTXO(20, Alice, nonExisting1.salt);
+
+    // add to our local SMT (but they don't exist on the chain)
+    await smtAlice.add(nonExisting1.hash, nonExisting1.hash);
+    await smtAlice.add(nonExisting2.hash, nonExisting2.hash);
+
+    // generate the nullifiers for the UTXOs to be spent
+    const nullifier1 = newNullifier(nonExisting1, Alice);
+    const nullifier2 = newNullifier(nonExisting2, Alice);
+
+    // generate inclusion proofs for the UTXOs to be spent
+    let root = await smtAlice.root();
+    const proof1 = await smtAlice.generateCircomVerifierProof(nonExisting1.hash, root);
+    const proof2 = await smtAlice.generateCircomVerifierProof(nonExisting2.hash, root);
+    const merkleProofs = [proof1.siblings.map((s) => s.bigInt()), proof2.siblings.map((s) => s.bigInt())];
+
+    // propose the output UTXOs
+    const _utxo1 = newUTXO(30, Charlie);
+    utxo7 = newUTXO(15, Bob);
+
+    const identitiesRoot = await smtKyc.root();
+    const proof3 = await smtKyc.generateCircomVerifierProof(kycHash(Alice.babyJubPublicKey), identitiesRoot);
+    const proof4 = await smtKyc.generateCircomVerifierProof(kycHash(Bob.babyJubPublicKey), identitiesRoot);
+    const proof5 = await smtKyc.generateCircomVerifierProof(kycHash(Charlie.babyJubPublicKey), identitiesRoot);
+    const identitiesMerkleProofs = [
+      proof3.siblings.map((s) => s.bigInt()), // identity proof for the sender (Alice)
+      proof4.siblings.map((s) => s.bigInt()), // identity proof for the 1st owner of the output UTXO (Bob)
+      proof5.siblings.map((s) => s.bigInt())  // identity proof for the 2nd owner of the output UTXO (Charlie)
+    ];
+
+    await expect(doTransfer(Alice, [nonExisting1, nonExisting2], [nullifier1, nullifier2], [utxo7, _utxo1], root.bigInt(), merkleProofs, identitiesRoot.bigInt(), identitiesMerkleProofs, [Bob, Charlie])).rejectedWith("UTXORootNotFound");
+  }).timeout(600000);
+
   async function doTransfer(signer: User, inputs: UTXO[], _nullifiers: UTXO[], outputs: UTXO[], utxosRoot: BigInt, utxosMerkleProofs: BigInt[][], identitiesRoot: BigInt, identitiesMerkleProof: BigInt[][], owners: User[]) {
     let nullifiers: [BigNumberish, BigNumberish];
     let outputCommitments: [BigNumberish, BigNumberish];

solidity/test/zeto_nf_anon_nullifier.ts

@@ -182,9 +182,6 @@ describe("Zeto based non-fungible token with anonymity using nullifiers without
     const _utxo1 = newAssetUTXO(nonExisting1.tokenId!, nonExisting1.uri!, Charlie);
 
     await expect(doTransfer(Alice, nonExisting1, nullifier1, _utxo1, root.bigInt(), merkleProof, Charlie)).rejectedWith("UTXORootNotFound");
-
-    // clean up the fake UTXOs from the local SMT
-    await smtAlice.delete(nonExisting1.hash);
   }).timeout(600000);
 
   async function doTransfer(signer: User, input: UTXO, _nullifier: UTXO, output: UTXO, root: BigInt, merkleProof: BigInt[], owner: User) {

zkp/circuits/gen-config.json

@@ -39,10 +39,6 @@
     "ptau": "powersOfTau28_hez_final_16",
     "skipSolidityGenaration": false
   },
-  "check_smt_proof": {
-    "ptau": "powersOfTau28_hez_final_16",
-    "skipSolidityGenaration": false
-  },
   "check_nullifiers": {
     "ptau": "powersOfTau28_hez_final_11",
     "skipSolidityGenaration": true

zkp/circuits/gen.js

@@ -3,24 +3,50 @@ const path = require('path');
 const { exec } = require('child_process');
 const { promisify } = require('util');
 const axios = require('axios');
-
-const provingKeysRoot = process.env.PROVING_KEYS_ROOT;
-const ptauDownload = process.env.PTAU_DOWNLOAD_PATH;
-const specificCircuit = process.argv[2];
+const yargs = require('yargs/yargs');
+const { hideBin } = require('yargs/helpers');
+const argv = yargs(hideBin(process.argv)).argv;
+
+const circuitsRoot = process.env.CIRCUITS_ROOT || argv.circuitsRoot;
+const provingKeysRoot = process.env.PROVING_KEYS_ROOT || argv.provingKeysRoot;
+const ptauDownload = process.env.PTAU_DOWNLOAD_PATH || argv.ptauDownloadPath;
+const specificCircuits = argv.c;
+const compileOnly = argv.compileOnly;
 const parallelLimit = parseInt(process.env.GEN_CONCURRENCY, 10) || 10; // Default to compile 10 circuits in parallel
 
 // check env vars
+if (!circuitsRoot) {
+  console.error('Error: CIRCUITS_ROOT is not set.');
+  process.exit(1);
+}
 
-if (!provingKeysRoot) {
+if (!compileOnly && !provingKeysRoot) {
   console.error('Error: PROVING_KEYS_ROOT is not set.');
   process.exit(1);
 }
 
-if (!ptauDownload) {
+if (!compileOnly && !ptauDownload) {
   console.error('Error: PTAU_DOWNLOAD_PATH is not set.');
   process.exit(1);
 }
 
+console.log(
+  'Generating circuits with the following settings:\n' +
+    JSON.stringify(
+      {
+        specificCircuits,
+        compileOnly,
+        parallelLimit,
+        circuitsRoot,
+        provingKeysRoot,
+        ptauDownload,
+      },
+      null,
+      2
+    ) +
+    '\n'
+);
+
 // load circuits
 
 const circuits = require('./gen-config.json');
@@ -54,7 +80,7 @@ const processCircuit = async (circuit, ptau, skipSolidityGenaration) => {
     return;
   }
 
-  if (!fs.existsSync(ptauFile)) {
+  if (!compileOnly && !fs.existsSync(ptauFile)) {
     log(circuit, `PTAU file does not exist, downloading: ${ptauFile}`);
     try {
       const response = await axios.get(
@@ -75,7 +101,13 @@ const processCircuit = async (circuit, ptau, skipSolidityGenaration) => {
   }
 
   log(circuit, `Compiling circuit`);
-  await execAsync(`circom ${circomInput} --output ../js/lib --sym --wasm`);
+  await execAsync(
+    `circom ${circomInput} --output ${circuitsRoot} --sym --wasm`
+  );
+  if (compileOnly) {
+    return;
+  }
+
   await execAsync(`circom ${circomInput} --output ${provingKeysRoot} --r1cs`);
 
   log(circuit, `Generating test proving key with ${ptau}`);
@@ -126,19 +158,26 @@ const processCircuit = async (circuit, ptau, skipSolidityGenaration) => {
 };
 
 const run = async () => {
-  if (specificCircuit) {
-    // if a specific circuit is provided, check it's in the map
-    if (!circuits[specificCircuit]) {
-      console.error(`Error: Unknown circuit: ${specificCircuit}`);
-      process.exit(1);
+  let onlyCircuits = specificCircuits;
+  if (specificCircuits) {
+    if (!Array.isArray(specificCircuits)) {
+      onlyCircuits = [specificCircuits];
+    }
+
+    // if specific circuits are provided, check it's in the map
+    for (const circuit of onlyCircuits) {
+      if (!circuits[circuit]) {
+        console.error(`Error: Unknown circuit: ${circuit}`);
+        process.exit(1);
+      }
     }
   }
 
   const circuitsArray = Object.entries(circuits);
   const activePromises = new Set();
 
   for (const [circuit, { ptau, skipSolidityGenaration }] of circuitsArray) {
-    if (specificCircuit && circuit !== specificCircuit) {
+    if (onlyCircuits && !onlyCircuits.includes(circuit)) {
       continue;
     }
 

zkp/circuits/package.json

@@ -11,6 +11,7 @@
   },
   "devDependencies": {
     "axios": "^1.7.3",
-    "p-limit": "^6.1.0"
+    "p-limit": "^6.1.0",
+    "yargs": "^17.7.2"
   }
 }

zkp/js/README.md

@@ -29,15 +29,16 @@ npm i
 
 - set where you want to store the generated verification keys and the downloaded PTAU files
   ```console
+  export CIRCUITS_ROOT="$HOME/circuits"
   export PROVING_KEYS_ROOT="$HOME/proving-keys"
   export PTAU_DOWNLOAD_PATH="$HOME/Downloads"
-  mkdir -p $PROVING_KEYS_ROOT $PTAU_DOWNLOAD_PATH
+  mkdir -p $PROVING_KEYS_ROOT $PTAU_DOWNLOAD_PATH $CIRCUITS_ROOT
   ```
 - run the generation script for **ALL** circuits
   ```console
   npm run gen
   ```
-  **run `npm run gen $circuit` for developing a single circuit**
+  **run `npm run gen -- -c $circuit` for developing a single circuit**
   **use `GEN_CONCURRENCY` to control how many circuits to be processed in parallel, default to 10**
 
 > Refer to [generation script explanation](#generation-script-explanation) for what the script does

zkp/js/index.js

@@ -23,8 +23,12 @@ function loadCircuit(type) {
   if (!type) {
     throw new Error('The circuit name must be provided');
   }
-  const WitnessCalculator = require(`./lib/${type}_js/witness_calculator.js`);
-  const buffer = readFileSync(path.join(__dirname, `./lib/${type}_js/${type}.wasm`));
+  const circuitsRoot = process.env.CIRCUITS_ROOT;
+  if (!circuitsRoot) {
+    throw new Error('CIRCUITS_ROOT is not set');
+  }
+  const WitnessCalculator = require(path.join(circuitsRoot, `${type}_js/witness_calculator.js`));
+  const buffer = readFileSync(path.join(circuitsRoot, `${type}_js/${type}.wasm`));
   return WitnessCalculator(buffer);
 }