feat: build merkle root, merkle tree and merkle proof with Rust

merkly/accelerator/build.sh

@@ -0,0 +1 @@
+rustc --crate-type cdylib libmerkle_root.rs -o libmerkle_root.dylib

merkly/accelerator/libmerkle_root.rs

@@ -0,0 +1,48 @@
+use std::slice;
+
+/// # Safety
+///
+#[no_mangle]
+pub unsafe extern "C" fn free_32(ptr: *mut u8) {
+    unsafe {
+        let _ = Box::from_raw(slice::from_raw_parts_mut(ptr, 32));
+    }
+}
+
+/// # Safety
+///
+#[no_mangle]
+pub unsafe extern "C" fn make_root(
+    callback: extern "C" fn(input: *const u8, output: *mut u8),
+    leafs_ptr: *const *const u8,
+    len: usize,
+) -> *mut u8 {
+    let leafs = unsafe { slice::from_raw_parts(leafs_ptr, len) }
+        .iter()
+        .map(|&ptr| unsafe { Vec::from(slice::from_raw_parts(ptr, 1)) })
+        .collect::<Vec<Vec<u8>>>();
+
+
+    let mut nodes = leafs.clone();
+    while nodes.len() > 1 {
+        nodes = nodes
+            .chunks(2)
+            .map(|chunk| {
+                let result = if chunk.len() == 2 {
+                    let concat = [chunk[0].as_slice(), chunk[1].as_slice()].concat();
+                    let mut buffer: [u8; 32] = [0; 32];
+                    callback(concat.as_ptr(), buffer.as_mut_ptr());
+                    buffer.to_vec()
+                } else {
+                    chunk[0].to_vec()
+                };
+                result
+            })
+            .collect()
+    }
+
+    let root = nodes.into_iter().next().unwrap_or_default(); 
+    let boxed_array = root.into_boxed_slice();
+    Box::into_raw(boxed_array) as *mut u8
+}
+

merkly/accelerator/libmerkle_root/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "libmerkle_root"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+[lib]
+type = "cdylib"

merkly/accelerator/libmerkle_root/src/lib.rs

@@ -0,0 +1,45 @@
+use std::slice;
+
+/// # Safety
+///
+#[no_mangle]
+pub unsafe extern "C" fn free_32(ptr: *mut u8) {
+    unsafe {
+        let _ = Box::from_raw(slice::from_raw_parts_mut(ptr, 32));
+    }
+}
+
+/// # Safety
+///
+#[no_mangle]
+pub unsafe extern "C" fn make_root(
+    callback: extern "C" fn(input: *const u8, output: *mut u8),
+    leafs_ptr: *const *const u8,
+    len: usize,
+) -> *mut u8 {
+    let leafs = unsafe { slice::from_raw_parts(leafs_ptr, len) }
+        .iter()
+        .map(|&ptr| unsafe { Vec::from(slice::from_raw_parts(ptr, 32)) })
+        .collect::<Vec<Vec<u8>>>();
+
+    let mut nodes = leafs.clone();
+    while nodes.len() > 1 {
+        nodes = nodes
+            .chunks(2)
+            .map(|chunk| {
+                if chunk.len() == 2 {
+                    let concat = [chunk[0].as_slice(), chunk[1].as_slice()].concat();
+                    let mut buffer: [u8; 32] = [0; 32];
+                    callback(concat.as_ptr(), buffer.as_mut_ptr());
+                    return buffer.to_vec();
+                } else {
+                    return chunk[0].to_vec();
+                };
+            })
+            .collect()
+    }
+
+    let root = nodes.into_iter().next().unwrap_or_default(); 
+    let boxed_array = root.into_boxed_slice();
+    Box::into_raw(boxed_array) as *mut u8
+}

merkly/accelerator/libmerkle_root/target/.rustc_info.json

@@ -0,0 +1 @@
+{"rustc_fingerprint":2383946568438019878,"outputs":{"15481046163696847946":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n/Users/olivmath/.rustup/toolchains/stable-aarch64-apple-darwin\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_arch=\"aarch64\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"unix\"\ntarget_feature=\"aes\"\ntarget_feature=\"crc\"\ntarget_feature=\"dit\"\ntarget_feature=\"dotprod\"\ntarget_feature=\"dpb\"\ntarget_feature=\"dpb2\"\ntarget_feature=\"fcma\"\ntarget_feature=\"fhm\"\ntarget_feature=\"flagm\"\ntarget_feature=\"fp16\"\ntarget_feature=\"frintts\"\ntarget_feature=\"jsconv\"\ntarget_feature=\"lor\"\ntarget_feature=\"lse\"\ntarget_feature=\"neon\"\ntarget_feature=\"paca\"\ntarget_feature=\"pacg\"\ntarget_feature=\"pan\"\ntarget_feature=\"pmuv3\"\ntarget_feature=\"ras\"\ntarget_feature=\"rcpc\"\ntarget_feature=\"rcpc2\"\ntarget_feature=\"rdm\"\ntarget_feature=\"sb\"\ntarget_feature=\"sha2\"\ntarget_feature=\"sha3\"\ntarget_feature=\"ssbs\"\ntarget_feature=\"vh\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"macos\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"apple\"\nunix\n","stderr":""},"4614504638168534921":{"success":true,"status":"","code":0,"stdout":"rustc 1.72.1 (d5c2e9c34 2023-09-13)\nbinary: rustc\ncommit-hash: d5c2e9c342b358556da91d61ed4133f6f50fc0c3\ncommit-date: 2023-09-13\nhost: aarch64-apple-darwin\nrelease: 1.72.1\nLLVM version: 16.0.5\n","stderr":""},"15729799797837862367":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n/Users/olivmath/.rustup/toolchains/stable-aarch64-apple-darwin\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_arch=\"aarch64\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"unix\"\ntarget_feature=\"aes\"\ntarget_feature=\"crc\"\ntarget_feature=\"dit\"\ntarget_feature=\"dotprod\"\ntarget_feature=\"dpb\"\ntarget_feature=\"dpb2\"\ntarget_feature=\"fcma\"\ntarget_feature=\"fhm\"\ntarget_feature=\"flagm\"\ntarget_feature=\"fp16\"\ntarget_feature=\"frintts\"\ntarget_feature=\"jsconv\"\ntarget_feature=\"lor\"\ntarget_feature=\"lse\"\ntarget_feature=\"neon\"\ntarget_feature=\"paca\"\ntarget_feature=\"pacg\"\ntarget_feature=\"pan\"\ntarget_feature=\"pmuv3\"\ntarget_feature=\"ras\"\ntarget_feature=\"rcpc\"\ntarget_feature=\"rcpc2\"\ntarget_feature=\"rdm\"\ntarget_feature=\"sb\"\ntarget_feature=\"sha2\"\ntarget_feature=\"sha3\"\ntarget_feature=\"ssbs\"\ntarget_feature=\"vh\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"macos\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"apple\"\nunix\n","stderr":""}},"successes":{}}

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-609819427e5e50ec/invoked.timestamp

@@ -0,0 +1 @@
+This file has an mtime of when this was started.

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-609819427e5e50ec/lib-libmerkle_root

@@ -0,0 +1 @@
+3e9ee6cfee97481a

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-609819427e5e50ec/lib-libmerkle_root.json

@@ -0,0 +1 @@
+{"rustc":15699327122048238920,"features":"[]","target":14212015080961893296,"profile":237655285757591511,"path":17523903030608720598,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/libmerkle_root-609819427e5e50ec/dep-lib-libmerkle_root"}}],"rustflags":[],"metadata":7797948686568424061,"config":2202906307356721367,"compile_kind":0}

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-a127695c2afe5bc3/invoked.timestamp

@@ -0,0 +1 @@
+This file has an mtime of when this was started.

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-a127695c2afe5bc3/lib-libmerkle_root

@@ -0,0 +1 @@
+50fac16352fd4033

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-a127695c2afe5bc3/lib-libmerkle_root.json

@@ -0,0 +1 @@
+{"rustc":15699327122048238920,"features":"[]","target":14212015080961893296,"profile":13396965805329499462,"path":17523903030608720598,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/libmerkle_root-a127695c2afe5bc3/dep-lib-libmerkle_root"}}],"rustflags":[],"metadata":7797948686568424061,"config":2202906307356721367,"compile_kind":0}

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-cc456d9e613e2a16/invoked.timestamp

@@ -0,0 +1 @@
+This file has an mtime of when this was started.

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-cc456d9e613e2a16/test-lib-libmerkle_root

@@ -0,0 +1 @@
+b31a33733f0d9c77

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-cc456d9e613e2a16/test-lib-libmerkle_root.json

@@ -0,0 +1 @@
+{"rustc":15699327122048238920,"features":"[]","target":14212015080961893296,"profile":6823863257117401121,"path":17523903030608720598,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/libmerkle_root-cc456d9e613e2a16/dep-test-lib-libmerkle_root"}}],"rustflags":[],"metadata":7797948686568424061,"config":2202906307356721367,"compile_kind":0}

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-f8b0ececa86b90d9/invoked.timestamp

@@ -0,0 +1 @@
+This file has an mtime of when this was started.

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-f8b0ececa86b90d9/test-lib-libmerkle_root

@@ -0,0 +1 @@
+0c398f6cfdb186f5

merkly/accelerator/libmerkle_root/target/debug/.fingerprint/libmerkle_root-f8b0ececa86b90d9/test-lib-libmerkle_root.json

@@ -0,0 +1 @@
+{"rustc":15699327122048238920,"features":"[]","target":14212015080961893296,"profile":13053956386274884697,"path":17523903030608720598,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/libmerkle_root-f8b0ececa86b90d9/dep-test-lib-libmerkle_root"}}],"rustflags":[],"metadata":7797948686568424061,"config":2202906307356721367,"compile_kind":0}

merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-609819427e5e50ec.d

@@ -0,0 +1,7 @@
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-609819427e5e50ec.rmeta: src/lib.rs
+
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/liblibmerkle_root-609819427e5e50ec.rlib: src/lib.rs
+
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-609819427e5e50ec.d: src/lib.rs
+
+src/lib.rs:

merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-a127695c2afe5bc3.d

@@ -0,0 +1,5 @@
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-a127695c2afe5bc3.rmeta: src/lib.rs
+
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-a127695c2afe5bc3.d: src/lib.rs
+
+src/lib.rs:

merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-cc456d9e613e2a16.d

@@ -0,0 +1,5 @@
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-cc456d9e613e2a16: src/lib.rs
+
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-cc456d9e613e2a16.d: src/lib.rs
+
+src/lib.rs:

merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-f8b0ececa86b90d9.d

@@ -0,0 +1,5 @@
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-f8b0ececa86b90d9.rmeta: src/lib.rs
+
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/deps/libmerkle_root-f8b0ececa86b90d9.d: src/lib.rs
+
+src/lib.rs:

merkly/accelerator/libmerkle_root/target/debug/liblibmerkle_root.d

@@ -0,0 +1 @@
+/Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/target/debug/liblibmerkle_root.rlib: /Users/olivmath/Documents/dev/personal/merkly/merkly/accelerator/libmerkle_root/src/lib.rs

merkly/accelerator/mtreers.py

@@ -0,0 +1,54 @@
+from ctypes import (
+    CDLL,
+    CFUNCTYPE,
+    c_ubyte,
+    c_size_t,
+    POINTER,
+    cast,
+    memmove,
+)
+from typing import List
+from Crypto.Hash import keccak as cryptodome_keccak
+
+
+class MTreers:
+    def __init__(self) -> None:
+        callback_function = CFUNCTYPE(None, POINTER(c_ubyte), POINTER(c_ubyte))
+
+        @callback_function
+        def python_callback(ptr, output_ptr):
+            print("PYTHON FROM RUST")
+            data = bytes(ptr[:64])
+
+            keccak_256 = cryptodome_keccak.new(digest_bits=256)
+            keccak_256.update(data)
+
+            memmove(output_ptr, keccak_256.digest(), 32)
+
+        self.keccak = python_callback
+
+        self.lib = CDLL("./merkly/accelerator/libmerkle_root.dylib")
+        self.lib.make_root.argtypes = (
+            callback_function,
+            POINTER(POINTER(c_ubyte)),
+            c_size_t,
+        )
+        self.lib.make_root.restype = POINTER(c_ubyte)
+
+        self.lib.free_32.argtypes = [POINTER(c_ubyte)]
+        self.lib.free_32.restype = None
+
+    def make_root(self, leafs: List[bytes]) -> bytes:
+        pointer_array_type = POINTER(c_ubyte) * len(leafs)
+        pointers = pointer_array_type(
+            *[
+                cast((c_ubyte * 32).from_buffer_copy(leaf), POINTER(c_ubyte))
+                for leaf in leafs
+            ]
+        )
+
+        result_ptr = self.lib.make_root(self.keccak, pointers, len(leafs))
+        result = bytes(result_ptr[:32])
+        self.lib.free_32(result_ptr)
+
+        return result

pyproject.toml

@@ -32,7 +32,8 @@ classifiers = [
 [tool.poetry.dependencies]
     pycryptodome = "^3.19.0"
     pydantic = "^1.10.2"
-    python = "^3.8"
+    python = ">=3.9,<4.0"
+matplotlib = "^3.8.2"
 
 [tool.poetry.dev-dependencies]
     conventional-pre-commit = "^3.0.0"

test/accelerator/test_accelerator.py

@@ -0,0 +1,32 @@
+"""
+Testing Merkle Tree
+"""
+
+from merkly.accelerator.mtreers import MTreers
+from merkly.mtree import MerkleTree
+from Crypto.Hash import keccak
+
+
+def hashing(x: bytes, y: bytes = bytes()) -> bytes:
+    data: bytes = x + y
+    keccak_256 = keccak.new(digest_bits=256)
+    keccak_256.update(data)
+    return keccak_256.digest()
+
+
+def test_merkle_root_rust_and_merkle_root_python():
+    """
+    Instantiated a simple Merkle Tree
+    """
+    leafs = ["a", "b", "c", "d"]
+    tree = MerkleTree(leafs)
+    treers = MTreers()
+
+    bytes_leaves = list(map(lambda x: x.encode(), leafs))
+    hash_leaves = list(map(hashing, bytes_leaves))
+    assert hash_leaves == tree.leaves
+
+    resultrs = treers.make_root(hash_leaves)
+    resultpy = tree.make_root(hash_leaves)
+
+    assert resultrs.hex() == resultpy.hex()

test/merkletreejs/compare.py

@@ -0,0 +1,31 @@
+import hashlib
+
+
+def hash_function(data):
+    return hashlib.sha256(data).digest()
+
+
+def merkle_tree_root(leaves):
+    # Converte as folhas para hashed leaves se necessário
+    hashed_leaves = [hash_function(leaf.encode()) for leaf in leaves]
+
+    # Constrói as camadas da árvore
+    while len(hashed_leaves) > 1:
+        # Se o número de nós for ímpar, duplica o último nó
+        if len(hashed_leaves) % 2 == 1:
+            hashed_leaves.append(hashed_leaves[-1])
+
+        # Combina cada par de nós consecutivos
+        hashed_leaves = [
+            hash_function(hashed_leaves[i] + hashed_leaves[i + 1])
+            for i in range(0, len(hashed_leaves), 2)
+        ]
+
+    # Retorna a raiz da Merkle
+    return hashed_leaves[0]
+
+
+# Exemplo de uso
+leaves = ["a", "b", "c", "d", "e", "f", "g"]
+root = merkle_tree_root(leaves)
+print(root.hex())  # Retorna a raiz da Merkle em formato hexadecimal

test/merkletreejs/index.js

@@ -0,0 +1,15 @@
+const { MerkleTree } = require('merkletreejs')
+const web3 = require("web3")
+const SHA256 = require('crypto-js/sha256')
+
+
+// const keccak = (data) => web3.utils.keccak256(data)
+
+
+const leaves = ['a', 'b', 'c', 'd', 'e', 'f', 'g'].map(SHA256)
+const tree = new MerkleTree(leaves, SHA256, {})
+const root = tree.getRoot().toString('hex')
+
+// console.log(leaves)
+console.log(root)
+

test/merkletreejs/index.py

@@ -0,0 +1,45 @@
+from Crypto.Hash import keccak
+
+import hashlib
+
+
+def hashing(data):
+    return hashlib.sha256(data.encode()).hexdigest()
+
+
+def hashing2(data: str):
+    keccak_256 = keccak.new(digest_bits=256)
+    keccak_256.update(data.encode())
+    return keccak_256.hexdigest()
+
+
+class MerkleTree:
+    def __init__(self, leaves):
+        self.leaves = [hashing2(leaf) for leaf in leaves]
+        self.tree = self.build_tree(self.leaves)
+
+    def build_tree(self, leaves):
+        tree = [leaves]
+        while len(tree[-1]) > 1:
+            layer = tree[-1]
+            next_layer = []
+            for i in range(0, len(layer), 2):
+                left = layer[i]
+                right = layer[i + 1] if i + 1 < len(layer) else left
+                next_layer.append(hashing2(left + right))
+            tree.append(next_layer)
+        return tree
+
+    def get_merkle_root(self):
+        return self.tree[-1][0] if self.tree else None
+
+
+leaves = ["a", "b", "c", "d"]
+tree = MerkleTree(leaves)
+
+root = tree.get_merkle_root()
+leafs = tree.leaves
+
+for i in leafs:
+    print(i)
+print(root)