build(local): improve local build process and update README (#201)

* build(local): improve local build process and update README

* build: update geth version with fhe rand

* chore: bump fhevm-tfhe-cli and fhevm version

.env

@@ -1,8 +1,6 @@
-# TFHE_RS_VERSION=0.2.1
+# TFHE_RS_VERSION=0.3.1
 # TFHE_RS_PATH=../tfhe-rs
-# ZBC_DEVELOPMENT_PATH=../zbc-development
-# ZBC_SOLIDITY_PATH=../zbc-solidity
-# ZBC_FHE_TOOL_PATH=../zbc-fhe-tool
-#LOCAL_BUILD=false
+# FHEVM_SOLIDITY_PATH=../fhevm-solidity
+# FHEVM_TFHE_CLI_PATH=../fhevm-tfhe-cli
+# LOCAL_BUILD=false
 LOCAL_BUILD=true
-GOPRIVATE=github.com/zama-ai/*

Makefile

@@ -41,15 +41,15 @@ TFHE_RS_VERSION ?= 0.3.1
 USE_DOCKER_FOR_FHE_KEYS ?= true
 FHEVM_TFHE_CLI_PATH ?= $(WORKDIR)/fhevm-tfhe-cli
 FHEVM_TFHE_CLI_PATH_EXISTS := $(shell test -d $(FHEVM_TFHE_CLI_PATH)/.git && echo "true" || echo "false")
-FHEVM_TFHE_CLI_VERSION ?= v0.2.0
+FHEVM_TFHE_CLI_VERSION ?= v0.2.1
 
 FHEVM_DECRYPTIONS_DB_PATH ?= $(WORKDIR)/fhevm-decryptions-db
 FHEVM_DECRYPTIONS_DB_PATH_EXISTS := $(shell test -d $(FHEVM_DECRYPTIONS_DB_PATH)/.git && echo "true" || echo "false")
 FHEVM_DECRYPTIONS_DB_VERSION ?= v0.2.0
 
 FHEVM_SOLIDITY_PATH ?= $(WORKDIR)/fhevm-solidity
 FHEVM_SOLIDITY_PATH_EXISTS := $(shell test -d $(FHEVM_SOLIDITY_PATH)/.git && echo "true" || echo "false")
-FHEVM_SOLIDITY_VERSION ?= v0.1.11
+FHEVM_SOLIDITY_VERSION ?= v0.1.12
 ETHERMINT_VERSION := $(shell ./scripts/get_module_version.sh go.mod zama.ai/ethermint)
 GO_ETHEREUM_VERSION := $(shell ./scripts/get_module_version.sh go.mod zama.ai/go-ethereum)
 UPDATE_GO_MOD = go.mod.updated
@@ -159,37 +159,35 @@ print-info:
 	@bash scripts/get_repository_info.sh fhevm-tfhe-cli $(FHEVM_TFHE_CLI_PATH)
 	@bash scripts/get_repository_info.sh fhevm-solidity $(FHEVM_SOLIDITY_PATH)
 
+copy_c_api_to_system_path:
+# In tfhe.go the library path is specified as following : #cgo LDFLAGS: -L/usr/lib/ -ltfhe
+	$(SUDO) cp $(TFHE_RS_PATH)/target/release/tfhe.h /usr/include/
+	$(SUDO) cp $(TFHE_RS_PATH)/target/release/libtfhe.* /usr/lib/
 
-
-build_c_api_tfhe:
+build_c_api_tfhe: check-tfhe-rs
 	$(info build tfhe-rs C API)
 	mkdir -p $(WORKDIR)/
 	$(info tfhe-rs path $(TFHE_RS_PATH))
 	$(info sudo_bin $(SUDO_BIN))
 	cd $(TFHE_RS_PATH) && RUSTFLAGS="" make build_c_api_experimental_deterministic_fft
 	ls $(TFHE_RS_PATH)/target/release
-# In tfhe.go the library path is specified as following : #cgo LDFLAGS: -L/usr/lib/tfhe -ltfhe
-	$(SUDO) cp $(TFHE_RS_PATH)/target/release/tfhe.h /usr/include/
-	$(SUDO) cp $(TFHE_RS_PATH)/target/release/libtfhe.* /usr/lib/
 
-build: 
-	BUILD_ARGS=-o $(BUILDDIR)
-	$(info build)
-
+
 build-linux:
 	$(info build-linux)
 	GOOS=linux GOARCH=amd64 LEDGER_ENABLED=false $(MAKE) build
 
-build-local: check-tfhe-rs  go.sum build_c_api_tfhe $(BUILDDIR)/
-	$(info build-local)
+$(BUILD_TARGETS): go.sum $(BUILDDIR)/
+	$(info build)
+	go install $(BUILD_FLAGS) $(BUILD_ARGS) ./...
+	@echo 'evmosd binary is ready in $(HOME)/go/bin'
+
+build-local:   go.sum build_c_api_tfhe copy_c_api_to_system_path $(BUILDDIR)/
+	$(info build-local for docker build)
 	go build $(BUILD_FLAGS) -o build $(BUILD_ARGS) ./...
 	@echo 'evmosd binary is ready in build folder'
 
 
-# $(BUILD_TARGETS): go.sum $(BUILDDIR)/
-$(BUILD_TARGETS): go.sum build_c_api_tfhe $(BUILDDIR)/
-	$(info BUILD_TARGETS)
-	go $@ $(BUILD_FLAGS) -o build $(BUILD_ARGS) ./...
 
 check-tfhe-rs: $(WORKDIR)/
 	$(info check-tfhe-rs)
@@ -483,6 +481,10 @@ clean: clean-node-storage
 	build
 	rm -f $(UPDATE_GO_MOD)
 
+clean-local-evmos:
+	rm -r $(HOME)/.evmosd/config
+	rm -r $(HOME)/.evmosd/keyring-test/
+	rm -r $(HOME)/.evmosd/data/
 
 all: build
 

README.md

@@ -57,14 +57,150 @@ The quick start is to follow this [section](#from-github-package-registry)
 
 # Local build
 
+## Prepare tfhe-rs C API
+
+### Build
+
+To build automatically the C library one can use the following commands:
+
+```bash
+make build_c_api_tfhe
+```
+
+This will clone **tfhe-rs** repository in work_dir folder and build the C api in __work_dir/tfhe-rs/target/release__. 
+
+If the developer has its own **tfhe-rs** repository the TFHE_RS_PATH env variable could be set in .env file. 
+
+### Copy tfhe header file and C library
+
+**Go-ethereum** needs the tfhe.h header file located in __go-ethereum/core/vm__ and the libtfhe.so (linux) or libtfhe.dylib for (Mac) in __go-ethereum/core/vm/lib__.
+
+```bash
+cp work_dir/tfhe-rs/target/release/tfhe.h ../go-ethereum/core/vm
+mkdir -p ../go-ethereum/core/vm/lib
+# Mac
+cp work_dir/tfhe-rs/target/release/libtfhe.dylib ../go-ethereum/core/vm/lib
+# Linux
+cp work_dir/tfhe-rs/target/release/libtfhe.so ../go-ethereum/core/vm/lib
+# For linux set LD_LIBRARY_PATH to libtfhe.so also
+```
+
+<details>
+  <summary>Why do we need to copy the header file and libtfhe?</summary>
+<br />
+
+In order to extend geth, we give access to all tfhe operations gathered in the lib c through pre-compiled smart contracts. One can check the file called **tfhe.go** in  __go-ethereum/core/vm__ to go deeper.
+
+</details>
+<br />
+
+
+## Prepare custom go-ethereum and ethermint repositories
+
+To use custom **go-ethereum** and **ethermint** repositories, clone them at the same level as evmos, make your changes and update the go.mod file accordingly:
+
+```bash
+-replace github.com/ethereum/go-ethereum v1.10.19 => github.com/zama-ai/go-ethereum v0.1.10
++replace github.com/ethereum/go-ethereum v1.10.19 => ../go-ethereum
+
+-replace github.com/evmos/ethermint v0.19.3 => github.com/zama-ai/ethermint v0.1.2
++replace github.com/evmos/ethermint v0.19.3 => ../ethermint
+```
+
+Here is the hierarchy of folders:
+
+```bash
+.
+├── evmos
+│   └── work_dir
+│       └── tfhe-rs
+├── go-ethereum
+├── ethermint
+```
+
+## Build evmosd binary
+
 To build evmosd binary directly in your system. 
 
 ```bash
-export GOPRIVATE=github.com/zama-ai/*
-make build-local
+make install
+```
+
+The binary is installed in your system go binary path (e.g. $HOME/go/bin).
+If needed update your **PATH** env variable to be able to run evmosd binary from anywhere. 
+
+## Run the node
+
+### Prepare FHE keys
+
+```bash
+LOCAL_BUILD_KEY_PATH="$HOME/.evmosd/zama/keys/network-fhe-keys" ./scripts/prepare_volumes_from_fhe_tool_docker.sh v0.2.0
+```
+
+This script generates fhe keys and copy them to evmos HOME folder in __$HOME/.evmosd/zama/keys/network-fhe-keys__.
+
+
+### Setup the node
+
+```bash
+# jq is required
+./setup.sh
+```
+
+### Start the node
+
+```bash
+./start.sh
+# in a new terminal run the fhevm-decryption-db
+docker run -p 8001:8001 ghcr.io/zama-ai/fhevm-decryptions-db:v0.1.5
 ```
 
-The binary is built in build folder.
+### Reset state
+
+```bash
+make clean-local-evmos
+# must run ./setup.sh after
+```
+
+
+IMPORTANT NOTES:
+
+
+<details>
+  <summary>Use the faucet</summary>
+<br />
+
+```bash
+# In evmos root folder
+# Replace with your ethereum address
+python3 faucet.py 0xa5e1defb98EFe38EBb2D958CEe052410247F4c80
+```
+
+</details>
+
+<details>
+  <summary>Check if evmosd is linked with the right tfhe-rs C libray - Linux</summary>
+<br />
+
+```bash
+ldd $HOME/go/bin/evmosd
+	linux-vdso.so.1 (0x00007ffdb6d73000)
+	libtfhe.so => /PATH_TO/tfhe-rs/target/release/libtfhe.so (0x00007fa87c3a7000)
+	libc.so.6 => /lib64/libc.so.6 (0x00007fa87c185000)
+	libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007fa87c165000)
+	libm.so.6 => /lib64/libm.so.6 (0x00007fa87c087000)
+	/lib64/ld-linux-x86-64.so.2 (0x00007fa87c9e5000)
+```
+
+If the user get:
+```bash
+evmosd: error while loading shared libraries: libtfhe.so: cannot open shared object file: No such file or directory
+```
+
+For linux one solution is to update the LD_LIBRARY_PATH to the libtfhe.so compiled in tfhe-rs
+
+</details>
+<br />
 
 Dependencies:
 
@@ -109,8 +245,8 @@ make build-docker
 - Build a base image (or retrieve it from ghcr.io) called __zama-zbc-build__.
 - Check tfhe-rs is available in TFHE_RS_PATH (default is work_dir/tfhe-rs)
 - In any case the custom version or the cloned (TFHE_RS_VERSION) one is copied into work_dir/tfhe-rs
-- Clone go-ethereum and ethermint to work_dir (version are parsed from go.mod to avoid handling ssh keys inside docker because those repositories are private)
-- Update go.mod to make it use local repositories (related to the just above changes)
+- Clone go-ethereum and ethermint to work_dir (version are parsed from go.mod)
+- Update go.mod to force use local repositories (related to the just above changes)
 - Build a container called __evmosnodelocal__.
 
 </details>
@@ -171,10 +307,7 @@ make stop_evmos
 - copy them at the right folder using scripts/prepare_demo_local.sh script
 - start evmosnodelocal0 and oracledb (local build) using docker-compose/docker-compose.local.yml file
 - run the e2e test 
-  - copy pks to encrypt user input using $(ZBC_SOLIDITY_PATH)/prepare_fhe_keys_for_e2e_test script
-  - start the test using $(ZBC_SOLIDITY_PATH)/run_ERC20_e2e_test.sh
-    - Get the private key of main account 
-    - Give it to the python test script $(ZBC_SOLIDITY_PATH)/ci/tests/ERC20.py
+  - start the test from fhevm-solidity
 
 </details>
 <br />
@@ -237,20 +370,6 @@ Here is a tutorial on [how to manage ghcr.io access](https://github.com/zama-ai/
   docker build . -t zama-zbc-build -f docker/Dockerfile.zbc.build
   ```
 </details>
-
-<details>
-  <summary>Troubleshoot go modules for local-build</summary>
-
-Because evmos depends on private [go-ethereum](https://github.com/zama-ai/go-ethereum) and [ethermint](https://github.com/zama-ai/ethermint) repositories, one need to pay attention to two points to allow go modules manager to work correctly.
-
-1. Check that GOPRIVATE is set to __github.com/zama-ai/*__ (normally this env variable is set by default in Makefile)
-2. Check you have the following lines in your gitconfig files:
-
-```bash
-[url "ssh://[email protected]/"]
-        insteadOf = https://github.com/
-```
-</details>
 <br />
 
 

docker/Dockerfile.evmos-node.developer

@@ -3,7 +3,7 @@
 FROM ghcr.io/zama-ai/fhevm-decryptions-db:v0.2.0 as oracle-env
 FROM ghcr.io/zama-ai/fhevm-tfhe-cli:v0.2.0 as tfhe-cli
 
-FROM ghcr.io/zama-ai/evmos-node:v0.1.9-alpha
+FROM ghcr.io/zama-ai/evmos-node:v0.1.9
 
 WORKDIR /config
 

docker/Dockerfile.evmos-node.local

@@ -11,7 +11,7 @@ RUN cp go.mod.updated /src/evmos/go.mod
 
 RUN tail /src/evmos/go.mod
 
-RUN make build
+RUN make build-local
 RUN ls /src/evmos
 RUN ls /src/evmos/build
 RUN mkdir -p /src/evmos/build

docker/Dockerfile.evmos-node.testnet

@@ -25,7 +25,7 @@ RUN cp /src/tfhe-rs/target/release/libtfhe.* /src/go-ethereum/core/vm/lib/
 # RUN cp /src/tfhe-rs/target/release/libtfhe.* /usr/lib/
 
 
-RUN make build
+RUN make build-local
 
 ARG ZBC_BUILD_IMAGE_TAG
 FROM ghcr.io/zama-ai/zama-zbc-build:$ZBC_BUILD_IMAGE_TAG

go.mod

@@ -173,6 +173,6 @@ replace (
 replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
 
 // TODO: Remove once our forks are public.
-replace github.com/ethereum/go-ethereum v1.10.19 => github.com/zama-ai/go-ethereum v0.1.10
+replace github.com/ethereum/go-ethereum v1.10.19 => github.com/zama-ai/go-ethereum v0.1.11
 
 replace github.com/evmos/ethermint v0.19.3 => github.com/zama-ai/ethermint v0.1.2

scripts/prepare_volumes_from_fhe_tool_docker.sh

@@ -28,9 +28,11 @@ echo "###########################################################"
 echo "Keys creation is done, they are stored in $KEYS_FULL_PATH"
 echo "###########################################################"
 
+NETWORK_KEYS_PUBLIC_PATH="${LOCAL_BUILD_KEY_PATH:-./volumes/network-public-fhe-keys}"
+NETWORK_KEYS_PRIVATE_PATH="${LOCAL_BUILD_KEY_PATH:-./volumes/network-private-fhe-keys}"
 
-NETWORK_KEYS_PUBLIC_PATH=./volumes/network-public-fhe-keys
-NETWORK_KEYS_PRIVATE_PATH=./volumes/network-private-fhe-keys
+echo "$NETWORK_KEYS_PUBLIC_PATH"
+echo "$NETWORK_KEYS_PRIVATE_PATH"
 
 MANDATORY_KEYS_LIST=('sks' 'cks' 'pks')