diff --git a/.gitignore b/.gitignore index bbd13e2319a..234ce65e5b4 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,6 @@ configs/* era-observability/ core/tests/ts-integration/deployments-zk transactions/ + +# Ecosystem backups +ecosystem_backups/ diff --git a/backup-ecosystem.sh b/backup-ecosystem.sh new file mode 100755 index 00000000000..dbdf82e3a38 --- /dev/null +++ b/backup-ecosystem.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Check if the ecosystem name was provided as an argument +if [ -z "$1" ]; then + echo "Usage: ./backup-ecosystem ECOSYSTEM_NAME" + exit 1 +fi + +# Store the first argument as ECOSYSTEM_NAME +ECOSYSTEM_NAME=$1 + +# Prompt for the Postgres password and store it in PGPASSWORD +read -sp "Enter Postgres password: " PGPASSWORD +export PGPASSWORD + +# Path to the secrets.yaml file +SECRETS_FILE="./chains/${ECOSYSTEM_NAME}/configs/secrets.yaml" + +# Check if the secrets.yaml file exists +if [ ! -f "$SECRETS_FILE" ]; then + echo "Error: $SECRETS_FILE does not exist." + exit 1 +fi + +# Extract server_url and prover_url from the secrets.yaml file +SERVER_DB_NAME=$(grep 'server_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}') +PROVER_DB_NAME=$(grep 'prover_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}') + +# Export the database names +echo "Extracted SERVER_DB_NAME: $SERVER_DB_NAME" +echo "Extracted PROVER_DB_NAME: $PROVER_DB_NAME" + +# Create backup directory +mkdir -p "./ecosystem_backups/${ECOSYSTEM_NAME}" + +# Run pg_dump commands +echo "Running pg_dump for $SERVER_DB_NAME..." +pg_dump -U postgres -h localhost "$SERVER_DB_NAME" > "ecosystem_backups/${ECOSYSTEM_NAME}/${SERVER_DB_NAME}_backup.sql" +echo "Running pg_dump for $PROVER_DB_NAME..." +pg_dump -U postgres -h localhost "$PROVER_DB_NAME" > "ecosystem_backups/${ECOSYSTEM_NAME}/${PROVER_DB_NAME}_backup.sql" + +# Unset the PGPASSWORD variable for security +unset PGPASSWORD + +# Copy the chain configuration files +cp -r "./chains/${ECOSYSTEM_NAME}" "./ecosystem_backups/${ECOSYSTEM_NAME}/" + +# Copy the configs directory +cp -r "./configs" "./ecosystem_backups/${ECOSYSTEM_NAME}/" + +echo "Backup completed." diff --git a/eigenda-integration.md b/eigenda-integration.md index 4d02fa6a824..638719e2b4d 100644 --- a/eigenda-integration.md +++ b/eigenda-integration.md @@ -71,3 +71,121 @@ zk_supervisor test integration --chain eigen_da ### Metrics Access Grafana at [http://localhost:3000/](http://localhost:3000/), go to dashboards and select `EigenDA`. + +## Holesky Setup + +### Modify localhost chain id number + +Modify line 32 in `zk_toolbox/crates/types/src/l1_network.rs`: + +```rs +L1Network::Localhost => 17000, +``` + +Then recompile the zk toolbox: + +```bash +./bin/zkt +``` + +### Used wallets + +Modify `etc/env/file_based/wallets.yaml` and `configs/wallets.yaml` with the following wallets: + +```yaml +# Use your own holesky wallets, be sure they have enough funds +``` + +> ⚠️ Some steps distribute ~5000ETH to some wallets, modify `AMOUNT_FOR_DISTRIBUTION_TO_WALLETS` to a lower value if needed. + +### EigenProxy RPC + +Get `EIGEN_SIGNER_PK` from 1password and set it as an `env` var: + +```bash +export EIGEN_SIGNER_PK= +export HOLESKY_RPC_URL= +``` + +Modify `docker-compose.yml` to use holesky RPCs: + +```rust + eigenda-proxy: + image: ghcr.io/layr-labs/eigenda-proxy + environment: + - EIGEN_SIGNER_PK=$EIGEN_SIGNER_PK + - HOLESKY_RPC_URL=$HOLESKY_RPC_URL + ports: + - "4242:4242" + command: ./eigenda-proxy --addr 0.0.0.0 --port 4242 --eigenda-disperser-rpc disperser-holesky.eigenda.xyz:443 --eigenda-signer-private-key-hex $EIGEN_SIGNER_PK --eigenda-eth-rpc $HOLESKY_RPC_URL --eigenda-svc-manager-addr 0xD4A7E1Bd8015057293f0D0A557088c286942e84b --eigenda-eth-confirmation-depth 0 +``` + +### Create and initialize the ecosystem + +(be sure to have postgres container running on the background) + +```bash +zk_inception chain create \ + --chain-name holesky_eigen_da \ + --chain-id 114411 \ + --prover-mode no-proofs \ + --wallet-creation localhost \ + --l1-batch-commit-data-generator-mode validium \ + --base-token-address 0x0000000000000000000000000000000000000001 \ + --base-token-price-nominator 1 \ + --base-token-price-denominator 1 \ + --set-as-default false + +zk_inception ecosystem init \ + --deploy-paymaster true \ + --deploy-erc20 true \ + --deploy-ecosystem true \ + --l1-rpc-url $HOLESKY_RPC_URL \ + --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ + --server-db-name=zksync_server_holesky_eigen_da \ + --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ + --prover-db-name=zksync_prover_holesky_eigen_da \ + --chain holesky_eigen_da \ + --verbose +``` + +### Start the server + +```bash +zk_inception server --chain holesky_eigen_da +``` + +## Backup and restoration + +It's possible to run the zk stack on one computer, and then migrate it to another, this is specially useful for holesky testing. + +### Backup + +Suppose that you want to make a backup of `holesky_eigen_da` ecosystem, you only need to run: + +```bash +./backup-ecosystem.sh holesky_eigen_da +``` + +This will generate a directory inside of `ecosystem_backups` with the name `holesky_eigen_da`. + +### Restoration + +1. Move the `ecoystem_backups/holesky_eigen_da` directory to the other computer, it should be placed in the root of the project. + +2. Restore the ecosystem with: + +```bash +./restore-ecosystem.sh holesky_eigen_da +``` + +Note that: + +- The `postgres` container has to be running. +- The `chain_id` can't be already in use. +- If you are restoring a local ecosystem, you have to use the same `reth` container as before. +- If no ecosystem has been `init`ialized on this computer before, run this command: + +```bash +git submodule update --init --recursive && zk_supervisor contracts +``` diff --git a/restore-ecosystem.sh b/restore-ecosystem.sh new file mode 100755 index 00000000000..7fbff3e9802 --- /dev/null +++ b/restore-ecosystem.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Check if the ecosystem name was provided as an argument +if [ -z "$1" ]; then + echo "Usage: ./restore-ecosystem ECOSYSTEM_NAME" + exit 1 +fi + +# Store the first argument as ECOSYSTEM_NAME +ECOSYSTEM_NAME=$1 + +# Prompt for the Postgres password and store it in PGPASSWORD +read -sp "Enter Postgres password: " PGPASSWORD +export PGPASSWORD + +# Check if the chain directory exists +CHAIN_PATH="./chains/${ECOSYSTEM_NAME}" +BACKUP_PATH="./ecosystem_backups/${ECOSYSTEM_NAME}" + +# Check if the backup directory exists +if [ ! -d "$BACKUP_PATH" ]; then + echo "Error: Backup not found at $BACKUP_PATH." + exit 1 +fi + +# Check if the postgres is running +if ! docker ps --filter "name=postgres" --filter "status=running" | grep "postgres" > /dev/null; then + echo "Error: postgres not running, set it up first with 'zk_inception containers'." + exit 1 +fi + +# Fix backup files $ZKSYNC_HOME paths +find_and_replace() { + local target_file=$1 + + sed -i '' -e "s|db_path:.*zksync-era/\./|db_path: $(pwd)/./|g" "$target_file" + sed -i '' -e "s|state_keeper_db_path:.*zksync-era/\./|state_keeper_db_path: $(pwd)/./|g" "$target_file" + sed -i '' -e "s|path:.*zksync-era/\./|path: $(pwd)/./|g" "$target_file" + sed -i '' -e "s|configs:.*zksync-era/\./|configs: $(pwd)/./|g" "$target_file" +} + +# Array of specific files to modify +files=("$BACKUP_PATH/$ECOSYSTEM_NAME/configs/general.yaml" "$BACKUP_PATH/$ECOSYSTEM_NAME/ZkStack.yaml") + +# Loop over the files and perform the find and replace +for file in "${files[@]}"; do + if [ -f "$file" ]; then + find_and_replace "$file" + else + # Exit with error code + echo "ERROR: backup file $file does not exist." + exit 1 + fi +done + +# Copy the ecosystem backup folder to the chains folder, replacing any existing files +echo "Copying backup files to $CHAIN_PATH..." +cp -r "$BACKUP_PATH/$ECOSYSTEM_NAME" "$CHAIN_PATH" + +# Copy the configs folder in the backup to the configs folder in the root of the project +# TODO: it may be suitable to warn the user about overwriting the existing configs +# and ask for confirmation before proceeding +echo "Copying configs folder from backup..." +cp -r "$BACKUP_PATH/configs" "./" + +# Path to the secrets.yaml file +SECRETS_FILE="$CHAIN_PATH/configs/secrets.yaml" + +# Check if the secrets.yaml file exists +if [ ! -f "$SECRETS_FILE" ]; then + echo "Error: $SECRETS_FILE does not exist." + exit 1 +fi + +# Extract server_url and prover_url from the secrets.yaml file +SERVER_DB_NAME=$(grep 'server_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}') +PROVER_DB_NAME=$(grep 'prover_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}') + +# Show the extracted database names +echo "Extracted SERVER_DB_NAME: $SERVER_DB_NAME" +echo "Extracted PROVER_DB_NAME: $PROVER_DB_NAME" + +# Create and restore the server database +echo "Creating database $SERVER_DB_NAME..." +createdb -U postgres -h localhost "$SERVER_DB_NAME" + +echo "Restoring $SERVER_DB_NAME from backup..." +psql -U postgres -h localhost -d "$SERVER_DB_NAME" -f "$BACKUP_PATH/${SERVER_DB_NAME}_backup.sql" + +# Create and restore the prover database +echo "Creating database $PROVER_DB_NAME..." +createdb -U postgres -h localhost "$PROVER_DB_NAME" + +echo "Restoring $PROVER_DB_NAME from backup..." +psql -U postgres -h localhost -d "$PROVER_DB_NAME" -f "$BACKUP_PATH/${PROVER_DB_NAME}_backup.sql" + +# Unset the PGPASSWORD variable for security +unset PGPASSWORD + +echo "Restore completed."