feat(zk_inception): Add ecosystem ports scanner

.github/workflows/ci-core-reusable.yml

@@ -245,7 +245,6 @@ jobs:
           --server-db-name=zksync_server_localhost_validium \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_validium \
-          --port-offset 2000 \
           --chain validium
 
       - name: Create and initialize chain with Custom Token
@@ -269,7 +268,6 @@ jobs:
           --server-db-name=zksync_server_localhost_custom_token \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_custom_token \
-          --port-offset 3000 \
           --chain custom_token
 
       - name: Create and register chain with transactions signed "offline"
@@ -328,7 +326,6 @@ jobs:
           --server-db-name=zksync_server_localhost_consensus \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_consensus \
-          --port-offset 4000 \
           --chain consensus
 
       - name: Build test dependencies

zk_toolbox/crates/config/src/general.rs

@@ -204,6 +204,7 @@ impl FileConfigWithDefaultName for GeneralConfig {
     const FILE_NAME: &'static str = GENERAL_FILE;
 }
 
+#[derive(Debug, Default, Clone)]
 pub struct PortsConfig {
     pub web3_json_rpc_http_port: u16,
     pub web3_json_rpc_ws_port: u16,

zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs

@@ -1,5 +1,3 @@
-use std::str::FromStr;
-
 use clap::Parser;
 use common::{forge::ForgeScriptArgs, Prompt};
 use config::ChainConfig;
@@ -13,35 +11,10 @@ use crate::{
     defaults::LOCAL_RPC_URL,
     messages::{
         MSG_DEPLOY_PAYMASTER_PROMPT, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP,
-        MSG_L1_RPC_URL_INVALID_ERR, MSG_L1_RPC_URL_PROMPT, MSG_PORT_OFFSET_HELP,
+        MSG_L1_RPC_URL_INVALID_ERR, MSG_L1_RPC_URL_PROMPT,
     },
 };
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct PortOffset(u16);
-
-impl PortOffset {
-    pub fn from_chain_id(chain_id: u16) -> Self {
-        Self((chain_id - 1) * 100)
-    }
-}
-
-impl FromStr for PortOffset {
-    type Err = String;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        s.parse::<u16>()
-            .map(PortOffset)
-            .map_err(|_| "Invalid port offset".to_string())
-    }
-}
-
-impl From<PortOffset> for u16 {
-    fn from(port_offset: PortOffset) -> Self {
-        port_offset.0
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, Parser)]
 pub struct InitArgs {
     /// All ethereum environment related arguments
@@ -55,8 +28,6 @@ pub struct InitArgs {
     pub deploy_paymaster: Option<bool>,
     #[clap(long, help = MSG_L1_RPC_URL_HELP)]
     pub l1_rpc_url: Option<String>,
-    #[clap(long, help = MSG_PORT_OFFSET_HELP)]
-    pub port_offset: Option<PortOffset>,
 }
 
 impl InitArgs {
@@ -86,10 +57,6 @@ impl InitArgs {
             genesis_args: self.genesis_args.fill_values_with_prompt(config),
             deploy_paymaster,
             l1_rpc_url,
-            port_offset: self
-                .port_offset
-                .unwrap_or(PortOffset::from_chain_id(config.id as u16))
-                .into(),
         }
     }
 }
@@ -100,5 +67,4 @@ pub struct InitArgsFinal {
     pub genesis_args: GenesisArgsFinal,
     pub deploy_paymaster: bool,
     pub l1_rpc_url: String,
-    pub port_offset: u16,
 }

zk_toolbox/crates/zk_inception/src/commands/chain/init.rs

@@ -1,8 +1,8 @@
-use anyhow::{bail, Context};
+use anyhow::Context;
 use common::{config::global_config, git, logger, spinner::Spinner};
 use config::{
     copy_configs, ports_config, set_l1_rpc_url, traits::SaveConfigWithBasePath,
-    update_from_chain_config, update_ports, ChainConfig, EcosystemConfig, GeneralConfig,
+    update_from_chain_config, ChainConfig, EcosystemConfig,
 };
 use types::BaseToken;
 use xshell::Shell;
@@ -27,7 +27,10 @@ use crate::{
         MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG,
         MSG_UPDATING_TOKEN_MULTIPLIER_SETTER_SPINNER, MSG_WALLET_TOKEN_MULTIPLIER_SETTER_NOT_FOUND,
     },
-    utils::consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets},
+    utils::{
+        consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets},
+        ports::EcosystemPortsScanner,
+    },
 };
 
 pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> {
@@ -54,10 +57,12 @@ pub async fn init(
     ecosystem_config: &EcosystemConfig,
     chain_config: &ChainConfig,
 ) -> anyhow::Result<()> {
+    let mut ecosystem_ports = EcosystemPortsScanner::scan(shell)?;
     copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?;
 
     let mut general_config = chain_config.get_general_config()?;
-    apply_port_offset(init_args.port_offset, &mut general_config)?;
+    ecosystem_ports.allocate_ports(&mut general_config)?;
+
     let ports = ports_config(&general_config).context(MSG_PORTS_CONFIG_ERR)?;
 
     let consensus_keys = generate_consensus_keys();
@@ -176,15 +181,3 @@ pub async fn init(
 
     Ok(())
 }
-
-fn apply_port_offset(port_offset: u16, general_config: &mut GeneralConfig) -> anyhow::Result<()> {
-    let Some(mut ports_config) = ports_config(general_config) else {
-        bail!("Missing ports config");
-    };
-
-    ports_config.apply_offset(port_offset);
-
-    update_ports(general_config, &ports_config)?;
-
-    Ok(())
-}

zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs

@@ -31,7 +31,7 @@ use super::{
 use crate::{
     accept_ownership::{accept_admin, accept_owner},
     commands::{
-        chain::{self, args::init::PortOffset},
+        chain::{self},
         ecosystem::create_configs::{
             create_erc20_deployment_config, create_initial_deployments_config,
         },
@@ -112,7 +112,6 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> {
             genesis_args: genesis_args.clone().fill_values_with_prompt(&chain_config),
             deploy_paymaster: final_ecosystem_args.deploy_paymaster,
             l1_rpc_url: final_ecosystem_args.ecosystem.l1_rpc_url.clone(),
-            port_offset: PortOffset::from_chain_id(chain_config.id as u16).into(),
         };
 
         chain::init::init(

zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs

@@ -11,14 +11,14 @@ use url::Url;
 use xshell::Shell;
 
 use crate::{
-    commands::chain::args::init::PortOffset,
     consts::L2_BASE_TOKEN_ADDRESS,
-    defaults::{generate_explorer_db_name, DATABASE_EXPLORER_URL},
+    defaults::{generate_explorer_db_name, DATABASE_EXPLORER_URL, PORT_RANGE},
     messages::{
         msg_chain_load_err, msg_explorer_db_name_prompt, msg_explorer_db_url_prompt,
         msg_explorer_initializing_database_for, MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR,
         MSG_EXPLORER_INITIALIZED,
     },
+    utils::ports::{EcosystemPorts, EcosystemPortsScanner},
 };
 
 pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> {
@@ -28,6 +28,7 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> {
         Some(ref chain_name) => vec![chain_name.clone()],
         None => ecosystem_config.list_of_chains(),
     };
+    let mut ports = EcosystemPortsScanner::scan(shell)?;
     // Initialize chains one by one
     let mut explorer_config = ExplorerConfig::read_or_create_default(shell)?;
     for chain_name in chains_enabled.iter() {
@@ -36,7 +37,7 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> {
             .load_chain(Some(chain_name.clone()))
             .context(msg_chain_load_err(chain_name))?;
         // Build backend config - parameters required to create explorer backend services
-        let backend_config = build_backend_config(&chain_config);
+        let backend_config = build_backend_config(&mut ports, &chain_config)?;
         // Initialize explorer database
         initialize_explorer_database(&backend_config.database_url).await?;
         // Create explorer backend docker compose file
@@ -58,16 +59,22 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> {
     Ok(())
 }
 
-fn build_backend_config(chain_config: &ChainConfig) -> ExplorerBackendConfig {
+fn build_backend_config(
+    ports: &mut EcosystemPorts,
+    chain_config: &ChainConfig,
+) -> anyhow::Result<ExplorerBackendConfig> {
     // Prompt explorer database name
     logger::info(msg_explorer_initializing_database_for(&chain_config.name));
     let db_config = fill_database_values_with_prompt(chain_config);
 
     // Allocate ports for backend services
-    let backend_ports = allocate_explorer_services_ports(chain_config);
+    let backend_ports = allocate_explorer_services_ports(ports)?;
 
     // Build explorer backend config
-    ExplorerBackendConfig::new(db_config.full_url(), &backend_ports)
+    Ok(ExplorerBackendConfig::new(
+        db_config.full_url(),
+        &backend_ports,
+    ))
 }
 
 async fn initialize_explorer_database(db_url: &Url) -> anyhow::Result<()> {
@@ -92,10 +99,15 @@ fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig
     db::DatabaseConfig::new(explorer_db_url, explorer_db_name)
 }
 
-fn allocate_explorer_services_ports(chain_config: &ChainConfig) -> ExplorerBackendPorts {
-    // Try to allocate intuitive ports with an offset from the defaults
-    let offset: u16 = PortOffset::from_chain_id(chain_config.id as u16).into();
-    ExplorerBackendPorts::default().with_offset(offset)
+fn allocate_explorer_services_ports(
+    ports: &mut EcosystemPorts,
+) -> anyhow::Result<ExplorerBackendPorts> {
+    Ok(ExplorerBackendPorts {
+        api_http_port: ports.allocate_port(PORT_RANGE, "Explorer API".to_string())?,
+        data_fetcher_http_port: ports
+            .allocate_port(PORT_RANGE, "Explorer Data Fetcher".to_string())?,
+        worker_http_port: ports.allocate_port(PORT_RANGE, "Explorer Worker".to_string())?,
+    })
 }
 
 fn build_explorer_chain_config(

zk_toolbox/crates/zk_inception/src/defaults.rs

@@ -1,3 +1,5 @@
+use std::ops::Range;
+
 use config::ChainConfig;
 use lazy_static::lazy_static;
 use url::Url;
@@ -11,6 +13,11 @@ lazy_static! {
         Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap();
 }
 
+// Default port range
+pub const PORT_RANGE_START: u16 = 3000;
+pub const PORT_RANGE_END: u16 = 5000;
+pub const PORT_RANGE: Range<u16> = PORT_RANGE_START..PORT_RANGE_END;
+
 pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper";
 pub const ROCKS_DB_TREE: &str = "tree";
 pub const ROCKS_DB_PROTECTIVE_READS: &str = "protective_reads";

zk_toolbox/crates/zk_inception/src/messages.rs

@@ -51,7 +51,6 @@ pub(super) fn msg_path_to_zksync_does_not_exist_err(path: &str) -> String {
 
 /// Ecosystem and chain init related messages
 pub(super) const MSG_L1_RPC_URL_HELP: &str = "L1 RPC URL";
-pub(super) const MSG_PORT_OFFSET_HELP: &str = "Add a costant offset to the ports exposed by the components. Useful when running multiple chains on the same machine";
 pub(super) const MSG_GENESIS_ARGS_HELP: &str = "Genesis options";
 pub(super) const MSG_DEV_ARG_HELP: &str =
     "Deploy ecosystem  using all defaults. Suitable for local development";

zk_toolbox/crates/zk_inception/src/utils/mod.rs

@@ -1,3 +1,4 @@
 pub mod consensus;
 pub mod forge;
+pub mod ports;
 pub mod rocks_db;