feat(zk_inception): Add ecosystem ports scanner

.github/workflows/ci-core-reusable.yml

@@ -245,7 +245,6 @@ jobs:
           --server-db-name=zksync_server_localhost_validium \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_validium \
-          --port-offset 2000 \
           --chain validium
 
       - name: Create and initialize chain with Custom Token
@@ -269,7 +268,6 @@ jobs:
           --server-db-name=zksync_server_localhost_custom_token \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_custom_token \
-          --port-offset 3000 \
           --chain custom_token
 
       - name: Create and register chain with transactions signed "offline"
@@ -328,7 +326,6 @@ jobs:
           --server-db-name=zksync_server_localhost_consensus \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_consensus \
-          --port-offset 4000 \
           --chain consensus
 
       - name: Build test dependencies

etc/env/file_based/general.yaml

@@ -80,7 +80,7 @@ operations_manager:
 contract_verifier:
   compilation_timeout: 240
   polling_interval: 1000
-  prometheus_port: 3314
+  prometheus_port: 3318
   port: 3070
   url: http://127.0.0.1:3070
   threads_per_server: 128

zk_toolbox/crates/config/src/consts.rs

@@ -64,6 +64,18 @@ pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002;
 pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040;
 /// Default port for consensus service
 pub const DEFAULT_CONSENSUS_PORT: u16 = 3054;
+/// Default port for the web3 json rpc service
+pub const DEFAULT_WEB3_JSON_RPC_PORT: u16 = 3050;
+/// Default port for the web3 ws rpc service
+pub const DEFAULT_WEB3_WS_RPC_PORT: u16 = 3051;
+/// Default port for the healthcheck service
+pub const DEFAULT_HEALTHCHECK_PORT: u16 = 3071;
+/// Default port for the merkle tree
+pub const DEFAULT_MERKLE_TREE_PORT: u16 = 3072;
+/// Default port for the prometheus service
+pub const DEFAULT_PROMETHEUS_PORT: u16 = 3314;
+/// Default port for the contract verifier service
+pub const DEFAULT_CONTRACT_VERIFIER_PORT: u16 = 3070;
 
 pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api";
 pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher";

zk_toolbox/crates/config/src/general.rs

@@ -113,66 +113,12 @@ pub fn set_file_artifacts(config: &mut GeneralConfig, file_artifacts: FileArtifa
     set_artifact_path!(config.core_object_store, file_artifacts.core_object_store);
 }
 
-pub fn ports_config(config: &GeneralConfig) -> Option<PortsConfig> {
-    let api = config.api_config.as_ref()?;
-    let contract_verifier = config.contract_verifier.as_ref()?;
-    let consensus_port = if let Some(consensus_config) = config.clone().consensus_config {
-        consensus_config.server_addr.port()
-    } else {
-        DEFAULT_CONSENSUS_PORT
-    };
-
-    Some(PortsConfig {
-        web3_json_rpc_http_port: api.web3_json_rpc.http_port,
-        web3_json_rpc_ws_port: api.web3_json_rpc.ws_port,
-        healthcheck_port: api.healthcheck.port,
-        merkle_tree_port: api.merkle_tree.port,
-        prometheus_listener_port: api.prometheus.listener_port,
-        contract_verifier_port: contract_verifier.port,
-        consensus_port,
-    })
-}
-
-pub fn update_ports(config: &mut GeneralConfig, ports_config: &PortsConfig) -> anyhow::Result<()> {
-    let api = config
-        .api_config
-        .as_mut()
-        .context("Api config is not presented")?;
-    let contract_verifier = config
-        .contract_verifier
-        .as_mut()
-        .context("Contract Verifier config is not presented")?;
-    let prometheus = config
-        .prometheus_config
-        .as_mut()
-        .context("Prometheus config is not presented")?;
-    if let Some(consensus) = config.consensus_config.as_mut() {
-        consensus.server_addr.set_port(ports_config.consensus_port);
-        update_port_in_host(&mut consensus.public_addr, ports_config.consensus_port)?;
-    }
-
-    api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port;
-    update_port_in_url(
-        &mut api.web3_json_rpc.http_url,
-        ports_config.web3_json_rpc_http_port,
-    )?;
-    api.web3_json_rpc.ws_port = ports_config.web3_json_rpc_ws_port;
-    update_port_in_url(
-        &mut api.web3_json_rpc.ws_url,
-        ports_config.web3_json_rpc_ws_port,
-    )?;
-    contract_verifier.port = ports_config.contract_verifier_port;
-    update_port_in_url(
-        &mut contract_verifier.url,
-        ports_config.contract_verifier_port,
-    )?;
-    api.healthcheck.port = ports_config.healthcheck_port;
-    api.merkle_tree.port = ports_config.merkle_tree_port;
-    api.prometheus.listener_port = ports_config.prometheus_listener_port;
-
-    prometheus.listener_port = ports_config.prometheus_listener_port;
-
-    Ok(())
+pub fn get_consensus_port(config: &GeneralConfig) -> u16 {
+    config
+        .consensus_config
+        .as_ref()
+        .map(|c| c.server_addr.port())
+        .unwrap_or(DEFAULT_CONSENSUS_PORT)
 }
 
 pub fn override_config(shell: &Shell, path: PathBuf, chain: &ChainConfig) -> anyhow::Result<()> {
@@ -184,7 +130,7 @@ pub fn override_config(shell: &Shell, path: PathBuf, chain: &ChainConfig) -> any
     Ok(())
 }
 
-fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> {
+pub fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> {
     let mut http_url_url = Url::parse(http_url)?;
     if let Err(()) = http_url_url.set_port(Some(port)) {
         anyhow::bail!("Wrong url, setting port is impossible");
@@ -193,7 +139,7 @@ fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> {
     Ok(())
 }
 
-fn update_port_in_host(host: &mut Host, port: u16) -> anyhow::Result<()> {
+pub fn update_port_in_host(host: &mut Host, port: u16) -> anyhow::Result<()> {
     let url = Url::parse(&format!("http://{}", host.0))?;
     let host_str = url.host_str().context("Failed to get host")?;
     host.0 = format!("{host_str}:{port}");
@@ -204,40 +150,6 @@ impl FileConfigWithDefaultName for GeneralConfig {
     const FILE_NAME: &'static str = GENERAL_FILE;
 }
 
-pub struct PortsConfig {
-    pub web3_json_rpc_http_port: u16,
-    pub web3_json_rpc_ws_port: u16,
-    pub healthcheck_port: u16,
-    pub merkle_tree_port: u16,
-    pub prometheus_listener_port: u16,
-    pub contract_verifier_port: u16,
-    pub consensus_port: u16,
-}
-
-impl PortsConfig {
-    pub fn apply_offset(&mut self, offset: u16) {
-        self.web3_json_rpc_http_port += offset;
-        self.web3_json_rpc_ws_port += offset;
-        self.healthcheck_port += offset;
-        self.merkle_tree_port += offset;
-        self.prometheus_listener_port += offset;
-        self.contract_verifier_port += offset;
-        self.consensus_port += offset;
-    }
-
-    pub fn next_empty_ports_config(&self) -> PortsConfig {
-        Self {
-            web3_json_rpc_http_port: self.web3_json_rpc_http_port + 100,
-            web3_json_rpc_ws_port: self.web3_json_rpc_ws_port + 100,
-            healthcheck_port: self.healthcheck_port + 100,
-            merkle_tree_port: self.merkle_tree_port + 100,
-            prometheus_listener_port: self.prometheus_listener_port + 100,
-            contract_verifier_port: self.contract_verifier_port + 100,
-            consensus_port: self.consensus_port + 100,
-        }
-    }
-}
-
 impl SaveConfig for GeneralConfig {
     fn save(&self, shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<()> {
         let bytes =

zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs

@@ -1,5 +1,3 @@
-use std::str::FromStr;
-
 use clap::Parser;
 use common::{forge::ForgeScriptArgs, Prompt};
 use config::ChainConfig;
@@ -13,35 +11,10 @@ use crate::{
     defaults::LOCAL_RPC_URL,
     messages::{
         MSG_DEPLOY_PAYMASTER_PROMPT, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP,
-        MSG_L1_RPC_URL_INVALID_ERR, MSG_L1_RPC_URL_PROMPT, MSG_PORT_OFFSET_HELP,
+        MSG_L1_RPC_URL_INVALID_ERR, MSG_L1_RPC_URL_PROMPT, MSG_NO_PORT_REALLOCATION_HELP,
     },
 };
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct PortOffset(u16);
-
-impl PortOffset {
-    pub fn from_chain_id(chain_id: u16) -> Self {
-        Self((chain_id - 1) * 100)
-    }
-}
-
-impl FromStr for PortOffset {
-    type Err = String;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        s.parse::<u16>()
-            .map(PortOffset)
-            .map_err(|_| "Invalid port offset".to_string())
-    }
-}
-
-impl From<PortOffset> for u16 {
-    fn from(port_offset: PortOffset) -> Self {
-        port_offset.0
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, Parser)]
 pub struct InitArgs {
     /// All ethereum environment related arguments
@@ -55,8 +28,8 @@ pub struct InitArgs {
     pub deploy_paymaster: Option<bool>,
     #[clap(long, help = MSG_L1_RPC_URL_HELP)]
     pub l1_rpc_url: Option<String>,
-    #[clap(long, help = MSG_PORT_OFFSET_HELP)]
-    pub port_offset: Option<PortOffset>,
+    #[clap(long, help = MSG_NO_PORT_REALLOCATION_HELP, default_missing_value = "true", num_args = 0..=1)]
+    pub no_port_reallocation: Option<bool>,
 }
 
 impl InitArgs {
@@ -81,15 +54,14 @@ impl InitArgs {
                 .ask()
         });
 
+        let no_port_reallocation = self.no_port_reallocation.unwrap_or(false);
+
         InitArgsFinal {
             forge_args: self.forge_args,
             genesis_args: self.genesis_args.fill_values_with_prompt(config),
             deploy_paymaster,
             l1_rpc_url,
-            port_offset: self
-                .port_offset
-                .unwrap_or(PortOffset::from_chain_id(config.id as u16))
-                .into(),
+            no_port_reallocation,
         }
     }
 }
@@ -100,5 +72,5 @@ pub struct InitArgsFinal {
     pub genesis_args: GenesisArgsFinal,
     pub deploy_paymaster: bool,
     pub l1_rpc_url: String,
-    pub port_offset: u16,
+    pub no_port_reallocation: bool,
 }

zk_toolbox/crates/zk_inception/src/commands/chain/init.rs

@@ -1,8 +1,8 @@
-use anyhow::{bail, Context};
+use anyhow::Context;
 use common::{config::global_config, git, logger, spinner::Spinner};
 use config::{
-    copy_configs, ports_config, set_l1_rpc_url, traits::SaveConfigWithBasePath,
-    update_from_chain_config, update_ports, ChainConfig, EcosystemConfig, GeneralConfig,
+    copy_configs, get_consensus_port, set_l1_rpc_url, traits::SaveConfigWithBasePath,
+    update_from_chain_config, ChainConfig, EcosystemConfig,
 };
 use types::BaseToken;
 use xshell::Shell;
@@ -23,11 +23,13 @@ use crate::{
     messages::{
         msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED,
         MSG_CHAIN_NOT_FOUND_ERR, MSG_DEPLOYING_PAYMASTER, MSG_GENESIS_DATABASE_ERR,
-        MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_PORTS_CONFIG_ERR,
-        MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG,
+        MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG,
         MSG_UPDATING_TOKEN_MULTIPLIER_SETTER_SPINNER, MSG_WALLET_TOKEN_MULTIPLIER_SETTER_NOT_FOUND,
     },
-    utils::consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets},
+    utils::{
+        consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets},
+        ports::EcosystemPortsScanner,
+    },
 };
 
 pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> {
@@ -54,15 +56,24 @@ pub async fn init(
     ecosystem_config: &EcosystemConfig,
     chain_config: &ChainConfig,
 ) -> anyhow::Result<()> {
+    let mut ecosystem_ports = EcosystemPortsScanner::scan(shell)?;
     copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?;
 
     let mut general_config = chain_config.get_general_config()?;
-    apply_port_offset(init_args.port_offset, &mut general_config)?;
-    let ports = ports_config(&general_config).context(MSG_PORTS_CONFIG_ERR)?;
+    if !init_args.no_port_reallocation {
+        ecosystem_ports
+            .allocate_ports_with_offset_from_defaults(&mut general_config, chain_config.id)?;
+    }
+
+    let consensus_port = get_consensus_port(&general_config);
 
     let consensus_keys = generate_consensus_keys();
-    let consensus_config =
-        get_consensus_config(chain_config, ports, Some(consensus_keys.clone()), None)?;
+    let consensus_config = get_consensus_config(
+        chain_config,
+        consensus_port,
+        Some(consensus_keys.clone()),
+        None,
+    )?;
     general_config.consensus_config = Some(consensus_config);
     general_config.save_with_base_path(shell, &chain_config.configs)?;
 
@@ -176,15 +187,3 @@ pub async fn init(
 
     Ok(())
 }
-
-fn apply_port_offset(port_offset: u16, general_config: &mut GeneralConfig) -> anyhow::Result<()> {
-    let Some(mut ports_config) = ports_config(general_config) else {
-        bail!("Missing ports config");
-    };
-
-    ports_config.apply_offset(port_offset);
-
-    update_ports(general_config, &ports_config)?;
-
-    Ok(())
-}

zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/init.rs

@@ -12,7 +12,8 @@ use crate::{
     messages::{
         MSG_DEPLOY_ECOSYSTEM_PROMPT, MSG_DEPLOY_ERC20_PROMPT, MSG_DEPLOY_PAYMASTER_PROMPT,
         MSG_DEV_ARG_HELP, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP, MSG_L1_RPC_URL_INVALID_ERR,
-        MSG_L1_RPC_URL_PROMPT, MSG_OBSERVABILITY_HELP, MSG_OBSERVABILITY_PROMPT,
+        MSG_L1_RPC_URL_PROMPT, MSG_NO_PORT_REALLOCATION_HELP, MSG_OBSERVABILITY_HELP,
+        MSG_OBSERVABILITY_PROMPT,
     },
 };
 
@@ -92,6 +93,8 @@ pub struct EcosystemInitArgs {
     pub dev: bool,
     #[clap(long, short = 'o', help = MSG_OBSERVABILITY_HELP, default_missing_value = "true", num_args = 0..=1)]
     pub observability: Option<bool>,
+    #[clap(long, help = MSG_NO_PORT_REALLOCATION_HELP, default_missing_value = "true", num_args = 0..=1)]
+    pub no_port_reallocation: Option<bool>,
 }
 
 impl EcosystemInitArgs {
@@ -121,6 +124,7 @@ impl EcosystemInitArgs {
                     .ask()
             })
         };
+        let no_port_reallocation = self.no_port_reallocation.unwrap_or(false);
 
         EcosystemInitArgsFinal {
             deploy_paymaster,
@@ -129,6 +133,7 @@ impl EcosystemInitArgs {
             forge_args: self.forge_args.clone(),
             dev: self.dev,
             observability,
+            no_port_reallocation,
         }
     }
 }
@@ -141,4 +146,5 @@ pub struct EcosystemInitArgsFinal {
     pub forge_args: ForgeScriptArgs,
     pub dev: bool,
     pub observability: bool,
+    pub no_port_reallocation: bool,
 }

zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs

@@ -31,7 +31,7 @@ use super::{
 use crate::{
     accept_ownership::{accept_admin, accept_owner},
     commands::{
-        chain::{self, args::init::PortOffset},
+        chain::{self},
         ecosystem::create_configs::{
             create_erc20_deployment_config, create_initial_deployments_config,
         },
@@ -112,7 +112,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> {
             genesis_args: genesis_args.clone().fill_values_with_prompt(&chain_config),
             deploy_paymaster: final_ecosystem_args.deploy_paymaster,
             l1_rpc_url: final_ecosystem_args.ecosystem.l1_rpc_url.clone(),
-            port_offset: PortOffset::from_chain_id(chain_config.id as u16).into(),
+            no_port_reallocation: final_ecosystem_args.no_port_reallocation,
         };
 
         chain::init::init(