Update from upstream

.github/workflows/ci-core-reusable.yml

@@ -356,10 +356,17 @@ jobs:
         run: |
           ci_run zkstack dev test build
 
+      - name: Build tested binaries
+        run: |
+          ci_run zkstack server build
+          ci_run zkstack external-node build
+          ci_run zkstack contract-verifier build
+
       - name: Initialize Contract verifier
         run: |
           ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
           ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
+          ci_run zkstack contract-verifier wait --chain era --verbose
 
       - name: Run servers
         run: |
@@ -374,10 +381,14 @@ jobs:
             --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \
             &> ${{ env.SERVER_LOGS_DIR }}/consensus.log &
 
-          ci_run sleep 5
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain era
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain validium
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain custom_token
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain consensus
 
-      - name: Setup attester committee for the consensus chain
+      - name: Set up attester committee for the consensus chain
         run: |
+          ci_run zkstack consensus wait-for-registry --ignore-prerequisites --verbose --chain consensus
           ci_run zkstack consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 
 
       - name: Run integration tests
@@ -416,8 +427,11 @@ jobs:
           ci_run zkstack external-node run --ignore-prerequisites --chain validium &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/validium.log &
           ci_run zkstack external-node run --ignore-prerequisites --chain custom_token &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/custom_token.log &
           ci_run zkstack external-node run --ignore-prerequisites --chain consensus --enable-consensus &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/consensus.log &
-          # Wait for nodes to start
-          ci_run sleep 5
+          
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain era
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain validium
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain custom_token
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain consensus
 
       - name: Run integration tests en
         run: |

.github/workflows/release-test-stage.yml

@@ -61,11 +61,10 @@ jobs:
   build-push-core-images:
     name: Build and push images
     needs: [setup, changed_files]
-    uses: ./.github/workflows/new-build-core-template.yml
+    uses: ./.github/workflows/build-core-template.yml
     if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true'
     with:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
-      action: "push"
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -85,40 +84,37 @@ jobs:
   build-push-contract-verifier:
     name: Build and push images
     needs: [setup, changed_files]
-    uses: ./.github/workflows/new-build-contract-verifier-template.yml
+    uses: ./.github/workflows/build-contract-verifier-template.yml
     if: needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true'
     with:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
-      action: "push"
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   build-push-prover-images:
     name: Build and push images
     needs: [setup, changed_files]
-    uses: ./.github/workflows/new-build-prover-template.yml
+    uses: ./.github/workflows/build-prover-template.yml
     if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true'
     with:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
       ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: "60;70;75;80;89"
-      action: "push"
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   build-push-witness-generator-image-avx512:
     name: Build and push prover images with avx512 instructions
     needs: [setup, changed_files]
-    uses: ./.github/workflows/new-build-witness-generator-template.yml
+    uses: ./.github/workflows/build-witness-generator-template.yml
     if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true'
     with:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
       ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: "60;70;75;80;89"
       WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl "
-      action: push
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

core/bin/external_node/src/config/mod.rs

@@ -408,6 +408,9 @@ pub(crate) struct OptionalENConfig {
     /// Timeout to wait for the Merkle tree database to run compaction on stalled writes.
     #[serde(default = "OptionalENConfig::default_merkle_tree_stalled_writes_timeout_sec")]
     merkle_tree_stalled_writes_timeout_sec: u64,
+    /// Enables the stale keys repair task for the Merkle tree.
+    #[serde(default)]
+    pub merkle_tree_repair_stale_keys: bool,
 
     // Postgres config (new parameters)
     /// Threshold in milliseconds for the DB connection lifetime to denote it as long-living and log its details.
@@ -639,6 +642,12 @@ impl OptionalENConfig {
                 merkle_tree.stalled_writes_timeout_sec,
                 default_merkle_tree_stalled_writes_timeout_sec
             ),
+            merkle_tree_repair_stale_keys: general_config
+                .db_config
+                .as_ref()
+                .map_or(false, |config| {
+                    config.experimental.merkle_tree_repair_stale_keys
+                }),
             database_long_connection_threshold_ms: load_config!(
                 general_config.postgres_config,
                 long_connection_threshold_ms

core/bin/external_node/src/node_builder.rs

@@ -378,6 +378,11 @@ impl ExternalNodeBuilder {
             layer = layer.with_tree_api_config(merkle_tree_api_config);
         }
 
+        // Add stale keys repair task if requested.
+        if self.config.optional.merkle_tree_repair_stale_keys {
+            layer = layer.with_stale_keys_repair();
+        }
+
         // Add tree pruning if needed.
         if self.config.optional.pruning_enabled {
             layer = layer.with_pruning_config(self.config.optional.pruning_removal_delay());

core/lib/config/src/configs/experimental.rs

@@ -29,6 +29,9 @@ pub struct ExperimentalDBConfig {
     /// correspondingly; otherwise, RocksDB performance can significantly degrade.
     #[serde(default)]
     pub include_indices_and_filters_in_block_cache: bool,
+    /// Enables the stale keys repair task for the Merkle tree.
+    #[serde(default)]
+    pub merkle_tree_repair_stale_keys: bool,
 }
 
 impl Default for ExperimentalDBConfig {
@@ -40,6 +43,7 @@ impl Default for ExperimentalDBConfig {
             protective_reads_persistence_enabled: false,
             processing_delay_ms: Self::default_merkle_tree_processing_delay_ms(),
             include_indices_and_filters_in_block_cache: false,
+            merkle_tree_repair_stale_keys: false,
         }
     }
 }

core/lib/config/src/testonly.rs

@@ -305,6 +305,7 @@ impl Distribution<configs::ExperimentalDBConfig> for EncodeDist {
             protective_reads_persistence_enabled: self.sample(rng),
             processing_delay_ms: self.sample(rng),
             include_indices_and_filters_in_block_cache: self.sample(rng),
+            merkle_tree_repair_stale_keys: self.sample(rng),
         }
     }
 }

core/lib/contract_verifier/Cargo.toml

@@ -34,4 +34,6 @@ semver.workspace = true
 [dev-dependencies]
 zksync_node_test_utils.workspace = true
 zksync_vm_interface.workspace = true
+
+assert_matches.workspace = true
 test-casing.workspace = true

core/lib/contract_verifier/src/compilers/mod.rs

@@ -1,25 +1,69 @@
+use std::collections::HashMap;
+
 use anyhow::Context as _;
 use serde::{Deserialize, Serialize};
 use zksync_types::contract_verification_api::CompilationArtifacts;
 
 pub(crate) use self::{
     solc::{Solc, SolcInput},
+    vyper::{Vyper, VyperInput},
     zksolc::{ZkSolc, ZkSolcInput},
-    zkvyper::{ZkVyper, ZkVyperInput},
+    zkvyper::ZkVyper,
 };
 use crate::error::ContractVerifierError;
 
 mod solc;
+mod vyper;
 mod zksolc;
 mod zkvyper;
 
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct StandardJson {
+    pub language: String,
+    pub sources: HashMap<String, Source>,
+    #[serde(default)]
+    settings: Settings,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct Settings {
+    /// The output selection filters.
+    output_selection: Option<serde_json::Value>,
+    /// Other settings (only filled when parsing `StandardJson` input from the request).
+    #[serde(flatten)]
+    other: serde_json::Value,
+}
+
+impl Default for Settings {
+    fn default() -> Self {
+        Self {
+            output_selection: None,
+            other: serde_json::json!({}),
+        }
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Source {
     /// The source code file content.
     pub content: String,
 }
 
+/// Users may provide either just contract name or source file name and contract name joined with ":".
+fn process_contract_name(original_name: &str, extension: &str) -> (String, String) {
+    if let Some((file_name, contract_name)) = original_name.rsplit_once(':') {
+        (file_name.to_owned(), contract_name.to_owned())
+    } else {
+        (
+            format!("{original_name}.{extension}"),
+            original_name.to_owned(),
+        )
+    }
+}
+
 /// Parsing logic shared between `solc` and `zksolc`.
 fn parse_standard_json_output(
     output: &serde_json::Value,
@@ -31,11 +75,16 @@ fn parse_standard_json_output(
         let errors = errors.as_array().unwrap().clone();
         if errors
             .iter()
-            .any(|err| err["severity"].as_str().unwrap() == "error")
+            .any(|err| err["severity"].as_str() == Some("error"))
         {
             let error_messages = errors
                 .into_iter()
-                .map(|err| err["formattedMessage"].clone())
+                .filter_map(|err| {
+                    // `formattedMessage` is an optional field
+                    err.get("formattedMessage")
+                        .or_else(|| err.get("message"))
+                        .cloned()
+                })
                 .collect();
             return Err(ContractVerifierError::CompilationError(
                 serde_json::Value::Array(error_messages),
@@ -50,28 +99,35 @@ fn parse_standard_json_output(
         return Err(ContractVerifierError::MissingContract(contract_name));
     };
 
-    let Some(bytecode_str) = contract
-        .pointer("/evm/bytecode/object")
-        .context("missing bytecode in solc / zksolc output")?
-        .as_str()
-    else {
+    let Some(bytecode_str) = contract.pointer("/evm/bytecode/object") else {
         return Err(ContractVerifierError::AbstractContract(contract_name));
     };
+    let bytecode_str = bytecode_str
+        .as_str()
+        .context("unexpected `/evm/bytecode/object` value")?;
+    // Strip an optional `0x` prefix (output by `vyper`, but not by `solc` / `zksolc`)
+    let bytecode_str = bytecode_str.strip_prefix("0x").unwrap_or(bytecode_str);
     let bytecode = hex::decode(bytecode_str).context("invalid bytecode")?;
 
     let deployed_bytecode = if get_deployed_bytecode {
-        let bytecode_str = contract
-            .pointer("/evm/deployedBytecode/object")
-            .context("missing deployed bytecode in solc output")?
+        let Some(bytecode_str) = contract.pointer("/evm/deployedBytecode/object") else {
+            return Err(ContractVerifierError::AbstractContract(contract_name));
+        };
+        let bytecode_str = bytecode_str
             .as_str()
-            .ok_or(ContractVerifierError::AbstractContract(contract_name))?;
+            .context("unexpected `/evm/deployedBytecode/object` value")?;
+        let bytecode_str = bytecode_str.strip_prefix("0x").unwrap_or(bytecode_str);
         Some(hex::decode(bytecode_str).context("invalid deployed bytecode")?)
     } else {
         None
     };
 
-    let abi = contract["abi"].clone();
-    if !abi.is_array() {
+    let mut abi = contract["abi"].clone();
+    if abi.is_null() {
+        // ABI is undefined for Yul contracts when compiled with standalone `solc`. For uniformity with `zksolc`,
+        // replace it with an empty array.
+        abi = serde_json::json!([]);
+    } else if !abi.is_array() {
         let err = anyhow::anyhow!(
             "unexpected value for ABI: {}",
             serde_json::to_string_pretty(&abi).unwrap()

core/lib/contract_verifier/src/compilers/solc.rs

@@ -1,14 +1,13 @@
 use std::{collections::HashMap, path::PathBuf, process::Stdio};
 
 use anyhow::Context;
-use serde::{Deserialize, Serialize};
 use tokio::io::AsyncWriteExt;
 use zksync_queued_job_processor::async_trait;
 use zksync_types::contract_verification_api::{
     CompilationArtifacts, SourceCodeData, VerificationIncomingRequest,
 };
 
-use super::{parse_standard_json_output, Source};
+use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson};
 use crate::{error::ContractVerifierError, resolver::Compiler};
 
 // Here and below, fields are public for testing purposes.
@@ -19,24 +18,6 @@ pub(crate) struct SolcInput {
     pub file_name: String,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct StandardJson {
-    pub language: String,
-    pub sources: HashMap<String, Source>,
-    settings: Settings,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct Settings {
-    /// The output selection filters.
-    output_selection: Option<serde_json::Value>,
-    /// Other settings (only filled when parsing `StandardJson` input from the request).
-    #[serde(flatten)]
-    other: serde_json::Value,
-}
-
 #[derive(Debug)]
 pub(crate) struct Solc {
     path: PathBuf,
@@ -50,17 +31,7 @@ impl Solc {
     pub fn build_input(
         req: VerificationIncomingRequest,
     ) -> Result<SolcInput, ContractVerifierError> {
-        // Users may provide either just contract name or
-        // source file name and contract name joined with ":".
-        let (file_name, contract_name) =
-            if let Some((file_name, contract_name)) = req.contract_name.rsplit_once(':') {
-                (file_name.to_string(), contract_name.to_string())
-            } else {
-                (
-                    format!("{}.sol", req.contract_name),
-                    req.contract_name.clone(),
-                )
-            };
+        let (file_name, contract_name) = process_contract_name(&req.contract_name, "sol");
         let default_output_selection = serde_json::json!({
             "*": {
                 "*": [ "abi", "evm.bytecode", "evm.deployedBytecode" ],