Merge branch 'main' into bench-analysis

rpc-state-reader/src/reader.rs

@@ -1,4 +1,10 @@
-use std::{env, fmt, num::NonZeroU128, sync::Arc, time::Instant};
+use std::{
+    env, fmt,
+    num::NonZeroU128,
+    sync::Arc,
+    thread,
+    time::{Duration, Instant},
+};
 
 use blockifier::{
     blockifier::block::{BlockInfo, GasPrices},
@@ -9,17 +15,22 @@ use blockifier::{
 };
 use cairo_lang_utils::bigint::BigUintAsHex;
 use cairo_vm::types::program::Program;
+use serde::Serialize;
+use serde_json::Value;
 use starknet::core::types::ContractClass as SNContractClass;
 use starknet_api::{
     block::{BlockNumber, GasPrice},
-    core::{ChainId, ClassHash, CompiledClassHash, ContractAddress},
+    core::{ChainId, ClassHash, CompiledClassHash, ContractAddress, Nonce},
     state::StorageKey,
     transaction::{Transaction, TransactionHash},
 };
 use starknet_gateway::{
     config::RpcStateReaderConfig,
-    errors::serde_err_to_state_err,
-    rpc_objects::{BlockHeader, GetBlockWithTxHashesParams},
+    errors::{serde_err_to_state_err, RPCStateReaderError, RPCStateReaderResult},
+    rpc_objects::{
+        BlockHeader, GetBlockWithTxHashesParams, GetClassHashAtParams, GetNonceParams,
+        GetStorageAtParams,
+    },
     rpc_state_reader::RpcStateReader as GatewayRpcStateReader,
 };
 use tracing::{info, info_span};
@@ -58,6 +69,13 @@ impl From<RpcChain> for ChainId {
     }
 }
 
+const MAX_RETRIES: u32 = 10;
+const RETRY_SLEEP_MS: u64 = 10000;
+
+// The following structure is heavily inspired by the underlying starkware-libs/sequencer implementation.
+// It uses sequencer's RpcStateReader under the hood in some situations, while in other situation
+// the actual implementation has been copied and modified to our needs.
+
 pub struct RpcStateReader {
     chain: RpcChain,
     inner: GatewayRpcStateReader,
@@ -82,13 +100,21 @@ impl RpcStateReader {
         }
     }
 
+    pub fn send_rpc_request_with_retry(
+        &self,
+        method: &str,
+        params: impl Serialize,
+    ) -> RPCStateReaderResult<Value> {
+        retry(|| self.inner.send_rpc_request(method, &params))
+    }
+
     pub fn get_contract_class(&self, class_hash: &ClassHash) -> StateResult<SNContractClass> {
         let params = json!({
             "block_id": self.inner.block_id,
             "class_hash": class_hash.to_string(),
         });
 
-        serde_json::from_value(self.inner.send_rpc_request("starknet_getClass", params)?)
+        serde_json::from_value(self.send_rpc_request_with_retry("starknet_getClass", params)?)
             .map_err(serde_err_to_state_err)
     }
 
@@ -103,18 +129,15 @@ impl RpcStateReader {
         let params = json!([hash]);
 
         serde_json::from_value(
-            self.inner
-                .send_rpc_request("starknet_traceTransaction", params)?,
+            self.send_rpc_request_with_retry("starknet_traceTransaction", params)?,
         )
         .map_err(serde_err_to_state_err)
     }
 
     pub fn get_transaction(&self, hash: &TransactionHash) -> StateResult<Transaction> {
         let params = json!([hash]);
 
-        let tx = self
-            .inner
-            .send_rpc_request("starknet_getTransactionByHash", params)?;
+        let tx = self.send_rpc_request_with_retry("starknet_getTransactionByHash", params)?;
 
         objects::deser::transaction_from_json(tx).map_err(serde_err_to_state_err)
     }
@@ -131,8 +154,7 @@ impl RpcStateReader {
         };
 
         let header: BlockHeader = serde_json::from_value(
-            self.inner
-                .send_rpc_request("starknet_getBlockWithTxHashes", params)?,
+            self.send_rpc_request_with_retry("starknet_getBlockWithTxHashes", params)?,
         )
         .map_err(serde_err_to_state_err)?;
 
@@ -158,8 +180,7 @@ impl RpcStateReader {
         };
 
         serde_json::from_value(
-            self.inner
-                .send_rpc_request("starknet_getBlockWithTxHashes", params)?,
+            self.send_rpc_request_with_retry("starknet_getBlockWithTxHashes", params)?,
         )
         .map_err(serde_err_to_state_err)
     }
@@ -170,8 +191,7 @@ impl RpcStateReader {
         };
 
         serde_json::from_value(
-            self.inner
-                .send_rpc_request("starknet_getBlockWithTxs", params)?,
+            self.send_rpc_request_with_retry("starknet_getBlockWithTxs", params)?,
         )
         .map_err(serde_err_to_state_err)
     }
@@ -183,8 +203,7 @@ impl RpcStateReader {
         let params = json!([hash]);
 
         serde_json::from_value(
-            self.inner
-                .send_rpc_request("starknet_getTransactionReceipt", params)?,
+            self.send_rpc_request_with_retry("starknet_getTransactionReceipt", params)?,
         )
         .map_err(serde_err_to_state_err)
     }
@@ -213,27 +232,60 @@ impl StateReader for RpcStateReader {
         contract_address: ContractAddress,
         key: StorageKey,
     ) -> StateResult<cairo_vm::Felt252> {
-        Ok(self
-            .inner
-            .get_storage_at(contract_address, key)
-            .unwrap_or_default())
+        let get_storage_at_params = GetStorageAtParams {
+            block_id: self.inner.block_id,
+            contract_address,
+            key,
+        };
+
+        let result =
+            self.send_rpc_request_with_retry("starknet_getStorageAt", &get_storage_at_params);
+        match result {
+            Ok(value) => Ok(serde_json::from_value(value).map_err(serde_err_to_state_err)?),
+            Err(RPCStateReaderError::ContractAddressNotFound(_)) => {
+                Ok(cairo_vm::Felt252::default())
+            }
+            Err(e) => Err(e)?,
+        }
     }
 
     fn get_nonce_at(
         &self,
         contract_address: ContractAddress,
     ) -> StateResult<starknet_api::core::Nonce> {
-        Ok(self
-            .inner
-            .get_nonce_at(contract_address)
-            .unwrap_or_default())
+        let get_nonce_params = GetNonceParams {
+            block_id: self.inner.block_id,
+            contract_address,
+        };
+
+        let result = self.send_rpc_request_with_retry("starknet_getNonce", get_nonce_params);
+        match result {
+            Ok(value) => {
+                let nonce: Nonce = serde_json::from_value(value).map_err(serde_err_to_state_err)?;
+                Ok(nonce)
+            }
+            Err(RPCStateReaderError::ContractAddressNotFound(_)) => Ok(Nonce::default()),
+            Err(e) => Err(e)?,
+        }
     }
 
     fn get_class_hash_at(&self, contract_address: ContractAddress) -> StateResult<ClassHash> {
-        Ok(self
-            .inner
-            .get_class_hash_at(contract_address)
-            .unwrap_or_default())
+        let get_class_hash_at_params = GetClassHashAtParams {
+            contract_address,
+            block_id: self.inner.block_id,
+        };
+
+        let result =
+            self.send_rpc_request_with_retry("starknet_getClassHashAt", get_class_hash_at_params);
+        match result {
+            Ok(value) => {
+                let class_hash: ClassHash =
+                    serde_json::from_value(value).map_err(serde_err_to_state_err)?;
+                Ok(class_hash)
+            }
+            Err(RPCStateReaderError::ContractAddressNotFound(_)) => Ok(ClassHash::default()),
+            Err(e) => Err(e)?,
+        }
     }
 
     fn get_compiled_contract_class(&self, class_hash: ClassHash) -> StateResult<ContractClass> {
@@ -314,6 +366,30 @@ fn compile_legacy_cc(
     ContractClass::V0(ContractClassV0(inner))
 }
 
+/// Retries the closure `MAX_RETRIES` times on RPC errors,
+/// waiting RETRY_SLEEP_MS after each retry
+fn retry(f: impl Fn() -> RPCStateReaderResult<Value>) -> RPCStateReaderResult<Value> {
+    let mut attempt = 0;
+    loop {
+        let result = f();
+        attempt += 1;
+
+        // only retry on rpc or request error
+        if !matches!(
+            result,
+            Err(RPCStateReaderError::RPCError(_) | RPCStateReaderError::ReqwestError(_))
+        ) {
+            return result;
+        }
+
+        if attempt >= MAX_RETRIES {
+            return result;
+        }
+
+        thread::sleep(Duration::from_millis(RETRY_SLEEP_MS))
+    }
+}
+
 fn bytecode_size(data: &[BigUintAsHex]) -> usize {
     data.iter().map(|n| n.value.to_bytes_be().len()).sum()
 }

scripts/cmp_state_dumps.sh

@@ -1,14 +1,30 @@
 #!/usr/bin/env bash
 
+# Compares state dump files between two directories: 'state_dumps/vm' and 'state_dumps/native'.
+# It iterates over all JSON files in the 'state_dumps/vm' directory and checks if the corresponding
+# file exists in 'state_dumps/native'.
+# If the corresponding file does not exist, it skips the comparison and counts the missing files.
+# For existing pairs, it compares the contents, ignoring the lines containing the "reverted" field, because of error message diference in Native and VM.
+# It counts and displays the number of matching, differing, and missing state dumps.
+
 matching=0
 diffing=0
+missing=0
 
+# Iterate over state_dumps/vm dumps
 for vm_dump in state_dumps/vm/*/*.json; do
   [ -f "$vm_dump" ] || continue
 
   native_dump="${vm_dump//vm/native}"
 
-  base=$(basename "$native_dump")
+  # Check if the corresponding native_dump file exists, if not, skip
+  if [ ! -f "$native_dump" ]; then
+    echo "Missing: $native_dump (file not found)"
+    missing=$((missing+1))
+    continue
+  fi
+
+  base=$(basename "$vm_dump")
 
   if ! cmp -s \
       <(sed '/"reverted": /d' "$native_dump") \
@@ -26,3 +42,4 @@ echo
 echo "Finished comparison"
 echo "- Matching: $matching"
 echo "- Diffing:  $diffing"
+echo "- Missing: $missing"

scripts/delta_state_dumps.sh

@@ -20,8 +20,14 @@ for block in state_dumps/vm/*/; do
   # Compares the files in ascending order, by creation date
   IFS=$'\n'
   for tx_name in $(ls -trU1 $block); do
-    vm_tx="state_dumps/vm/$block_name/$tx_name"
     native_tx="state_dumps/native/$block_name/$tx_name"
+    vm_tx="state_dumps/vm/$block_name/$tx_name"
+
+    # Check if the corresponding native_tx file exists, if not, skip
+    if [ ! -f "$native_tx" ]; then
+      echo "Skipping: $native_tx (file not found)"
+      continue
+    fi
 
     if cmp -s \
       <(sed '/"reverted": /d' "$native_tx") \
@@ -33,7 +39,7 @@ for block in state_dumps/vm/*/; do
     echo "Tx ${tx_name//.*/}"
 
     prompt_continue && {
-      delta "$native_tx" "$vm_tx" --side-by-side --paging always --wrap-max-lines unlimited 
+      delta "$native_tx" "$vm_tx" --side-by-side --paging always --wrap-max-lines unlimited
     }
   done
 done