Eth_simulate blob related bugfixes (NethermindEth#7308)

Co-authored-by: lukasz.rozmej <[email protected]>

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs

@@ -18,28 +18,22 @@ namespace Nethermind.Consensus.Processing
 {
     public partial class BlockProcessor
     {
-        public class BlockValidationTransactionsExecutor : IBlockProcessor.IBlockTransactionsExecutor
+        public class BlockValidationTransactionsExecutor(
+            ITransactionProcessorAdapter transactionProcessor,
+            IWorldState stateProvider)
+            : IBlockProcessor.IBlockTransactionsExecutor
         {
-            private readonly ITransactionProcessorAdapter _transactionProcessor;
-            private readonly IWorldState _stateProvider;
-
             public BlockValidationTransactionsExecutor(ITransactionProcessor transactionProcessor, IWorldState stateProvider)
                 : this(new ExecuteTransactionProcessorAdapter(transactionProcessor), stateProvider)
             {
             }
 
-            public BlockValidationTransactionsExecutor(ITransactionProcessorAdapter transactionProcessor, IWorldState stateProvider)
-            {
-                _transactionProcessor = transactionProcessor;
-                _stateProvider = stateProvider;
-            }
-
             public event EventHandler<TxProcessedEventArgs>? TransactionProcessed;
 
             public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processingOptions, BlockReceiptsTracer receiptsTracer, IReleaseSpec spec)
             {
                 Metrics.ResetBlockStats();
-                BlockExecutionContext blkCtx = new(block.Header);
+                BlockExecutionContext blkCtx = CreateBlockExecutionContext(block);
                 for (int i = 0; i < block.Transactions.Length; i++)
                 {
                     block.TransactionProcessed = i;
@@ -49,9 +43,11 @@ public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processing
                 return receiptsTracer.TxReceipts.ToArray();
             }
 
+            protected virtual BlockExecutionContext CreateBlockExecutionContext(Block block) => new(block.Header);
+
             protected virtual void ProcessTransaction(in BlockExecutionContext blkCtx, Transaction currentTx, int index, BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions)
             {
-                TransactionResult result = _transactionProcessor.ProcessTransaction(in blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
+                TransactionResult result = transactionProcessor.ProcessTransaction(in blkCtx, currentTx, receiptsTracer, processingOptions, stateProvider);
                 if (!result) ThrowInvalidBlockException(result, blkCtx.Header, currentTx, index);
                 TransactionProcessed?.Invoke(this, new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index]));
             }

src/Nethermind/Nethermind.Core/Crypto/Hash256.cs

@@ -142,6 +142,7 @@ public readonly struct Hash256AsKey(Hash256 key) : IEquatable<Hash256AsKey>
     public sealed class Hash256 : IEquatable<Hash256>, IComparable<Hash256>
     {
         public const int Size = 32;
+        public static readonly Hash256 Zero = new("0x0000000000000000000000000000000000000000000000000000000000000000");
 
         public const int MemorySize =
             MemorySizes.SmallObjectOverhead -

src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs

@@ -121,9 +121,9 @@ public void GetTrace_memory_should_not_bleed_between_txs()
                 0x5b, 0x36, 0x59, 0x3a, 0x34, 0x60, 0x5b, 0x59, 0x05, 0x30, 0xf4, 0x3a,
                 0x56};
 
-            var a = run(second).ToString();
-            run(first);
-            var b = run(second).ToString();
+            var a = Run(second).ToString();
+            Run(first);
+            var b = Run(second).ToString();
 
             Assert.That(b, Is.EqualTo(a));
         }
@@ -134,7 +134,7 @@ public void GetTrace_memory_should_not_overflow()
             var input = new byte[] {
                 0x5b, 0x59, 0x60, 0x20, 0x59, 0x81, 0x91, 0x52, 0x44, 0x36, 0x5a, 0x3b,
                 0x59, 0xf4, 0x5b, 0x31, 0x56, 0x08};
-            run(input);
+            Run(input);
         }
 
         private static readonly PrivateKey PrivateKeyD = new("0000000000000000000000000000000000000000000000000000001000000000");
@@ -144,7 +144,7 @@ public void GetTrace_memory_should_not_overflow()
         private static readonly Address coinbase = new Address("0x4444588443C3a91288c5002483449Aba1054192b");
         // for testing purposes, particular chain id does not matter. Maybe make random id so it captures the idea that signature should would irrespective of chain
         private static readonly EthereumEcdsa ethereumEcdsa = new(BlockchainIds.GenericNonRealNetwork, LimboLogs.Instance);
-        private static string run(byte[] input)
+        private static string Run(byte[] input)
         {
             long blocknr = 12965000;
             long gas = 34218;

src/Nethermind/Nethermind.Evm/BlockExecutionContext.cs

@@ -25,9 +25,11 @@ public BlockExecutionContext(BlockHeader blockHeader)
         }
     }
 
-    public static implicit operator BlockExecutionContext(BlockHeader header)
+    public BlockExecutionContext(BlockHeader blockHeader, UInt256 forceBlobBaseFee)
     {
-        return new BlockExecutionContext(header);
+        Header = blockHeader;
+        BlobBaseFee = forceBlobBaseFee;
     }
 
+    public static implicit operator BlockExecutionContext(BlockHeader header) => new(header);
 }

src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs

@@ -432,6 +432,8 @@ protected ExecutionEnvironment BuildExecutionEnvironment(
             );
         }
 
+        protected virtual bool ShouldValidate(ExecutionOptions opts) => !opts.HasFlag(ExecutionOptions.NoValidation);
+
         protected void ExecuteEvmCall(
             Transaction tx,
             BlockHeader header,
@@ -444,7 +446,7 @@ protected void ExecuteEvmCall(
             out long spentGas,
             out byte statusCode)
         {
-            bool validate = !opts.HasFlag(ExecutionOptions.NoValidation);
+            bool validate = ShouldValidate(opts);
 
             substate = null;
             spentGas = tx.GasLimit;

src/Nethermind/Nethermind.Facade/BlockchainBridge.cs

@@ -169,7 +169,7 @@ public SimulateOutput Simulate(BlockHeader header, SimulatePayload<TransactionWi
             SimulateOutput result = new();
             try
             {
-                if (!_simulateBridgeHelper.TrySimulate(header, payload, new CancellationBlockTracer(tracer, cancellationToken), out string error))
+                if (!_simulateBridgeHelper.TrySimulate(header, payload, simulateOutputTracer, new CancellationBlockTracer(tracer, cancellationToken), out string error))
                 {
                     result.Error = error;
                 }

src/Nethermind/Nethermind.Facade/Eth/BlockForRpc.cs

@@ -84,7 +84,8 @@ public BlockForRpc(Block block, bool includeFullTransactionData, ISpecProvider s
         WithdrawalsRoot = block.Header.WithdrawalsRoot;
     }
 
-    public Address Author { get; set; }
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public Address? Author { get; set; }
     public UInt256 Difficulty { get; set; }
     public byte[] ExtraData { get; set; }
     public long GasLimit { get; set; }

src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockOverride.cs

@@ -18,6 +18,7 @@ public class BlockOverride
     public ulong? GasLimit { get; set; }
     public Address? FeeRecipient { get; set; }
     public UInt256? BaseFeePerGas { get; set; }
+    public UInt256? BlobBaseFee { get; set; }
 
     public BlockHeader GetBlockHeader(BlockHeader parent, IBlocksConfig cfg, IReleaseSpec spec)
     {

src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulateBlockResult.cs

@@ -15,5 +15,4 @@ public class SimulateBlockResult(Block source, bool includeFullTransactionData,
     : BlockForRpc(source, includeFullTransactionData, specProvider)
 {
     public List<SimulateCallResult> Calls { get; set; } = new();
-    public UInt256 BlobBaseFee { get; set; }
 }

src/Nethermind/Nethermind.Facade/Simulate/SimulateBlockTracer.cs

@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Nethermind.Core;
+using Nethermind.Core.Crypto;
 using Nethermind.Core.Specs;
 using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
@@ -46,14 +47,6 @@ public override void EndBlockTrace()
         {
             Calls = _txTracers.Select(t => t.TraceResult).ToList(),
         };
-        if (_currentBlock.Header.ExcessBlobGas is not null)
-        {
-            if (!BlobGasCalculator.TryCalculateBlobGasPricePerUnit(_currentBlock.Header.ExcessBlobGas.Value,
-                    out UInt256 blobGasPricePerUnit))
-            {
-                result.BlobBaseFee = blobGasPricePerUnit;
-            }
-        }
 
         Results.Add(result);
     }

src/Nethermind/Nethermind.Facade/Simulate/SimulateBridgeHelper.cs

@@ -1,11 +1,6 @@
 // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-using System.Runtime.InteropServices;
 using Nethermind.Blockchain;
 using Nethermind.Config;
 using Nethermind.Consensus.Processing;
@@ -14,11 +9,17 @@
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Specs;
 using Nethermind.Crypto;
+using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
-using Nethermind.Evm.TransactionProcessing;
 using Nethermind.Facade.Proxy.Models.Simulate;
 using Nethermind.Int256;
 using Nethermind.State;
+using Nethermind.State.Proofs;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Runtime.InteropServices;
 using Transaction = Nethermind.Core.Transaction;
 
 namespace Nethermind.Facade.Simulate;
@@ -58,14 +59,16 @@ private void PrepareState(BlockHeader blockHeader,
     public bool TrySimulate(
         BlockHeader parent,
         SimulatePayload<TransactionWithSourceDetails> payload,
+        SimulateBlockTracer simulateOutputTracer,
         IBlockTracer tracer,
         [NotNullWhen(false)] out string? error) =>
-        TrySimulate(parent, payload, tracer, simulateProcessingEnvFactory.Create(payload.Validation), out error);
+        TrySimulate(parent, payload, simulateOutputTracer, tracer, simulateProcessingEnvFactory.Create(payload.Validation), out error);
 
 
     private bool TrySimulate(
         BlockHeader parent,
         SimulatePayload<TransactionWithSourceDetails> payload,
+        SimulateBlockTracer simulateOutputTracer,
         IBlockTracer tracer,
         SimulateReadOnlyBlocksProcessingEnv env,
         [NotNullWhen(false)] out string? error)
@@ -83,40 +86,31 @@ private bool TrySimulate(
             foreach (BlockStateCall<TransactionWithSourceDetails> blockCall in payload.BlockStateCalls)
             {
                 nonceCache.Clear();
+
                 BlockHeader callHeader = GetCallHeader(blockCall, parent, payload.Validation, spec); //currentSpec is still parent spec
                 spec = env.SpecProvider.GetSpec(callHeader);
                 PrepareState(callHeader, parent, blockCall, env.WorldState, env.CodeInfoRepository, spec);
-
-                if (blockCall.BlockOverrides is { BaseFeePerGas: not null })
-                {
-                    callHeader.BaseFeePerGas = blockCall.BlockOverrides.BaseFeePerGas.Value;
-                }
-                else if (!payload.Validation)
-                {
-                    callHeader.BaseFeePerGas = 0;
-                }
+                Transaction[] transactions = CreateTransactions(payload, blockCall, callHeader, stateProvider, nonceCache);
+                callHeader.TxRoot = TxTrie.CalculateRoot(transactions);
                 callHeader.Hash = callHeader.CalculateHash();
 
-                Transaction[] transactions = CreateTransactions(payload, blockCall, callHeader, stateProvider, nonceCache);
                 if (!TryGetBlock(payload, env, callHeader, transactions, out Block currentBlock, out error))
                 {
                     return false;
                 }
 
-                ProcessingOptions processingFlags = SimulateProcessingOptions;
-
-                if (!payload.Validation)
-                {
-                    processingFlags |= ProcessingOptions.NoValidation;
-                }
+                ProcessingOptions processingFlags = payload.Validation
+                    ? SimulateProcessingOptions
+                    : SimulateProcessingOptions | ProcessingOptions.NoValidation;
 
                 suggestedBlocks[0] = currentBlock;
 
-                IBlockProcessor processor = env.GetProcessor(payload.Validation);
-                Block processedBlock =
-                        processor.Process(stateProvider.StateRoot, suggestedBlocks, processingFlags, tracer)[0];
+                IBlockProcessor processor = env.GetProcessor(payload.Validation, spec.IsEip4844Enabled ? blockCall.BlockOverrides?.BlobBaseFee : null);
+                Block processedBlock = processor.Process(stateProvider.StateRoot, suggestedBlocks, processingFlags, tracer)[0];
 
                 FinalizeStateAndBlock(stateProvider, processedBlock, spec, currentBlock, blockTree);
+                CheckMisssingAndSetTracedDefaults(simulateOutputTracer, processedBlock);
+
                 parent = processedBlock.Header;
             }
         }
@@ -125,6 +119,18 @@ private bool TrySimulate(
         return true;
     }
 
+    private static void CheckMisssingAndSetTracedDefaults(SimulateBlockTracer simulateOutputTracer, Block processedBlock)
+    {
+        SimulateBlockResult current = simulateOutputTracer.Results.Last();
+        current.StateRoot = processedBlock.StateRoot ?? Hash256.Zero;
+        current.ParentBeaconBlockRoot = processedBlock.ParentBeaconBlockRoot ?? Hash256.Zero;
+        current.TransactionsRoot = processedBlock.Header.TxRoot;
+        current.WithdrawalsRoot = processedBlock.WithdrawalsRoot ?? Keccak.EmptyTreeHash;
+        current.ExcessBlobGas = processedBlock.ExcessBlobGas ?? 0;
+        current.Withdrawals = processedBlock.Withdrawals ?? [];
+        current.Author = null;
+    }
+
     private static void FinalizeStateAndBlock(IWorldState stateProvider, Block processedBlock, IReleaseSpec currentSpec, Block currentBlock, IBlockTree blockTree)
     {
         stateProvider.StateRoot = processedBlock.StateRoot!;
@@ -223,7 +229,6 @@ private Transaction CreateTransaction(TransactionWithSourceDetails transactionDe
     {
         Transaction? transaction = transactionDetails.Transaction;
         transaction.SenderAddress ??= Address.Zero;
-        transaction.To ??= Address.Zero;
         transaction.Data ??= Memory<byte>.Empty;
 
         if (!transactionDetails.HadNonceInRequest)
@@ -261,13 +266,15 @@ private Transaction CreateTransaction(TransactionWithSourceDetails transactionDe
         }
 
         transaction.Hash ??= transaction.CalculateHash();
+        callHeader.BlobGasUsed += BlobGasCalculator.CalculateBlobGas(transaction);
 
         return transaction;
     }
 
-    private BlockHeader GetCallHeader(BlockStateCall<TransactionWithSourceDetails> block, BlockHeader parent, bool payloadValidation, IReleaseSpec parentSpec) =>
-        block.BlockOverrides is not null
-            ? block.BlockOverrides.GetBlockHeader(parent, blocksConfig, parentSpec)
+    private BlockHeader GetCallHeader(BlockStateCall<TransactionWithSourceDetails> block, BlockHeader parent, bool payloadValidation, IReleaseSpec spec)
+    {
+        BlockHeader result = block.BlockOverrides is not null
+            ? block.BlockOverrides.GetBlockHeader(parent, blocksConfig, spec)
             : new BlockHeader(
                 parent.Hash!,
                 Keccak.OfAnEmptySequenceRlp,
@@ -278,8 +285,18 @@ block.BlockOverrides is not null
                 parent.Timestamp + 1,
                 Array.Empty<byte>())
             {
-                BaseFeePerGas = !payloadValidation ? 0 : BaseFeeCalculator.Calculate(parent, parentSpec),
                 MixHash = parent.MixHash,
-                IsPostMerge = parent.Difficulty == 0
+                IsPostMerge = parent.Difficulty == 0,
             };
+        result.Timestamp = parent.Timestamp + 1;
+        result.BaseFeePerGas = block.BlockOverrides is { BaseFeePerGas: not null }
+            ? block.BlockOverrides.BaseFeePerGas.Value
+            : !payloadValidation
+                ? 0
+                : BaseFeeCalculator.Calculate(parent, spec);
+
+        result.ExcessBlobGas = spec.IsEip4844Enabled ? BlobGasCalculator.CalculateExcessBlobGas(parent, spec) : (ulong?)0;
+
+        return result;
+    }
 }

src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs

@@ -15,26 +15,30 @@
 using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
 using Nethermind.Evm.TransactionProcessing;
+using Nethermind.Int256;
 using Nethermind.Logging;
 using Nethermind.State;
 using static Nethermind.Consensus.Processing.BlockProcessor;
 
 namespace Nethermind.Facade.Simulate;
 
-public class SimulateBlockValidationTransactionsExecutor : BlockValidationTransactionsExecutor
+public class SimulateBlockValidationTransactionsExecutor(
+    ITransactionProcessor transactionProcessor,
+    IWorldState stateProvider,
+    bool validate,
+    UInt256? blobBaseFeeOverride)
+    : BlockValidationTransactionsExecutor(transactionProcessor, stateProvider)
 {
-    public SimulateBlockValidationTransactionsExecutor(ITransactionProcessor transactionProcessor, IWorldState stateProvider) : base(transactionProcessor, stateProvider)
-    {
-    }
+    protected override BlockExecutionContext CreateBlockExecutionContext(Block block) =>
+        blobBaseFeeOverride is not null ? new BlockExecutionContext(block.Header, blobBaseFeeOverride.Value) : base.CreateBlockExecutionContext(block);
 
-    public SimulateBlockValidationTransactionsExecutor(ITransactionProcessorAdapter transactionProcessor, IWorldState stateProvider) : base(transactionProcessor, stateProvider)
+    protected override void ProcessTransaction(in BlockExecutionContext blkCtx, Transaction currentTx, int index, BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions)
     {
-    }
+        if (!validate)
+        {
+            processingOptions |= ProcessingOptions.ForceProcessing | ProcessingOptions.DoNotVerifyNonce | ProcessingOptions.NoValidation;
+        }
 
-    protected override void ProcessTransaction(in BlockExecutionContext blkCtx, Transaction currentTx, int index,
-        BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions)
-    {
-        processingOptions |= ProcessingOptions.ForceProcessing | ProcessingOptions.DoNotVerifyNonce | ProcessingOptions.NoValidation;
         base.ProcessTransaction(in blkCtx, currentTx, index, receiptsTracer, processingOptions);
     }
 }
@@ -102,13 +106,11 @@ private SimulateBlockValidatorProxy CreateValidator()
         return new SimulateBlockValidatorProxy(blockValidator);
     }
 
-    public IBlockProcessor GetProcessor(bool validate) =>
+    public IBlockProcessor GetProcessor(bool validate, UInt256? blobBaseFeeOverride) =>
         new BlockProcessor(SpecProvider,
             _blockValidator,
             NoBlockRewards.Instance,
-            validate
-                ? new BlockValidationTransactionsExecutor(_transactionProcessor, StateProvider)
-                : new SimulateBlockValidationTransactionsExecutor(_transactionProcessor, StateProvider),
+            new SimulateBlockValidationTransactionsExecutor(_transactionProcessor, StateProvider, validate, blobBaseFeeOverride),
             StateProvider,
             NullReceiptStorage.Instance,
             new BlockhashStore(SpecProvider, StateProvider),