Updated blob related fields

CHANGELOG.md

@@ -4,6 +4,35 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md)
 for instructions to keep up to date.
 
+## v2.2.2
+
+The Cancun hard fork happened on Goerli and after further review, we decided to change the Protobuf definition for the new `BlockHeader`, `Transaction` and `TransactionReceipt` fields that are related to blob transaction.
+
+We made explicit that those fields are optional in the Protobuf definition which will render them in your language of choice using the appropriate "null" mechanism. For example on Golang, those fields are generated as `BlobGasUsed *uint64` and `ExcessBlobGas *uint64` which will make it clear that those fields are not populated at all.
+
+The affected fields are:
+  - [BlockHeader.blob_gas_used](./proto/sf/ethereum/type/v2/type.proto#L173), now `optional uint64`.
+  - [BlockHeader.excess_blob_gas](./proto/sf/ethereum/type/v2/type.proto#L176), now `optional uint64`.
+  - [TransactionTrace.blob_gas](./proto/sf/ethereum/type/v2/type.proto#L369), now `optional uint64`.
+  - [TransactionTrace.blob_gas_fee_cap](./proto/sf/ethereum/type/v2/type.proto#L377), now `optional BigInt`.
+  - [TransactionReceipt.blob_gas_used](./proto/sf/ethereum/type/v2/type.proto#L428), now `optional uint64`.
+  - [TransactionReceipt.blob_gas_price](./proto/sf/ethereum/type/v2/type.proto#L436), now `optional BigInt`.
+
+This is technically a breaking change for those that could have consumed those fields already but we think he impact is so minimal that it's better to make the change right now.
+
+### Operators
+
+You will need to reprocess a small Goerli range. You should update to new version to produce the newer version and the reprocess from block 10377700 up to when you upgraded to v2.2.2.
+
+The block 10377700 was chosen since it is the block at the time of the first release we did supporting Cancun where we introduced those new field. If you know when you deploy either `v2.2.0` or `v2.2.1`, you should reprocess from that point.
+
+An alternative to reprocessing is updating your blocks by having a [StreamingFast API Token](https://substreams.streamingfast.io/getting-started/quickstart#run-your-first-substreams) and using `fireeth tools download-from-firehose goerli.eth.streamingfast.io:443 -a SUBSTREAMS_API_TOKEN 10377700:<recent block rounded to 100s> <destination>`.
+
+> [!NOTE]
+> You should download the blocks to a temporary destination and copy over to your production destination once you have them all.
+
+You can reach to us on Discord if you need help on something.
+
 ## v2.2.1
 
 * Updated the documentation for some of the upcoming new Cancun hard-fork fields:

codec/console_reader.go

@@ -524,11 +524,11 @@ func (ctx *parseCtx) readApplyTrxBegin(line string) error {
 	trxType := pbeth.TransactionTrace_Type(FromInt32(chunks[13], "BEGIN_APPLY_TRX trxType"))
 	ordinal := FromUint64(chunks[14], "BEGIN_APPLY_TRX ordinal")
 
-	blobDataGasUsed := uint64(0)
+	var blobDataGasUsed *uint64
 	var maxFeePerDataGas *pbeth.BigInt
 	var blobVersionedHashes [][]byte
-	if len(chunks) == 19 {
-		blobDataGasUsed = FromUint64(chunks[16], "BEGIN_APPLY_TRX blobDataGasUsed")
+	if len(chunks) == 19 && trxType == pbeth.TransactionTrace_TRX_TYPE_BLOB {
+		blobDataGasUsed = ptr(FromUint64(chunks[16], "BEGIN_APPLY_TRX blobDataGasUsed"))
 		maxFeePerDataGas = pbeth.BigIntFromBytes(FromHex(chunks[17], "BEGIN_APPLY_TRX maxFeePerDataGas"))
 		blobVersionedHashes = decodeBlobHashes(chunks[18], "BEGIN_APPLY_TRX blobVerifiedHashes")
 	}
@@ -897,14 +897,14 @@ func (ctx *parseCtx) readApplyTrxEnd(line string) error {
 	ordinal := FromUint64(chunks[4], "END_APPLY_TRX ordinal")
 
 	logChunkNum := 5
-	var blobGasUsed uint64
+	var blobGasUsed *uint64
 	var blobGasPrice *pbeth.BigInt
 	if ctx.readBlobGasUsed {
 		logChunkNum = 7
 
 		// Only in the blob trx type we compute those field for the receipt
 		if ctx.currentTrace.Type == pbeth.TransactionTrace_TRX_TYPE_BLOB {
-			blobGasUsed = FromUint64(chunks[5], "END_APPLY_TRX blobGasUsed")
+			blobGasUsed = ptr(FromUint64(chunks[5], "END_APPLY_TRX blobGasUsed"))
 			blobGasPrice = pbeth.BigIntFromBytes(FromHex(chunks[6], "END_APPLY_TRX blogGasPrice"))
 		}
 	}
@@ -1835,3 +1835,5 @@ func SplitInBoundedChunks(line string, count int) ([]string, error) {
 func Has0xPrefix(input string) bool {
 	return len(input) >= 2 && input[0] == '0' && (input[1] == 'x' || input[1] == 'X')
 }
+
+func ptr[T any](v T) *T { return &v }

codec/console_reader_test.go

@@ -61,6 +61,7 @@ func TestParseFromFile(t *testing.T) {
 		{"testdata/polygon_disordered.dmlog", nil, nil, false},
 		{"testdata/polygon_reorder_ordinals.dmlog", nil, nil, false},
 		{"testdata/polygon_validator.dmlogs", nil, nil, false},
+		{"testdata/ethereum_cancun_block_header.dmlog", nil, nil, false},
 		{"testdata/lachesis.dmlog", nil, nil, false},
 	}
 

codec/testdata/ethereum_cancun_block_header.dmlog

@@ -0,0 +1,4 @@
+FIRE INIT 2.0 polygon 4.0.0
+FIRE BEGIN_BLOCK 1
+FIRE FINALIZE_BLOCK 1
+FIRE END_BLOCK 1 995 {"header":{"parentHash":"0xc88b402659514206088d932e24c5e4e8ad384aab8b8d9b151f903b06a69b58a2","sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","miner":"0x0000000000000000000000000000000000000000","stateRoot":"0x91e7407285d3daf15ed29dd054320e366ec60ea2b93d359a9a663a0be3e18dcb","transactionsRoot":"0x3787bdcd430c94ce67bc3822c6d958d8cc59efa7c60222f97bedef68f82e9c89","receiptsRoot":"0xe174f9aa9cfaf537bbd946f4f28d1e070fc266e30b87e47f861e64031a4aa045","logsBloom":"0x00000000000000000000000000000000000000001000000000000000000000000000000000000004000000100000000000008000000000000000000000000000000000000000040000000000000000800000000000000000000100000000000000000000080000000000100000000000000000000000000080000000000000000000000000001000000000000000000000000000000000000000000000000000200000000000000000000000000000000200000000000000000000000000004000000000000000000001000000000200000000000000800000108000000000000010000000000000000000000000000000000000000000000000000000100000","difficulty":"0x7","number":"0x1","gasLimit":"0xec0a04","gasUsed":"0x5208","timestamp":"0x5ed28d9c","extraData":"0xd58301090083626f7286676f312e3133856c696e7578000000000000000000000375b2fc7140977c9c76d45421564e354ed42277000000000000000000000000000000000000271042eefcda06ead475cde3731b8eb138e88cd0bac300000000000000000000000000000000000027105973918275c01f50555d44e92c9d9b353cadad5400000000000000000000000000000000000027107fcd58c2d53d980b247f1612fdba93e9a76193e60000000000000000000000000000000000002710b702f1c9154ac9c08da247a8e30ee6f2f3373f410000000000000000000000000000000000002710b8bb158b93c94ed35c1970d610d1e2b34e26652c0000000000000000000000000000000000002710f84c74dea96df0ec22e11e7c33996c73fcc2d8220000000000000000000000000000000000002710b9dc800a95f8342af918585ff50f43fe85ebe0b42a9ed2d49bcfc3e6c0fbf0b81c66058dbdbca3a14bfabb76092a878d900ee699b80d0becb991022ab1c0365801","mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000","nonce":"0x0000000000000000","hash":"0x937b1a9a6686ca5d845d41cf6264586e4333caa009c9001ce71f174748881c4e", "blobGasUsed": "0x5207", "excessBlobGas": "0x1207", "parentBeaconBlockRoot":"0xd777402659514206088d932e24c5e4e8ad384aab8b8d9b151f903b06a69b58a2"},"uncles":[]}

codec/testdata/ethereum_cancun_block_header.dmlog.golden.json

@@ -0,0 +1,28 @@
+[
+  {
+    "hash": "937b1a9a6686ca5d845d41cf6264586e4333caa009c9001ce71f174748881c4e",
+    "header": {
+      "blobGasUsed": "20999",
+      "coinbase": "0000000000000000000000000000000000000000",
+      "difficulty": "07",
+      "excessBlobGas": "4615",
+      "extraData": "d58301090083626f7286676f312e3133856c696e7578000000000000000000000375b2fc7140977c9c76d45421564e354ed42277000000000000000000000000000000000000271042eefcda06ead475cde3731b8eb138e88cd0bac300000000000000000000000000000000000027105973918275c01f50555d44e92c9d9b353cadad5400000000000000000000000000000000000027107fcd58c2d53d980b247f1612fdba93e9a76193e60000000000000000000000000000000000002710b702f1c9154ac9c08da247a8e30ee6f2f3373f410000000000000000000000000000000000002710b8bb158b93c94ed35c1970d610d1e2b34e26652c0000000000000000000000000000000000002710f84c74dea96df0ec22e11e7c33996c73fcc2d8220000000000000000000000000000000000002710b9dc800a95f8342af918585ff50f43fe85ebe0b42a9ed2d49bcfc3e6c0fbf0b81c66058dbdbca3a14bfabb76092a878d900ee699b80d0becb991022ab1c0365801",
+      "gasLimit": "15469060",
+      "gasUsed": "21000",
+      "hash": "937b1a9a6686ca5d845d41cf6264586e4333caa009c9001ce71f174748881c4e",
+      "logsBloom": "00000000000000000000000000000000000000001000000000000000000000000000000000000004000000100000000000008000000000000000000000000000000000000000040000000000000000800000000000000000000100000000000000000000080000000000100000000000000000000000000080000000000000000000000000001000000000000000000000000000000000000000000000000000200000000000000000000000000000000200000000000000000000000000004000000000000000000001000000000200000000000000800000108000000000000010000000000000000000000000000000000000000000000000000000100000",
+      "mixHash": "0000000000000000000000000000000000000000000000000000000000000000",
+      "number": "1",
+      "parentBeaconRoot": "d777402659514206088d932e24c5e4e8ad384aab8b8d9b151f903b06a69b58a2",
+      "parentHash": "c88b402659514206088d932e24c5e4e8ad384aab8b8d9b151f903b06a69b58a2",
+      "receiptRoot": "e174f9aa9cfaf537bbd946f4f28d1e070fc266e30b87e47f861e64031a4aa045",
+      "stateRoot": "91e7407285d3daf15ed29dd054320e366ec60ea2b93d359a9a663a0be3e18dcb",
+      "timestamp": "2020-05-30T16:45:16Z",
+      "transactionsRoot": "3787bdcd430c94ce67bc3822c6d958d8cc59efa7c60222f97bedef68f82e9c89",
+      "uncleHash": "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
+    },
+    "number": "1",
+    "size": "995",
+    "ver": 3
+  }
+]

codec/testdata/read_transaction_blob.dmlog.golden.json

@@ -46,6 +46,7 @@
     "r": "9b7b4ad21ac016e1766749e203d4776d351ac9c7d56cdd9b3c2d5b4f940c1f7e",
     "receipt": {
       "blobGasPrice": "00",
+      "blobGasUsed": "0",
       "cumulativeGasUsed": "23358",
       "logsBloom": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
     },

codec/testdata/read_transaction_blob_no_hashes.dmlog.golden.json

@@ -42,6 +42,7 @@
     "r": "9b7b4ad21ac016e1766749e203d4776d351ac9c7d56cdd9b3c2d5b4f940c1f7e",
     "receipt": {
       "blobGasPrice": "00",
+      "blobGasUsed": "0",
       "cumulativeGasUsed": "23358",
       "logsBloom": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
     },

codec/types.go

@@ -37,8 +37,8 @@ type BlockHeader struct {
 	Nonce            eth.Uint64     `json:"nonce"`
 	BaseFeePerGas    eth.Hex        `json:"baseFeePerGas"`
 	WithdrawalsHash  eth.Hash       `json:"withdrawalsRoot"`
-	BlobGasUsed      eth.Uint64     `json:"blobGasUsed"`
-	ExcessBlobGas    eth.Uint64     `json:"excessBlobGas"`
+	BlobGasUsed      *eth.Uint64    `json:"blobGasUsed"`
+	ExcessBlobGas    *eth.Uint64    `json:"excessBlobGas"`
 	ParentBeaconRoot eth.Hash       `json:"parentBeaconBlockRoot"`
 	TxDependency     [][]eth.Uint64 `json:"txDependency"`
 }

codec/utils.go

@@ -137,9 +137,8 @@ func FromHeader(header *BlockHeader) *pbeth.BlockHeader {
 		BaseFeePerGas:    pbeth.BigIntFromBytes(header.BaseFeePerGas),
 		WithdrawalsRoot:  header.WithdrawalsHash,
 		TxDependency:     pbeth.Uint64NestedArrayFromEthUint(header.TxDependency),
-
-		BlobGasUsed:      uint64(header.BlobGasUsed),
-		ExcessBlobGas:    uint64(header.ExcessBlobGas),
+		BlobGasUsed:      (*uint64)(header.BlobGasUsed),
+		ExcessBlobGas:    (*uint64)(header.ExcessBlobGas),
 		ParentBeaconRoot: header.ParentBeaconRoot,
 	}
 }

proto/sf/ethereum/type/v2/type.proto

@@ -170,10 +170,10 @@ message BlockHeader {
   Uint64NestedArray tx_dependency = 20;
 
 	// BlobGasUsed was added by EIP-4844 and is ignored in legacy headers.
-	uint64 blob_gas_used = 22;
+	optional uint64 blob_gas_used = 22;
 
 	// ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers.
-	uint64 excess_blob_gas = 23;
+	optional uint64 excess_blob_gas = 23;
 
 	// ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers.
 	bytes parent_beacon_root = 24;
@@ -366,15 +366,15 @@ message TransactionTrace {
   //
   // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only
   // if Cancun fork is active on the chain.
-  uint64 blob_gas = 33;
+  optional uint64 blob_gas = 33;
 
   // BlobGasFeeCap is the maximum fee per data gas the user is willing to pay for the data gas used.
   //
   // This is specified by https://eips.ethereum.org/EIPS/eip-4844
   //
   // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only
   // if Cancun fork is active on the chain.
-  BigInt blob_gas_fee_cap = 34;
+  optional BigInt blob_gas_fee_cap = 34;
 
   // BlobHashes field represents a list of hash outputs from 'kzg_to_versioned_hash' which
   // essentially is a version byte + the sha256 hash of the blob commitment (e.g.
@@ -425,15 +425,15 @@ message TransactionReceipt {
   //
   // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only
   // if Cancun fork is active on the chain.
-  uint64 blob_gas_used = 5;
+  optional uint64 blob_gas_used = 5;
 
   // BlobGasPrice is the amount to pay per blob item in the transaction.
   //
   // This is specified by https://eips.ethereum.org/EIPS/eip-4844
   //
   // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only
   // if Cancun fork is active on the chain.
-  BigInt blob_gas_price = 6;
+  optional BigInt blob_gas_price = 6;
 }
 
 message Log {

types/pb/last_generate.txt

@@ -1,3 +1,3 @@
-generate.sh - Tue Jan 16 16:25:06 EST 2024 - maoueh
+generate.sh - Wed Jan 17 11:49:31 EST 2024 - maoueh
 streamingfast/proto revision: 43ccc05e2374e18dee0ce81f8634f078770fe94e
-streamingfast/firehose-ethereum/proto revision: 345b349
+streamingfast/firehose-ethereum/proto revision: ab7428a