Add prev_headers to EVM state (#913)

* Remove recent uncles from state

* Add prev_headers to EVM state

* Fix block hash getter in vm

* Fix pre_headers set in shard_state

* Fix tests

* Fix prev headers update and add test case

* Minor

quarkchain/cluster/shard_state.py

@@ -317,7 +317,10 @@ def init_from_root_block(self, root_block):
         self.confirmed_header_tip = confirmed_header_tip
         sender_disallow_map = self._get_sender_disallow_map(header_tip)
         self.evm_state = self.__create_evm_state(
-            self.meta_tip.hash_evm_state_root, sender_disallow_map, int(time.time())
+            self.meta_tip.hash_evm_state_root,
+            sender_disallow_map,
+            int(time.time()),
+            self.header_tip.get_hash(),
         )
         check(
             self.db.get_minor_block_evm_root_hash_by_hash(header_tip_hash)
@@ -333,6 +336,7 @@ def __create_evm_state(
         trie_root_hash: Optional[bytes],
         sender_disallow_map: Dict[bytes, int],
         timestamp: Optional[int] = None,
+        block_hash: Optional[bytes] = None,
     ):
         """EVM state with given root hash and block hash AFTER which being evaluated."""
         state = EvmState(
@@ -344,6 +348,14 @@ def __create_evm_state(
         state.sender_disallow_map = sender_disallow_map
         if timestamp:
             state.timestamp = timestamp
+        # iterate until reaches genesis or header list reaches 256
+        # since most headers are in LRU cache, this should not affect performance too much
+        while block_hash and len(state.prev_headers) < 256:
+            h = self.db.get_minor_block_header_by_hash(block_hash)
+            if not h:
+                break
+            state.prev_headers.append(h)
+            block_hash = h.hash_prev_minor_block
         return state
 
     def init_genesis_state(self, root_block):
@@ -382,7 +394,9 @@ def init_genesis_state(self, root_block):
         self.header_tip = genesis_block.header
         self.meta_tip = genesis_block.meta
         self.evm_state = self.__create_evm_state(
-            genesis_block.meta.hash_evm_state_root, sender_disallow_map={}
+            genesis_block.meta.hash_evm_state_root,
+            sender_disallow_map={},
+            block_hash=genesis_block.header.get_hash(),
         )
 
         Logger.info(
@@ -569,21 +583,15 @@ def _get_evm_state_for_new_block(self, block, ephemeral=True):
         )
 
         state = self.__create_evm_state(
-            root_hash, sender_disallow_map, block.header.create_time
+            root_hash, sender_disallow_map, block.header.create_time, prev_minor_hash
         )
         if ephemeral:
             state = state.ephemeral_clone()
         state.gas_limit = block.header.evm_gas_limit
         state.block_number = block.header.height
-        state.recent_uncles[
-            state.block_number
-        ] = []  # TODO [x.hash for x in block.uncles]
-        # TODO: create a account with shard info if the account is not created
-        # Right now the full_shard_key for coinbase actually comes from the first tx that got applied
         state.block_coinbase = coinbase_recipient
         state.block_difficulty = block.header.difficulty
         state.block_reward = 0
-        state.prev_headers = []  # TODO: state.add_block_header(block.header)
         return state
 
     def __is_same_minor_chain(self, longer_block_header, shorter_block_header):
@@ -606,7 +614,13 @@ def __is_same_root_chain(self, longer_block_header, shorter_block_header):
             header = self.db.get_root_block_header_by_hash(header.hash_prev_block)
         return header == shorter_block_header
 
-    def validate_block(self, block: MinorBlock, gas_limit=None, xshard_gas_limit=None):
+    def validate_block(
+        self,
+        block: MinorBlock,
+        gas_limit=None,
+        xshard_gas_limit=None,
+        validate_time=True,
+    ):
         """ Validate a block before running evm transactions
         """
         if block.header.version != 0:
@@ -635,13 +649,13 @@ def validate_block(self, block: MinorBlock, gas_limit=None, xshard_gas_limit=Non
         if block.header.branch != self.branch:
             raise ValueError("branch mismatch")
 
-        if (
+        if validate_time and (
             block.header.create_time
             > time_ms() // 1000 + ALLOWED_FUTURE_BLOCKS_TIME_VALIDATION
         ):
             raise ValueError("block too far into future")
 
-        if block.header.create_time <= prev_header.create_time:
+        if validate_time and block.header.create_time <= prev_header.create_time:
             raise ValueError(
                 "incorrect create time tip time {}, new block time {}".format(
                     block.header.create_time, self.chain[-1].create_time
@@ -738,11 +752,10 @@ def validate_diff_match_prev(self, curr_header, prev_header):
             if diff != curr_header.difficulty:
                 raise ValueError("incorrect difficulty")
 
-    def run_block(self, block, evm_state=None, x_shard_receive_tx_list=None):
+    def run_block(self, block, x_shard_receive_tx_list=None):
         if x_shard_receive_tx_list is None:
             x_shard_receive_tx_list = []
-        if evm_state is None:
-            evm_state = self._get_evm_state_for_new_block(block, ephemeral=False)
+        evm_state = self._get_evm_state_for_new_block(block, ephemeral=False)
         (
             xtx_list,
             evm_state.xshard_tx_cursor_info,
@@ -774,6 +787,7 @@ def run_block(self, block, evm_state=None, x_shard_receive_tx_list=None):
         for k, v in pure_coinbase_amount.balance_map.items():
             evm_state.delta_token_balance(evm_state.block_coinbase, k, v)
 
+        evm_state.add_block_header(block.header)
         # Update actual root hash
         evm_state.commit()
         return evm_state
@@ -868,6 +882,7 @@ def add_block(
         xshard_gas_limit=None,
         force=False,
         write_db=True,
+        validate_time=True,
     ):
         """  Add a block to local db.  Perform validate and update tip accordingly
         gas_limit and xshard_gas_limit are used for testing only.
@@ -904,7 +919,10 @@ def add_block(
         x_shard_receive_tx_list = []
         # Throw exception if fail to run
         self.validate_block(
-            block, gas_limit=gas_limit, xshard_gas_limit=xshard_gas_limit
+            block,
+            gas_limit=gas_limit,
+            xshard_gas_limit=xshard_gas_limit,
+            validate_time=validate_time,
         )
         evm_state = self.run_block(
             block, x_shard_receive_tx_list=x_shard_receive_tx_list
@@ -1040,15 +1058,22 @@ def get_coinbase_amount_map(self, height) -> TokenBalanceMap:
     def get_tip(self) -> MinorBlock:
         return self.db.get_minor_block_by_hash(self.header_tip.get_hash())
 
-    def finalize_and_add_block(self, block, gas_limit=None, xshard_gas_limit=None):
+    def finalize_and_add_block(
+        self, block, gas_limit=None, xshard_gas_limit=None, validate_time=True
+    ):
         """ Finalize the block by filling post-tx data including tx fee collected
         gas_limit and xshard_gas_limit is used to verify customized gas limits and they are for test purpose only
         """
         evm_state = self.run_block(block)
         coinbase_amount_map = self.get_coinbase_amount_map(block.header.height)
         coinbase_amount_map.add(evm_state.block_fee_tokens)
         block.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map)
-        self.add_block(block, gas_limit=gas_limit, xshard_gas_limit=xshard_gas_limit)
+        self.add_block(
+            block,
+            gas_limit=gas_limit,
+            xshard_gas_limit=xshard_gas_limit,
+            validate_time=validate_time,
+        )
 
     def get_token_balance(
         self, recipient: bytes, token_id: int, height: Optional[int] = None
@@ -1484,7 +1509,10 @@ def add_root_block(self, root_block: RootBlock):
             b = self.db.get_minor_block_by_hash(h)
             sender_disallow_map = self._get_sender_disallow_map(b.header)
             evm_state = self.__create_evm_state(
-                b.meta.hash_evm_state_root, sender_disallow_map, int(time.time())
+                b.meta.hash_evm_state_root,
+                sender_disallow_map,
+                int(time.time()),
+                b.header.get_hash(),
             )
             self.__update_tip(b, evm_state)
             Logger.info(

quarkchain/cluster/tests/test_shard_state.py

@@ -3814,3 +3814,73 @@ def test_posw_stake_by_block_decay_by_epoch(self):
         b1.header.create_time = 99
         posw_info = state._posw_info(b1)
         self.assertEqual(posw_info.posw_mineable_blocks, 200 / 100)
+
+    def test_blockhash_in_evm(self):
+        id1 = Identity.create_random_identity()
+        acc1 = Address.create_from_identity(id1, full_shard_key=0)
+
+        env = get_test_env(genesis_account=acc1, genesis_minor_quarkash=10 ** 18)
+        state = create_default_shard_state(env=env)
+        # Only has genesis header
+        self.assertEqual(len(state.evm_state.prev_headers), 1)
+
+        contract_bytecode = "6080604052348015600f57600080fd5b5060ae8061001e6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806307310fae14602d575b600080fd5b605660048036036020811015604157600080fd5b8101908080359060200190929190505050606c565b6040518082815260200191505060405180910390f35b600081430340905091905056fea265627a7a723158205df0e0c36db38808e196b2d7cf91b07c71a980d1b6e4bb80ae42537db54c061f64736f6c63430005110032"
+        """
+        pragma solidity >=0.4.22 <0.6.0;
+        contract C {
+            function bh(uint256 offset) public view returns (bytes32) {
+                return blockhash(block.number - offset);
+            }
+        }       
+        """
+        tx = contract_creation_tx(
+            shard_state=state,
+            key=id1.get_key(),
+            from_address=acc1,
+            to_full_shard_key=acc1.full_shard_key,
+            bytecode=contract_bytecode,
+            gas=1000000,
+        )
+
+        self.assertTrue(state.add_tx(tx))
+        b = state.create_block_to_mine(address=acc1)
+        state.finalize_and_add_block(b)
+        self.assertEqual(state.header_tip, b.header)
+
+        self.assertEqual(len(state.evm_state.receipts), 1)
+        contract_addr = state.evm_state.receipts[0].contract_address
+
+        tx_gen = lambda data: create_transfer_transaction(
+            shard_state=state,
+            key=id1.get_key(),
+            from_address=acc1,
+            to_address=Address(contract_addr, 0),
+            value=0,
+            data=data,
+            gas=100000,
+        )
+        query = lambda n: state.execute_tx(
+            tx_gen(bytes.fromhex("07310fae") + n.to_bytes(32, byteorder="big")), acc1
+        )
+
+        self.assertEqual(state.header_tip.height, 1)
+        # State has genesis + height 1 block
+        self.assertEqual(len(state.evm_state.prev_headers), 2)
+        bh = query(0)
+        # Doesn't support querying current block hash
+        self.assertEqual(bh, b"\x00" * 32)
+        bh = query(1)
+        self.assertEqual(bh, state.header_tip.get_hash())
+        bh = query(2)
+        self.assertEqual(bh, b"\x00" * 32)  # invalid query
+
+        # Try inserting more blocks
+        for _ in range(300):
+            b = state.create_block_to_mine(address=acc1)
+            state.finalize_and_add_block(b, validate_time=False)
+        self.assertEqual(len(state.evm_state.prev_headers), 256)
+
+        bh = query(256)
+        self.assertNotEqual(bh, b"\x00" * 32)
+        bh = query(257)
+        self.assertEqual(bh, b"\x00" * 32)

quarkchain/evm/messages.py

@@ -547,11 +547,7 @@ def __init__(self, state, sender, gas_price):
         self.log_storage = lambda x: state.account_to_dict(x)
         self.add_suicide = lambda x: state.add_suicide(x)
         self.add_refund = lambda x: state.set_param("refunds", state.refunds + x)
-        self.block_hash = (
-            lambda x: state.get_block_hash(state.block_number - x - 1)
-            if (1 <= state.block_number - x <= 256 and x <= state.block_number)
-            else b""
-        )
+        self.block_hash = lambda x: state.get_block_hash(state.block_number - x - 1)
         self.block_coinbase = state.block_coinbase
         self.block_timestamp = state.timestamp
         self.block_number = state.block_number

quarkchain/evm/state.py

@@ -1,5 +1,7 @@
 # Modified based on pyethereum under MIT license
 import copy
+from collections import deque
+
 import rlp
 from rlp.sedes.lists import CountableList
 from rlp.sedes import binary
@@ -53,8 +55,7 @@ def snapshot_form(val):
     "receipts": [],
     "xshard_deposit_receipts": [],
     "suicides": [],
-    "recent_uncles": {},
-    "prev_headers": [],
+    "prev_headers": deque(),
     "refunds": 0,
     "xshard_list": [],
     "full_shard_key": 0,  # should be updated before applying each tx
@@ -348,18 +349,15 @@ def get_bloom(self):
         return bloom
 
     def get_block_hash(self, n):
-        if self.block_number < n or n > 256 or n < 0:
-            o = b"\x00" * 32
-        else:
-            o = (
-                self.prev_headers[n].get_hash()
-                if self.prev_headers[n]
-                else b"\x00" * 32
-            )
-        return o
+        # invariant: len(self.prev_headers) is between 0 and 255 inclusive
+        if 0 <= n < min(256, len(self.prev_headers)):
+            return self.prev_headers[n].get_hash()
+        return b"\x00" * 32
 
     def add_block_header(self, block_header):
-        self.prev_headers = [block_header] + self.prev_headers
+        self.prev_headers.appendleft(block_header)
+        while len(self.prev_headers) > 256:
+            self.prev_headers.pop()
 
     def get_and_cache_account(self, address, should_cache=True):
         if address in self.cache:
@@ -638,11 +636,6 @@ def to_snapshot(self, no_prevblocks=False):
                     prev_header_to_dict(h)
                     for h in v[: self.config["PREV_HEADER_DEPTH"]]
                 ]
-            elif k == "recent_uncles" and not no_prevblocks:
-                snapshot[k] = {
-                    str(n): ["0x" + encode_hex(h) for h in headers]
-                    for n, headers in v.items()
-                }
         return snapshot
 
     # Creates a state from a snapshot
@@ -666,16 +659,6 @@ def from_snapshot(cls, snapshot_data, env):
                 else:
                     headers = default
                 setattr(state, k, headers)
-            elif k == "recent_uncles":
-                if k in snapshot_data:
-                    uncles = {}
-                    for height, _uncles in v.items():
-                        uncles[int(height)] = []
-                        for uncle in _uncles:
-                            uncles[int(height)].append(parse_as_bin(uncle))
-                else:
-                    uncles = default
-                setattr(state, k, uncles)
         state.commit()
         return state
 
@@ -685,7 +668,6 @@ def ephemeral_clone(self):
         s = State.from_snapshot(snapshot, env2)
         for param in STATE_DEFAULTS:
             setattr(s, param, getattr(self, param))
-        s.recent_uncles = self.recent_uncles
         s.prev_headers = self.prev_headers
         for acct in self.cache.values():
             assert not acct.touched or not acct.deleted