Optimize keccak operations (#832)

- Use `fast_keccak`, `fast_keccak_text`, `fast_is_checksum_address` when posible
- Use caching for keccak operations. Memory usage is better than CPU usage

gnosis/eth/account_abstraction/user_operation.py

@@ -5,9 +5,8 @@
 from eth_abi import encode as abi_encode
 from eth_typing import ChecksumAddress, HexStr
 from hexbytes import HexBytes
-from web3 import Web3
 
-from gnosis.eth.utils import fast_keccak
+from gnosis.eth.utils import fast_keccak, fast_to_checksum_address
 
 
 @dataclasses.dataclass(eq=True, frozen=True)
@@ -73,7 +72,7 @@ def __str__(self):
     @cached_property
     def paymaster(self) -> Optional[ChecksumAddress]:
         if self.paymaster_and_data and len(self.paymaster_and_data) >= 20:
-            return Web3.to_checksum_address(self.paymaster_and_data[:20])
+            return fast_to_checksum_address(self.paymaster_and_data[:20])
         return None
 
     @cached_property

gnosis/eth/django/tests/test_forms.py

@@ -2,8 +2,8 @@
 from django.test import TestCase
 
 from hexbytes import HexBytes
-from web3 import Web3
 
+from ...utils import fast_keccak_text
 from ..forms import EthereumAddressFieldForm, HexFieldForm, Keccak256FieldForm
 
 
@@ -77,7 +77,7 @@ def test_keccak256_field_form(self):
             form.errors["value"], ['"0x1234" keccak256 hash should be 32 bytes.']
         )
 
-        form = Keccak256Form(data={"value": Web3.keccak(text="testing").hex()})
+        form = Keccak256Form(data={"value": fast_keccak_text("testing").hex()})
         self.assertTrue(form.is_valid())
 
         form = Keccak256Form(data={"value": ""})

gnosis/eth/django/tests/test_models.py

@@ -5,10 +5,9 @@
 
 from eth_account import Account
 from faker import Faker
-from web3 import Web3
 
 from ...constants import NULL_ADDRESS, SENTINEL_ADDRESS
-from ...utils import fast_is_checksum_address
+from ...utils import fast_is_checksum_address, fast_keccak_text
 from .models import (
     EthereumAddress,
     EthereumAddressV2,
@@ -117,7 +116,7 @@ def test_uint32_field(self):
             Uint32.objects.create(value=-2)
 
     def test_sha3_hash_field(self):
-        value_hexbytes = Web3.keccak(text=faker.name())
+        value_hexbytes = fast_keccak_text(faker.name())
         value_hex_with_0x: str = value_hexbytes.hex()
         value_hex_without_0x: str = value_hex_with_0x[2:]
         value: bytes = bytes(value_hexbytes)
@@ -146,7 +145,7 @@ def test_sha3_hash_field(self):
                 Sha3Hash.objects.create(value=value_hex_invalid)
 
     def test_keccak256_field(self):
-        value_hexbytes = Web3.keccak(text=faker.name())
+        value_hexbytes = fast_keccak_text(faker.name())
         value_hex_with_0x: str = value_hexbytes.hex()
         value_hex_without_0x: str = value_hex_with_0x[2:]
         value: bytes = bytes(value_hexbytes)
@@ -228,7 +227,7 @@ def test_serialize_uint256_field_to_json(self):
         self.assertIn(str(value), serialized)
 
     def test_serialize_sha3_hash_to_json(self):
-        hash = Web3.keccak(text="testSerializer")
+        hash = fast_keccak_text("testSerializer")
         Sha3Hash.objects.create(value=hash)
         serialized = serialize("json", Sha3Hash.objects.all())
         # hash should be in serialized data

gnosis/eth/django/tests/test_serializers.py

@@ -3,10 +3,9 @@
 from eth_account import Account
 from hexbytes import HexBytes
 from rest_framework import serializers
-from web3 import Web3
 
 from ...constants import NULL_ADDRESS, SENTINEL_ADDRESS
-from ...utils import get_eth_address_with_invalid_checksum
+from ...utils import fast_keccak_text, get_eth_address_with_invalid_checksum
 from ..serializers import (
     EthereumAddressField,
     HexadecimalField,
@@ -148,7 +147,7 @@ class A:
             self.assertEqual(serializer.data["value"], HexBytes(hex_value).hex())
 
     def test_hash_serializer_field(self):
-        value = Web3.keccak(text="test").hex()
+        value = fast_keccak_text("test").hex()
         serializer = Sha3HashSerializerTest(data={"value": value})
         self.assertTrue(serializer.is_valid())
         self.assertEqual(serializer.validated_data["value"], HexBytes(value))

gnosis/eth/tests/eip712/test_eip712.py

@@ -86,7 +86,7 @@ def test_eip712_encode_hash(self):
         payload["types"] = self.types
         self.assertEqual(
             eip712_encode_hash(payload).hex(),
-            "7c02fe79823722257b42ea95720e7dd31d51c3f6769dc0f56a271800dd030ef1",
+            "0x7c02fe79823722257b42ea95720e7dd31d51c3f6769dc0f56a271800dd030ef1",
         )
 
     def test_eip712_encode_hash_string_uint(self):
@@ -104,7 +104,7 @@ def test_eip712_encode_hash_string_uint(self):
         }
         self.assertEqual(
             eip712_encode_hash(payload).hex(),
-            "7c02fe79823722257b42ea95720e7dd31d51c3f6769dc0f56a271800dd030ef1",
+            "0x7c02fe79823722257b42ea95720e7dd31d51c3f6769dc0f56a271800dd030ef1",
         )
 
     def test_eip712_encode_hash_string_bytes(self):
@@ -131,7 +131,7 @@ def test_eip712_encode_hash_string_bytes(self):
 
         self.assertEqual(
             eip712_encode_hash(payload).hex(),
-            "2950cf06416c6c20059f24a965e3baf51a24f4ef49a1e7b1a47ee13ee08cde1f",
+            "0x2950cf06416c6c20059f24a965e3baf51a24f4ef49a1e7b1a47ee13ee08cde1f",
         )
 
     def test_eip712_encode_nested_with_array(self):
@@ -206,7 +206,7 @@ def test_eip712_encode_nested_with_array(self):
         }
         self.assertEqual(
             eip712_encode_hash(payload).hex(),
-            "2f6856dbd51836973c1e61852b64949556aa2e7f253d9e20e682f9a02d436791",
+            "0x2f6856dbd51836973c1e61852b64949556aa2e7f253d9e20e682f9a02d436791",
         )
 
     def test_eip712_encode_nested_with_empty_array(self):
@@ -264,7 +264,7 @@ def test_eip712_encode_nested_with_empty_array(self):
         }
         self.assertEqual(
             eip712_encode_hash(payload).hex(),
-            "5dd3156111fb5e400606d4dd75ff097e36eb56614c84924b5eb6d1cf1b5038cf",
+            "0x5dd3156111fb5e400606d4dd75ff097e36eb56614c84924b5eb6d1cf1b5038cf",
         )
 
     def test_eip712_encode_nested_without_array(self):
@@ -321,5 +321,5 @@ def test_eip712_encode_nested_without_array(self):
 
         self.assertEqual(
             eip712_encode_hash(payload).hex(),
-            "9a55335a1d86221594e96018fc3df611a1485d95b5b2afbef1540ac51f63d249",
+            "0x9a55335a1d86221594e96018fc3df611a1485d95b5b2afbef1540ac51f63d249",
         )

gnosis/eth/utils.py

@@ -1,3 +1,4 @@
+from functools import lru_cache
 from typing import Union
 
 import eth_abi
@@ -20,13 +21,25 @@ def get_empty_tx_params() -> TxParams:
     }
 
 
+@lru_cache(maxsize=8192)
 def fast_keccak(value: bytes) -> Hash32:
     """
     Calculates ethereum keccak256 using fast library `pysha3`
+
+    :param value:
+    :return: Keccak256 used by ethereum as `HexBytes`
+    """
+    return HexBytes(keccak_256(value).digest())
+
+
+def fast_keccak_text(value: str) -> Hash32:
+    """
+    Calculates ethereum keccak256 using fast library `pysha3`
+
     :param value:
-    :return: Keccak256 used by ethereum as `bytes`
+    :return: Keccak256 used by ethereum as `HexBytes`
     """
-    return keccak_256(value).digest()
+    return fast_keccak(value.encode())
 
 
 def fast_keccak_hex(value: bytes) -> HexStr:
@@ -69,16 +82,27 @@ def _build_checksum_address(
     )
 
 
+@lru_cache(maxsize=8192)
+def _fast_to_checksum_address(address: HexAddress):
+    address_hash = fast_keccak_hex(address.encode())
+    return _build_checksum_address(address, address_hash)
+
+
 def fast_to_checksum_address(value: Union[AnyAddress, str, bytes]) -> ChecksumAddress:
     """
     Converts to checksum_address. Uses more optimal `pysha3` instead of `eth_utils` for keccak256 calculation
 
     :param value:
     :return:
     """
+    if isinstance(value, bytes):
+        if len(value) != 20:
+            raise ValueError(
+                "Cannot convert %s to a checksum address, 20 bytes were expected"
+            )
+
     norm_address = HexAddress(HexStr(to_normalized_address(value)[2:]))
-    address_hash = fast_keccak_hex(norm_address.encode())
-    return _build_checksum_address(norm_address, address_hash)
+    return _fast_to_checksum_address(norm_address)
 
 
 def fast_bytes_to_checksum_address(value: bytes) -> ChecksumAddress:
@@ -94,8 +118,7 @@ def fast_bytes_to_checksum_address(value: bytes) -> ChecksumAddress:
             "Cannot convert %s to a checksum address, 20 bytes were expected"
         )
     norm_address = HexAddress(HexStr(bytes(value).hex()))
-    address_hash = fast_keccak_hex(norm_address.encode())
-    return _build_checksum_address(norm_address, address_hash)
+    return _fast_to_checksum_address(norm_address)
 
 
 def fast_is_checksum_address(value: Union[AnyAddress, str, bytes]) -> bool:

gnosis/safe/api/transaction_service_api/transaction_service_api.py

@@ -4,9 +4,9 @@
 from eth_account.signers.local import LocalAccount
 from eth_typing import ChecksumAddress, HexStr
 from hexbytes import HexBytes
-from web3 import Web3
 
 from gnosis.eth import EthereumNetwork
+from gnosis.eth.utils import fast_keccak_text
 from gnosis.safe import SafeTx
 
 from ..base_api import SafeAPIException, SafeBaseAPI
@@ -37,7 +37,7 @@ class TransactionServiceApi(SafeBaseAPI):
 
     @classmethod
     def create_delegate_message_hash(cls, delegate_address: ChecksumAddress) -> str:
-        return Web3.keccak(text=get_delegate_message(delegate_address))
+        return fast_keccak_text(get_delegate_message(delegate_address))
 
     @classmethod
     def data_decoded_to_text(cls, data_decoded: Dict[str, Any]) -> Optional[str]:

gnosis/safe/safe.py

@@ -522,12 +522,12 @@ def get_message_hash(self, message: Union[str, Hash32]) -> Hash32:
             message = message.encode()
         message_hash = fast_keccak(message)
 
-        safe_message_hash = Web3.keccak(
+        safe_message_hash = fast_keccak(
             eth_abi.encode(
                 ["bytes32", "bytes32"], [self.SAFE_MESSAGE_TYPEHASH, message_hash]
             )
         )
-        return Web3.keccak(
+        return fast_keccak(
             encode_packed(
                 ["bytes1", "bytes1", "bytes32", "bytes32"],
                 [

gnosis/safe/tests/test_proxy_factory/test_proxy_factory.py

@@ -4,7 +4,6 @@
 from django.test import TestCase
 
 from eth_account import Account
-from web3 import Web3
 
 from gnosis.eth import EthereumClient
 from gnosis.eth.contracts import (
@@ -14,7 +13,7 @@
 )
 from gnosis.eth.exceptions import ContractAlreadyDeployed
 from gnosis.eth.tests.utils import just_test_if_mainnet_node
-from gnosis.eth.utils import compare_byte_code
+from gnosis.eth.utils import compare_byte_code, fast_is_checksum_address
 from gnosis.safe import Safe
 from gnosis.safe.proxy_factory import (
     ProxyFactory,
@@ -113,7 +112,7 @@ def test_calculate_proxy_address(self):
         address = self.proxy_factory.calculate_proxy_address(
             self.safe_contract_V1_4_1.address, b"", salt_nonce
         )
-        self.assertTrue(Web3.is_checksum_address(address))
+        self.assertTrue(fast_is_checksum_address(address))
         # Same call with same parameters should return the same address
         same_address = self.proxy_factory.calculate_proxy_address(
             self.safe_contract_V1_4_1.address, b"", salt_nonce
@@ -136,7 +135,7 @@ def test_calculate_proxy_address(self):
         chain_specific_address = self.proxy_factory.calculate_proxy_address(
             self.safe_contract_V1_4_1.address, b"", salt_nonce, chain_specific=True
         )
-        self.assertTrue(Web3.is_checksum_address(chain_specific_address))
+        self.assertTrue(fast_is_checksum_address(chain_specific_address))
         self.assertNotEqual(address, chain_specific_address)
         ethereum_tx_sent = self.proxy_factory.deploy_proxy_contract_with_nonce(
             self.ethereum_test_account,

gnosis/safe/tests/test_safe.py

@@ -4,11 +4,10 @@
 
 from eth_account import Account
 from hexbytes import HexBytes
-from web3 import Web3
 
 from gnosis.eth.constants import GAS_CALL_DATA_BYTE, NULL_ADDRESS
 from gnosis.eth.contracts import get_safe_contract, get_sign_message_lib_contract
-from gnosis.eth.utils import get_empty_tx_params
+from gnosis.eth.utils import fast_keccak_text, get_empty_tx_params
 
 from ..enums import SafeOperationEnum
 from ..exceptions import (
@@ -673,8 +672,8 @@ def test_retrieve_modules_unitialized_safe(self):
     def test_retrieve_is_hash_approved(self):
         safe = self.deploy_test_safe(owners=[self.ethereum_test_account.address])
         safe_contract = safe.contract
-        fake_tx_hash = Web3.keccak(text="Knopfler")
-        another_tx_hash = Web3.keccak(text="Marc")
+        fake_tx_hash = fast_keccak_text("Knopfler")
+        another_tx_hash = fast_keccak_text("Marc")
         tx = safe_contract.functions.approveHash(fake_tx_hash).build_transaction(
             {"from": self.ethereum_test_account.address}
         )

gnosis/safe/tests/test_safe_signature.py

@@ -10,6 +10,8 @@
 from hexbytes import HexBytes
 from web3 import Web3
 
+from gnosis.eth.utils import fast_keccak, fast_keccak_text
+
 from ...eth.tests.ethereum_test_case import EthereumTestCaseMixin
 from ..safe_signature import (
     SafeSignature,
@@ -70,7 +72,7 @@ def test_approved_hash_signature(self):
 
     def test_approved_hash_signature_build(self):
         owner = Account.create().address
-        safe_tx_hash = Web3.keccak(text="random")
+        safe_tx_hash = fast_keccak_text("random")
         safe_signature = SafeSignatureApprovedHash.build_for_owner(owner, safe_tx_hash)
         self.assertEqual(len(safe_signature.signature), 65)
         self.assertEqual(safe_signature.signature_type, SafeSignatureType.APPROVED_HASH)
@@ -132,7 +134,7 @@ def test_defunct_hash_message(self):
         encoded_message = encode_packed(
             ["(string,bytes32)"], [(ethereum_signed_message, HexBytes(safe_tx_hash))]
         )
-        encoded_hash = Web3.keccak(encoded_message)
+        encoded_hash = fast_keccak(encoded_message)
         self.assertEqual(encoded_hash, defunct_hash_message(primitive=safe_tx_hash))
 
     def test_parse_signature(self):
@@ -173,7 +175,7 @@ def test_parse_signature_with_trailing_zeroes(self):
         )
 
     def test_parse_signature_empty(self):
-        safe_tx_hash = Web3.keccak(text="Legoshi")
+        safe_tx_hash = fast_keccak_text("Legoshi")
         for value in (b"", "", None):
             self.assertEqual(SafeSignature.parse_signature(value, safe_tx_hash), [])
 
@@ -288,7 +290,7 @@ def test_contract_signature(self):
             owners=[owner_1.address], initial_funding_wei=Web3.to_wei(0.01, "ether")
         )
         safe_contract = safe.contract
-        safe_tx_hash = Web3.keccak(text="test")
+        safe_tx_hash = fast_keccak_text("test")
         signature_r = HexBytes(safe.address.replace("0x", "").rjust(64, "0"))
         signature_s = HexBytes(
             "41".rjust(64, "0")
@@ -315,7 +317,7 @@ def test_contract_signature(self):
         self.assertIsInstance(safe_signature, SafeSignatureContract)
 
         # Check with crafted signature
-        safe_tx_hash_2 = Web3.keccak(text="test2")
+        safe_tx_hash_2 = fast_keccak_text("test2")
         safe_signature = SafeSignature.parse_signature(signature, safe_tx_hash_2)[0]
         self.assertFalse(safe_signature.is_valid(self.ethereum_client, None))
 
@@ -349,7 +351,7 @@ def test_contract_multiple_signatures(self):
             owners=[owner_1.address], initial_funding_wei=Web3.to_wei(0.01, "ether")
         )
         safe_contract = safe.contract
-        safe_tx_hash = Web3.keccak(text="test")
+        safe_tx_hash = fast_keccak_text("test")
 
         tx = safe_contract.functions.signMessage(safe_tx_hash).build_transaction(
             {"from": safe.address}