Merge branch 'feature/deltaupdater-account' into develop

Makefile

@@ -1,8 +1,8 @@
 SHELL := /bin/bash
 PROJECT := graphsense-lib
 VENV := venv
-RELEASE := 'v24.02rc1'
-RELEASESEM := 'v2.2rc1'
+RELEASE := 'v24.02rc2'
+RELEASESEM := 'v2.2rc2'
 
 all: format lint test build
 

setup.cfg

@@ -77,9 +77,11 @@ install_requires =
     graphsense-bitcoin-etl==1.5.4
     base58~=2.1
     typeguard~=4.1
+    diskcache~=5.6
     grpcio
 
 
+
 [options.packages.find]
 where = src
 exclude =

src/graphsenselib/config/config.py

@@ -1,3 +1,4 @@
+import logging
 import math
 import os
 from typing import Dict, List, Optional
@@ -73,6 +74,11 @@ class KeyspaceSetupConfig(BaseModel):
     data_configuration: Dict[str, object] = Field(default_factory=lambda: {})
 
 
+class DeltaUpdaterConfig(BaseModel):
+    fs_cache: Optional[FileSink]
+    currency: str
+
+
 class KeyspaceConfig(BaseModel):
     raw_keyspace_name: str
     transformed_keyspace_name: str
@@ -230,5 +236,15 @@ def get_slack_hooks_by_topic(self, topic: str) -> Optional[SlackTopic]:
     def get_keyspace_config(self, env: str, currency: str) -> KeyspaceConfig:
         return self.get_environment(env).get_keyspace(currency)
 
+    def get_deltaupdater_config(self, env: str, currency: str) -> DeltaUpdaterConfig:
+        fs_cache = (
+            self.get_environment(env)
+            .keyspaces[currency]
+            .ingest_config.raw_keyspace_file_sinks.get("fs-cache")
+        )
+        if fs_cache is None:
+            logging.debug(f"fs-cache not configured for {currency} in {env}")
+        return DeltaUpdaterConfig(fs_cache=fs_cache, currency=currency)
+
 
 config = AppConfig(load=False)

src/graphsenselib/datatypes/__init__.py

@@ -2,3 +2,4 @@
 from .address import AddressAccount, AddressUtxo
 from .common import *
 from .errors import *
+from .transactionhash import TransactionHashAccount, TransactionHashUtxo

src/graphsenselib/datatypes/address.py

@@ -1,6 +1,6 @@
 from typing import Union
 
-from ..utils import hex_to_bytearray
+from ..utils import hex_to_bytes
 
 
 class AddressUtxo:
@@ -61,7 +61,7 @@ def __init__(self, adr: Union[str, bytearray], config):
         """
         self.prefix_length = int(config.address_prefix_length)
         if type(adr) == str:
-            self.address_bytes = hex_to_bytearray(adr)
+            self.address_bytes = hex_to_bytes(adr)
         elif type(adr) == bytearray:
             self.address_bytes = adr
         elif type(adr) == bytes:
@@ -94,3 +94,8 @@ def prefix(self) -> str:
     @property
     def bytearray(self) -> bytearray:  # noqa
         return self.address_bytes
+
+
+class AddressAccountTrx:
+    def __init__(self, adr: Union[str, bytearray], config):
+        raise NotImplementedError("AddressAccountTrx not implemented yet")

src/graphsenselib/datatypes/transactionhash.py

@@ -0,0 +1,62 @@
+"""
+Transaction Class. Functionality depending on the address type.
+"""
+
+from typing import Union
+
+from ..utils import hex_to_bytes
+
+
+class TransactionHashUtxo:
+    def __init__(self, txhash: Union[str, bytearray], config):
+        raise NotImplementedError("TransactionUtxo not implemented yet")
+
+
+class TransactionHashAccount:
+    def __init__(self, txhash: Union[str, bytearray], config):
+        """Init a transaction instance.
+
+        Args:
+            txhash (Union[str, bytearray]): transaction hash
+            config (ConfigRow): entry from the config table in the transformed keyspace
+
+        Raises:
+            Exception: Description
+            ValueError: Description
+        """
+        self.prefix_length = int(config.tx_prefix_length)
+        if type(txhash) == str:
+            self.tx_hash_bytes = hex_to_bytes(txhash)
+        elif type(txhash) == bytearray:
+            self.tx_hash_bytes = txhash
+        elif type(txhash) == bytes:
+            self.tx_hash_bytes = bytearray(txhash)
+        else:
+            raise Exception(f"Unknown type for txhash type: {type(txhash)}")
+
+        # todo potentially different length for tron, also saved in bytearray though
+        if len(self.tx_hash_bytes) != 32:
+            raise ValueError(
+                f"Address is not the right length {len(self.tx_hash_bytes)}"
+            )
+
+    @property
+    def hex(self) -> str:  # noqa
+        return self.tx_hash_bytes.hex()
+
+    @property
+    def db_encoding(self) -> str:
+        return self.bytearray
+        # return f"0x{self.hex}"
+
+    @property
+    def db_encoding_query(self) -> str:
+        return f"0x{self.hex}"
+
+    @property
+    def prefix(self) -> str:
+        return self.hex.upper()[: self.prefix_length]
+
+    @property
+    def bytearray(self) -> bytearray:  # noqa
+        return self.tx_hash_bytes

src/graphsenselib/db/account.py

@@ -7,10 +7,17 @@ class TransformedDbAccount(TransformedDb):
     def get_highest_cluster_id(self, sanity_check=True) -> Optional[int]:
         return None
 
+    def get_highest_transaction_id(self):
+        return self._get_hightest_id(
+            table="transaction_ids_by_transaction_id_group",
+            sanity_check=True,
+            id_col="transaction_id",
+        )
+
 
 class RawDbAccount(RawDb):
     def get_logs_in_block(self, block: int, topic0=None, contract=None) -> Iterable:
-        group = block // self.get_block_bucket_size()
+        group = self.get_id_group(block, self.get_block_bucket_size())
         if topic0 is None:
             data = self.select_safe(
                 "log", where={"block_id": block, "block_id_group": group}
@@ -25,13 +32,27 @@ def get_logs_in_block(self, block: int, topic0=None, contract=None) -> Iterable:
         return data
 
     def get_transaction_ids_in_block(self, block: int) -> Iterable:
-        raise Exception("Not yet implemented.")
+        raise NotImplementedError
 
     def get_transactions_in_block(self, block: int) -> Iterable:
-        raise Exception("Not yet implemented.")
+        result = self.select(
+            "transaction",
+            where={"block_id": block},
+        )
+        return result
+
+    def get_traces_in_block(self, block: int) -> Iterable:
+        block_bucket_size = self.get_block_bucket_size()
+        group = self.get_id_group(block, block_bucket_size)
+
+        results = self.select(
+            "trace", where={"block_id_group": group, "block_id": block}
+        )
+
+        return results
 
     def get_addresses_in_block(self, block: int) -> Iterable:
-        group = block // self.get_block_bucket_size()
+        group = self.get_id_group(block, self.get_block_bucket_size())
 
         # The fetch size is needed since traces currenly contain a lot of null values
         # the null values create tombestones and cassandra refuses to read more than
@@ -50,7 +71,7 @@ def get_addresses_in_block(self, block: int) -> Iterable:
 
 class RawDbAccountTrx(RawDbAccount):
     def get_addresses_in_block(self, block: int) -> Iterable:
-        group = block // self.get_block_bucket_size()
+        group = self.get_id_group(block, self.get_block_bucket_size())
 
         # The fetch size is needed since traces currenly contain a lot of null values
         # the null values create tombestones and cassandra refuses to read more than