From 0f260e2c76c59d85bfeca704880ba20f3279a8d1 Mon Sep 17 00:00:00 2001 From: "Michael F." Date: Tue, 16 Jul 2024 17:42:29 +0200 Subject: [PATCH] fix missing transferto_address in some tron traces --- CHANGELOG.md | 4 ++++ Makefile | 4 ++-- src/graphsenselib/ingest/transform.py | 12 ++++++++++++ .../schema/resources/parquet/account_trx.py | 13 ------------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b853f55..e09589e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [24.07.7/2.3.7] 2024-07-16 +### fixed +- tron delta-dump: fix missing transferto_address in some tron traces + ## [24.07.6/2.3.6] 2024-07-15 ### fixed - tron delta-dump freezes on grpc asyncio requests diff --git a/Makefile b/Makefile index 473befa..da66cad 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ SHELL := /bin/bash PROJECT := graphsense-lib VENV := venv -RELEASE := 'v24.07.6' -RELEASESEM := 'v2.3.6' +RELEASE := 'v24.07.7' +RELEASESEM := 'v2.3.7' all: format lint test build diff --git a/src/graphsenselib/ingest/transform.py b/src/graphsenselib/ingest/transform.py index 1174c39..49b1c7a 100644 --- a/src/graphsenselib/ingest/transform.py +++ b/src/graphsenselib/ingest/transform.py @@ -106,6 +106,18 @@ def transform(self, block_range_content: BlockRangeContent) -> BlockRangeContent traces = drop_columns_from_list(traces, ["block_id_group"]) logs = drop_columns_from_list(logs, ["block_id_group"]) + def fix_transferToAddress(item): + if len(item["transferto_address"]) == 0: + # for some very rare txs + # e.g. f0b31777dcc58cbca074380ff6f25f8495898edba2da0c43b099b3f276ae3d74 + # transferTo_address is empty which does not mach our schema. + # as a quick fix we set a dummy address for now. + dummy_address = b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00ikna" # noqa + item["transferto_address"] = dummy_address + return item + + traces = [fix_transferToAddress(trace) for trace in traces] + txs = to_bytes(txs, BINARY_COL_CONVERSION_MAP_ACCOUNT_TRX["transaction"]) blocks = to_bytes(blocks, BINARY_COL_CONVERSION_MAP_ACCOUNT_TRX["block"]) traces = to_bytes(traces, BINARY_COL_CONVERSION_MAP_ACCOUNT_TRX["trace"]) diff --git a/src/graphsenselib/schema/resources/parquet/account_trx.py b/src/graphsenselib/schema/resources/parquet/account_trx.py index 5f3049d..566426f 100644 --- a/src/graphsenselib/schema/resources/parquet/account_trx.py +++ b/src/graphsenselib/schema/resources/parquet/account_trx.py @@ -24,19 +24,6 @@ def set_field_type(schema, field_name, new_type): block_schema = set_field_type(block_schema, "gas_used", pa.int64()) ACCOUNT_TRX_SCHEMA_RAW["block"] = block_schema -# trace_schema = ACCOUNT_TRX_SCHEMA_RAW["trace"] -# # internal_index', 'transferto_address', 'call_info_index', -# # 'caller_address', 'call_value', 'rejected', 'call_token_id', 'note -# trace_schema = set_field_type(trace_schema, "internal_index", pa.int16()) -# trace_schema = set_field_type(trace_schema, "transferto_address", pa.binary(20)) -# trace_schema = set_field_type(trace_schema, "call_info_index", pa.int16()) -# trace_schema = set_field_type(trace_schema, "caller_address", pa.binary(20)) -# trace_schema = set_field_type(trace_schema, "call_value", pa.binary()) -# trace_schema = set_field_type(trace_schema, "rejected", pa.bool_()) -# trace_schema = set_field_type(trace_schema, "call_token_id", pa.int32()) -# trace_schema = set_field_type(trace_schema, "note", pa.string()) -# ACCOUNT_TRX_SCHEMA_RAW["trace"] = trace_schema - transaction_schema = ACCOUNT_TRX_SCHEMA_RAW["transaction"] transaction_schema = set_field_type(transaction_schema, "gas", pa.int64()) ACCOUNT_TRX_SCHEMA_RAW["transaction"] = transaction_schema