From 91228d6e42a76b12a510f217354b64f0257a09e1 Mon Sep 17 00:00:00 2001 From: harisang Date: Fri, 4 Oct 2024 03:13:52 +0300 Subject: [PATCH 1/3] remove conservative deletes --- src/transaction_processor.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 6ffe120..035c2bb 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -35,8 +35,8 @@ def __init__( def get_start_block(self) -> int: """ - Retrieve the most recent block already present in raw_token_imbalances table, - delete entries for that block, and return this block number as start_block. + Retrieve the most recent block X already present in raw_token_imbalances table, + and return block X+1 as start_block. If no entries are present, fallback to get_finalized_block_number(). """ try: @@ -53,13 +53,7 @@ def get_start_block(self) -> int: return self.blockchain_data.get_latest_block() logger.info("Fetched max block number from database: %d", max_block) - - # Delete entries for the max block from the table - delete_sql = read_sql_file("src/sql/delete_entries_max_block.sql") - self.db.execute_and_commit( - delete_sql, {"chain_name": self.chain_name, "block_number": max_block} - ) - return max_block + return max_block + 1 except Exception as e: logger.error("Error fetching start block from database: %s", e) raise From ddf1d66004c66634b06d1cf6b8a70c609b7c091a Mon Sep 17 00:00:00 2001 From: harisang Date: Fri, 4 Oct 2024 03:20:23 +0300 Subject: [PATCH 2/3] dont look too far back in the past --- src/transaction_processor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 035c2bb..7ac2a97 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -47,13 +47,17 @@ def get_start_block(self) -> int: ) row = result.fetchone() max_block = row[0] if row is not None else None + blockchain_latest_block = self.blockchain_data.get_latest_block() # If no entries present, fallback to get_latest_block() if max_block is None: - return self.blockchain_data.get_latest_block() + return blockchain_latest_block logger.info("Fetched max block number from database: %d", max_block) - return max_block + 1 + if max_block > blockchain_latest_block - 7200: + return max_block + 1 + else: + return blockchain_latest_block except Exception as e: logger.error("Error fetching start block from database: %s", e) raise From 64270cd3fc4e0388ef0242ef14e226728b205581 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 4 Oct 2024 11:01:33 +0200 Subject: [PATCH 3/3] added todos to remove the rule on at most filling one day --- src/transaction_processor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 7ac2a97..c695b92 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -37,6 +37,9 @@ def get_start_block(self) -> int: """ Retrieve the most recent block X already present in raw_token_imbalances table, and return block X+1 as start_block. + If block X is from more than 1 day ago, a recent finalized block is returned. + TODO: Remove that rule before moving to production. + If no entries are present, fallback to get_finalized_block_number(). """ try: @@ -57,6 +60,7 @@ def get_start_block(self) -> int: if max_block > blockchain_latest_block - 7200: return max_block + 1 else: + # TODO: Remove this rule before moving to production. return blockchain_latest_block except Exception as e: logger.error("Error fetching start block from database: %s", e)