diff --git a/src/config.ts b/src/config.ts index 70e1d610..0d31800b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,7 @@ import { + ScraperMode, + DEFAULT_RESCRAPE_BLOCKS, + DEFAULT_SCRAPER_MODE, DEFAULT_BASE_GITHUB_LOGO_URL, DEFAULT_BLOCKS_REORG_CHECK_INCREMENT, DEFAULT_BLOCK_FINALITY_THRESHOLD, @@ -69,8 +72,6 @@ interface BridgeContract { startingBlock: number; } -export const RESCRAPE_BLOCKS = 10; //TODO: delete me - const bridgeContracts = [ { contract: '0x1c29670f7a77f1052d30813a0a4f632c78a02610', startingBlock: 9613431 }, { contract: '0x991c745401d5b5e469b8c3e2cb02c748f08754f1', startingBlock: 9613441 }, @@ -111,6 +112,14 @@ export const POSTGRES_URI = process.env.POSTGRES_URI || DEFAULT_LOCAL_POSTGRES_U export const SHOULD_SYNCHRONIZE = process.env.SHOULD_SYNCHRONIZE === 'true'; export const ENABLE_PROMETHEUS_METRICS = getBoolConfig('ENABLE_PROMETHEUS_METRICS', DEFAULT_ENABLE_PROMETHEUS_METRICS); +export const SCRAPER_MODE: ScraperMode = + process.env.SCRAPER_MODE === undefined + ? DEFAULT_SCRAPER_MODE + : process.env.SCRAPER_MODE === 'BLOCKS' + ? 'BLOCKS' + : process.env.SCRAPER_MODE === 'EVENTS' + ? 'EVENTS' + : throwError('Wrong SCRAPER_MODE'); export const METRICS_PATH = process.env.METRICS_PATH || DEFAULT_METRICS_PATH; export const PROMETHEUS_PORT = getIntConfig('PROMETHEUS_PORT', DEFAULT_PROMETHEUS_PORT); @@ -374,6 +383,15 @@ export const FEAT_TOKENS_FROM_TRANSFERS = getBoolConfig( DEFAULT_FEAT_TOKENS_FROM_TRANSFERS, ); +export const TOKENS_FROM_TRANSFERS_START_BLOCK = getIntConfig('TOKENS_FROM_TRANSFERS_START_BLOCK', -1); + +validateStartBlock( + 'TOKENS_FROM_TRANSFERS_START_BLOCK', + TOKENS_FROM_TRANSFERS_START_BLOCK, + "FEAT_TOKENS_FROM_TRANSFERS && SCRAPER_MODE !== 'BLOCKS'", + FEAT_TOKENS_FROM_TRANSFERS && SCRAPER_MODE !== 'BLOCKS', +); + export const FEAT_META_TRANSACTION_EXECUTED_EVENT = getBoolConfig( 'FEAT_META_TRANSACTION_EXECUTED_EVENT', DEFAULT_FEAT_META_TRANSACTION_EXECUTED_EVENT, @@ -450,6 +468,8 @@ validateStartBlock( FEAT_SOCKET_BRIDGE_EVENT, ); +export const RESCRAPE_BLOCKS = getIntConfig('RESCRAPE_BLOCKS', DEFAULT_RESCRAPE_BLOCKS); + export const BLOCKS_REORG_CHECK_INCREMENT = getIntConfig( 'BLOCKS_REORG_CHECK_INCREMENT', DEFAULT_BLOCKS_REORG_CHECK_INCREMENT, diff --git a/src/constants.ts b/src/constants.ts index 4514f00e..6e22eb41 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,5 +1,8 @@ +export type ScraperMode = 'BLOCKS' | 'EVENTS'; + export * from './abis'; +export const DEFAULT_SCRAPER_MODE: ScraperMode = 'EVENTS'; export const DEFAULT_LOCAL_POSTGRES_URI = 'postgresql://user:password@localhost/events'; export const DEFAULT_MAX_BLOCKS_REORG = 35; export const DEFAULT_BLOCKS_REORG_CHECK_INCREMENT = 35; @@ -7,6 +10,7 @@ export const DEFAULT_MAX_BLOCKS_TO_PULL = 120; export const DEFAULT_MAX_BLOCKS_TO_SEARCH = 120; export const DEFAULT_MAX_TX_TO_PULL = 1000; export const DEFAULT_BLOCK_FINALITY_THRESHOLD = 10; +export const DEFAULT_RESCRAPE_BLOCKS = 0; export const DEFAULT_MINUTES_BETWEEN_RUNS = 3; export const DEFAULT_STAKING_POOLS_JSON_URL = 'https://raw.githubusercontent.com/0xProject/0x-staking-pool-registry/master/staking_pools.json'; diff --git a/src/scripts/pull_and_save_tokens_from_transfers.ts b/src/scripts/pull_and_save_tokens_from_transfers.ts index 69ac89e7..4aafdbe0 100644 --- a/src/scripts/pull_and_save_tokens_from_transfers.ts +++ b/src/scripts/pull_and_save_tokens_from_transfers.ts @@ -1,5 +1,5 @@ import { EVM_RPC_URL, MAX_BLOCKS_TO_SEARCH, TOKENS_FROM_TRANSFERS_START_BLOCK } from '../config'; -import { TOKEN_TRANSFER_EVENT_TOPIC } from '../constants'; +import { TRANSFER_EVENT_TOPIC_0 } from '../constants'; import { LogPullInfo, Web3Source } from '../data_sources/events/web3'; import { logger } from '../utils/logger'; import { SCRIPT_RUN_DURATION, SCAN_START_BLOCK, SCAN_END_BLOCK } from '../utils/metrics'; @@ -60,7 +60,7 @@ export class TokensFromTransfersScraper { address: 'nofilter', fromBlock: startBlockNumber, toBlock: endBlockNumber, - topics: TOKEN_TRANSFER_EVENT_TOPIC, + topics: [TRANSFER_EVENT_TOPIC_0], }; const rawLogsArray = await web3Source.getBatchLogInfoForContractsAsync([logPullInfo]);