From 5d6bdc30633ea649759533c363c3c2f98163e65d Mon Sep 17 00:00:00 2001 From: Yijing Date: Tue, 10 Oct 2023 17:07:45 -0500 Subject: [PATCH] add other chains --- build_local.sh | 12 +- docker-compose-dev.yml | 326 +++++++++++++++++++++++++++ docker-compose.yml | 9 +- src/scripts/utils/event_abi_utils.ts | 2 +- 4 files changed, 340 insertions(+), 9 deletions(-) create mode 100644 docker-compose-dev.yml diff --git a/build_local.sh b/build_local.sh index feb16cc5..d58120ab 100755 --- a/build_local.sh +++ b/build_local.sh @@ -1,13 +1,13 @@ #!/usr/bin/env sh -docker-compose -f docker-compose.yml up -d postgres -docker-compose -f docker-compose.yml build event-pipeline-ethereum -docker-compose -f docker-compose.yml build event-pipeline-base +docker-compose -f docker-compose-dev.yml up -d postgres +docker-compose -f docker-compose-dev.yml build event-pipeline-ethereum +docker-compose -f docker-compose-dev.yml build event-pipeline-base # `-f docker-compose.dev.yml` add environment variables for testing -docker-compose -f docker-compose.yml up event-pipeline-ethereum +docker-compose -f docker-compose-dev.yml up event-pipeline-ethereum -docker-compose -f docker-compose.yml build token-scraper-ethereum +docker-compose -f docker-compose-dev.yml build token-scraper-ethereum # `-f docker-compose.dev.yml` add environment variables for testing -docker-compose -f docker-compose.yml up token-scraper-ethereum +docker-compose -f docker-compose-dev.yml up token-scraper-ethereum diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 00000000..7d386963 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,326 @@ +version: '3' +services: + postgres: + image: postgres:13.4 + environment: + POSTGRES_USER: '${POSTGRES_USER}' + POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}' + POSTGRES_DB: '${POSTGRES_DB}' + POSTGRES_PORT: ${POSTGRES_PORT} + # persist the postgres data to disk so we don't lose it + # on rebuilds. + volumes: + - ./mounts/postgres:/var/lib/postgresql/data + - ./postgres-init.sql:/docker-entrypoint-initdb.d/postgres-init.sql + ports: + - '5432:5432' + command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr"] + + event-pipeline-ethereum: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' + CHAIN_ID: '1' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events' + # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" + # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 10247094 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + MAX_TX_TO_PULL: 1000 + BLOCK_FINALITY_THRESHOLD: 0 + SECONDS_BETWEEN_RUNS: 1 + FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" + UNISWAP_V2_VIP_SWAP_SOURCES: "UniswapV2,SushiSwap" + UNISWAP_V2_VIP_SWAP_START_BLOCK: 10917104 + FEAT_UNISWAP_V3_VIP_SWAP_EVENT: "true" + UNISWAP_V3_VIP_SWAP_START_BLOCK: 12553659 + FEAT_UNISWAP_V3_SWAP_EVENT: "true" + UNISWAP_V3_SWAP_START_BLOCK: 16670838 + FEAT_LIMIT_ORDERS: "true" + V4_NATIVE_FILL_START_BLOCK: "11591021" + FEAT_PLP_SWAP_EVENT: "true" + PLP_VIP_START_BLOCK: 11377457 + FEAT_OTC_ORDERS: "true" + OTC_ORDERS_FEATURE_START_BLOCK: 13143075 + FEAT_CANCEL_EVENTS: "true" + FEAT_STAKING: "true" + STAKING_DEPLOYMENT_BLOCK: 8952581 + FEAT_RFQ_EVENT: "true" + FEAT_V3_NATIVE_FILL: "true" + FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + FLASHWALLET_ADDRESS: "0x22f9dcf4647084d6c31b2765f6910cd85c178c18" + FLASHWALLET_DEPLOYMENT_BLOCK: 12231666 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 14258205 + FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" + UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "UniswapV2:0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f:10000835,SushiSwap:0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac:10794229" + FEAT_UNISWAP_V2_SYNC_EVENT: "true" + UNISWAP_V2_SYNC_START_BLOCK: 10000835 + FEAT_ONCHAIN_GOVERNANCE: "true" + ONCHAIN_GOVERNANCE_START_BLOCK: 16990159 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 10247094 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' + + event-pipeline-bsc: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_BSC}' + CHAIN_ID: '56' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_bsc' + MAX_BLOCKS_REORG: 50 + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 5375047 + MAX_BLOCKS_TO_SEARCH: 2000 + MAX_BLOCKS_TO_PULL: 5000 + SECONDS_BETWEEN_RUNS: 1 + FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" + UNISWAP_V2_VIP_SWAP_SOURCES: "PancakeSwap,BakerySwap,SushiSwap,CafeSwap,SwapLiquidity,ApeSwapFinance,CheeseSwap,Swap" + FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" + UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "PancakeSwap:0xbcfccbde45ce874adcb698cc183debcf17952812:586851,PancakeSwap!v2:0xca143ce32fe78f1f7019d7d551a6402fc5350c73:6809737,BakerySwap:0x01bf7c66c6bd861915cdaae475042d3c4bae16a7:470617,SushiSwap:0xc35dadb65012ec5796536bd9864ed8773abc74c4:5205069,CafeSwap:0x3e708fdbe3ada63fc94f8f61811196f1302137ad:5865260,SwapLiquidity:0x553990f2cba90272390f62c5bdb1681ffc899675:784352,ApeSwapFinance:0x0841bd0b734e4f5853f0dd8d7ea041c241fb0da6:4855901,CheeseSwap:0xdd538e4fd1b69b7863e1f741213276a6cf1efb3b:1569172" + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 15860129 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 5375047 + + event-pipeline-polygon: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_POLYGON}' + CHAIN_ID: '137' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_polygon' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 14391480 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + MINUTES_BETWEEN_RUNS: 1 + FEAT_SLINGSHOT_TRADE_EVENT: "true" + SLINGSHOT_DEPLOYMENT_BLOCK: 14500000 + FEAT_LIMIT_ORDERS: "true" + V4_NATIVE_FILL_START_BLOCK: "17587182" + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 25697854 + FEAT_OTC_ORDERS: "true" + OTC_ORDERS_FEATURE_START_BLOCK: 26479109 + FEAT_META_TRANSACTION_EXECUTED_EVENT: "true" + META_TRANSACTION_EXECUTED_START_BLOCK: 39406300 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 14391480 + + event-pipeline-avalanche: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_AVALANCHE}' + CHAIN_ID: '43114' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_avalanche' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 3601700 + MAX_BLOCKS_TO_SEARCH: 5000 + MAX_BLOCKS_TO_PULL: 2000 + MINUTES_BETWEEN_RUNS: 1 + FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498" + FLASHWALLET_DEPLOYMENT_BLOCK: 11805869 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 11816835 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 3601700 + + event-pipeline-fantom: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_FANTOM}' + CHAIN_ID: '250' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_fantom' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_ADDRESS: "0xDEF189DeAEF76E379df891899eb5A00a94cBC250" + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 18855765 + MAX_BLOCKS_TO_SEARCH: 2000 + MAX_BLOCKS_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 60 + FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + FLASHWALLET_ADDRESS: "0xb4d961671cadfed687e040b076eee29840c142e5" + FLASHWALLET_DEPLOYMENT_BLOCK: 18855797 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 32817445 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 18855765 + + event-pipeline-celo: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_CELO}' + CHAIN_ID: '42220' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_celo' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 9350111 + MAX_BLOCKS_TO_SEARCH: 5000 + MAX_BLOCKS_TO_PULL: 2000 + MINUTES_BETWEEN_RUNS: 1 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 11820000 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 9350111 + + event-pipeline-optimism: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_OPTIMISM}' + CHAIN_ID: '10' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_optimism' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 1691335 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 30 + EP_ADDRESS: "0xdef1abe32c034e558cdd535791643c58a13acc10" + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 4214981 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 1691335 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" + + event-pipeline-arbitrum: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_ARBITRUM}' + CHAIN_ID: '42161' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_arbitrum' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 4050733 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 30 + EP_ADDRESS: "0xdef1c0ded9bec7f1a1670819833240f027b25eff" + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 4050733 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 4050733 + + event-pipeline-base: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_BASE}' + CHAIN_ID: '8453' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_base' + EP_DEPLOYMENT_BLOCK: 1410394 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 100 + SECONDS_BETWEEN_RUNS: 1 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 1410394 + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_START_BLOCK: 1410394 + + token-scraper-ethereum: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' + CHAIN_ID: '1' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + LOG_LEVEL: "info" + FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" + TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 + EP_DEPLOYMENT_BLOCK: 10247094 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + MAX_TX_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 5 diff --git a/docker-compose.yml b/docker-compose.yml index 2b9e1543..ad9b325b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,6 +102,7 @@ services: NFT_FEATURE_START_BLOCK: 15860129 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_START_BLOCK: 5375047 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c' event-pipeline-polygon: depends_on: @@ -136,6 +137,7 @@ services: META_TRANSACTION_EXECUTED_START_BLOCK: 39406300 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_START_BLOCK: 14391480 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270' event-pipeline-avalanche: depends_on: @@ -165,6 +167,8 @@ services: NFT_FEATURE_START_BLOCK: 11816835 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_START_BLOCK: 3601700 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7' + event-pipeline-fantom: depends_on: @@ -195,6 +199,7 @@ services: NFT_FEATURE_START_BLOCK: 32817445 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_START_BLOCK: 18855765 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' event-pipeline-celo: depends_on: @@ -219,8 +224,6 @@ services: MINUTES_BETWEEN_RUNS: 1 FEAT_NFT: "true" NFT_FEATURE_START_BLOCK: 11820000 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_START_BLOCK: 9350111 event-pipeline-optimism: depends_on: @@ -274,6 +277,7 @@ services: NFT_FEATURE_START_BLOCK: 4050733 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_START_BLOCK: 4050733 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x82af49447d8a07e3bd95bd0d56f35241523fbab1' event-pipeline-base: depends_on: @@ -299,6 +303,7 @@ services: KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_START_BLOCK: 1410394 + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4200000000000000000000000000000000000006' token-scraper-ethereum: depends_on: diff --git a/src/scripts/utils/event_abi_utils.ts b/src/scripts/utils/event_abi_utils.ts index 36db5f18..8c446db0 100644 --- a/src/scripts/utils/event_abi_utils.ts +++ b/src/scripts/utils/event_abi_utils.ts @@ -269,7 +269,7 @@ export class PullAndSaveEventsByTopic { ); return { transactionHashes: txHashes, startBlockNumber, endBlockNumber }; } catch (err) { - logger.error(err) + logger.error(err); logger.error(`Failed to get logs for ${eventName}, retrying next time`); RPC_LOGS_ERROR.inc({ type: scrapingType, event: eventName }); return { transactionHashes: [], startBlockNumber: null, endBlockNumber: null };