Test Polygon

0xProject · Jan 17, 2024 · 7044e58 · 7044e58
1 parent 7151b19
commit 7044e58
Show file tree

Hide file tree

Showing 24 changed files with 397 additions and 307 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
-.env
+.env*
+!.env.example
 mounts/
 node_modules
 lib

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -118,7 +118,7 @@ services:
             dockerfile: Dockerfile
         restart: always
         environment:
-            EVM_RPC_URL: '${RPC_URL_POLYGON}'
+            EVM_RPC_URL: '${EVM_RPC_URL_POLYGON}'
             CHAIN_ID: '137'
             POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
             SCHEMA: 'events_polygon'

diff --git a/migrations/1705420603000-NewBackfillTables.ts b/migrations/1705420603000-NewBackfillTables.ts
@@ -0,0 +1,25 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class NewBackfillTables1705420603000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.events_backfill RENAME TO backfill_events;
+
+          CREATE TABLE ${schema}.backfill_blocks (
+            block_number BIGINT NOT NULL,
+            PRIMARY KEY (block_number)
+          );
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          DROP TABLE ${schema}.backfill_blocks;
+          ALTER TABLE ${schema}.backfill_events RENAME TO events_backfill;
+        `);
+    }
+}
diff --git a/migrations/ethereum/1615870349094-AddV4CancelAndMultiplexEvents.ts b/migrations/ethereum/1615870349094-AddV4CancelAndMultiplexEvents.ts
@@ -1,81 +1,53 @@
-import { MigrationInterface, QueryRunner, Table } from 'typeorm';
-
-const eventsV4CancelEvents = new Table({
-    name: 'events.v4_cancel_events',
-    columns: [
-        { name: 'observed_timestamp', type: 'bigint' },
-        { name: 'contract_address', type: 'varchar' },
-        { name: 'transaction_hash', type: 'varchar', isPrimary: true },
-        { name: 'transaction_index', type: 'bigint' },
-        { name: 'log_index', type: 'bigint', isPrimary: true },
-        { name: 'block_hash', type: 'varchar' },
-        { name: 'block_number', type: 'bigint' },
-
-        { name: 'maker', type: 'varchar' },
-        { name: 'order_hash', type: 'varchar' },
-    ],
-});
-
-const eventsExpiredRfqOrderEvents = new Table({
-    name: 'events.expired_rfq_order_events',
-    columns: [
-        { name: 'observed_timestamp', type: 'bigint' },
-        { name: 'contract_address', type: 'varchar' },
-        { name: 'transaction_hash', type: 'varchar', isPrimary: true },
-        { name: 'transaction_index', type: 'bigint' },
-        { name: 'log_index', type: 'bigint', isPrimary: true },
-        { name: 'block_hash', type: 'varchar' },
-        { name: 'block_number', type: 'bigint' },
-
-        { name: 'maker', type: 'varchar' },
-        { name: 'order_hash', type: 'varchar' },
-        { name: 'expiry', type: 'numeric' },
-    ],
-});
-
-const indexQuery = `
-
-    CREATE INDEX v4_cancel_events_transaction_hash_index
-        ON events.v4_cancel_events (transaction_hash);
-    CREATE INDEX v4_cancel_events_block_number_index
-        ON events.v4_cancel_events (block_number);
-    CREATE INDEX v4_cancel_events_maker_index
-        ON events.v4_cancel_events (maker);
-    CREATE INDEX v4_cancel_events_order_hash_index
-        ON events.v4_cancel_events (order_hash);
-
-    
-    CREATE INDEX expired_rfq_events_block_number_index
-        ON events.expired_rfq_order_events (block_number);
-    CREATE INDEX expired_rfq_events_transaction_hash_index
-        ON events.expired_rfq_order_events (transaction_hash);
-    CREATE INDEX expired_rfq_events_maker_index
-        ON events.expired_rfq_order_events (maker);
-    CREATE INDEX expired_rfq_events_order_hash_index
-        ON events.v4_cancel_events (order_hash);
-`;
-
-const dropIndexQuery = `
-    DROP INDEX events.v4_cancel_events_transaction_hash_index;
-    DROP INDEX events.v4_cancel_events_block_number_index;
-    DROP INDEX events.v4_cancel_events_maker_index;
-    DROP INDEX events.v4_cancel_events_order_hash_index;
-    DROP INDEX events.expired_rfq_events_block_number_index;
-    DROP INDEX events.expired_rfq_events_transaction_hash_index;
-    DROP INDEX events.expired_rfq_events_maker_index;
-    DROP INDEX events.expired_rfq_events_order_hash_index;
-`;
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
 
 export class AddV4CancelAndMultiplexEvents1615870349094 implements MigrationInterface {
     public async up(queryRunner: QueryRunner): Promise<any> {
-        await queryRunner.createTable(eventsV4CancelEvents);
-        await queryRunner.createTable(eventsExpiredRfqOrderEvents);
-        await queryRunner.query(indexQuery);
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          CREATE TABLE ${schema}.v4_cancel_events (
+            observed_timestamp int8 NOT NULL,
+            contract_address varchar NOT NULL,
+            transaction_hash varchar NOT NULL,
+            transaction_index int8 NOT NULL,
+            log_index int8 NOT NULL,
+            block_hash varchar NOT NULL,
+            block_number int8 NOT NULL,
+            maker varchar NOT NULL,
+            order_hash varchar NOT NULL,
+            CONSTRAINT v4_cancel_events_pk PRIMARY KEY (transaction_hash, log_index)
+          );
+          CREATE INDEX v4_xancel_events_order_hash_index ON ${schema}.v4_cancel_events USING btree (order_hash);
+          CREATE INDEX v4_cancel_events_block_number_index ON ${schema}.v4_cancel_events USING btree (block_number);
+          CREATE INDEX v4_cancel_events_maker_index ON ${schema}.v4_cancel_events USING btree (maker);
+          CREATE INDEX v4_cancel_events_order_hash_index ON ${schema}.v4_cancel_events USING btree (order_hash);
+          CREATE INDEX v4_cancel_events_transaction_hash_index ON ${schema}.v4_cancel_events USING btree (transaction_hash);
+
+          CREATE TABLE ${schema}.expired_rfq_order_events (
+            observed_timestamp int8 NOT NULL,
+            contract_address varchar NOT NULL,
+            transaction_hash varchar NOT NULL,
+            transaction_index int8 NOT NULL,
+            log_index int8 NOT NULL,
+            block_hash varchar NOT NULL,
+            block_number int8 NOT NULL,
+            maker varchar NOT NULL,
+            order_hash varchar NOT NULL,
+            expiry numeric NOT NULL,
+            CONSTRAINT expired_rfq_order_events_pk PRIMARY KEY (transaction_hash, log_index)
+          );
+          CREATE INDEX expired_rfq_events_block_number_index ON ${schema}.expired_rfq_order_events USING btree (block_number);
+          CREATE INDEX expired_rfq_events_maker_index ON ${schema}.expired_rfq_order_events USING btree (maker);
+          CREATE INDEX expired_rfq_events_transaction_hash_index ON ${schema}.expired_rfq_order_events USING btree (transaction_hash);
+`);
     }
 
     public async down(queryRunner: QueryRunner): Promise<any> {
-        await queryRunner.query(dropIndexQuery);
-        await queryRunner.dropTable(eventsExpiredRfqOrderEvents);
-        await queryRunner.dropTable(eventsV4CancelEvents);
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          DROP TABLE ${schema}.v4_cancel_events;
+          DROP TABLE ${schema}.expired_rfq_order_events;
+        `);
     }
 }
diff --git a/src/abis.ts b/src/abis.ts
@@ -24,6 +24,31 @@ export const EXPIRED_RFQ_ORDER_ABI = {
     type: 'event',
 };
 
+export const V3_CANCEL_ABI = {
+    anonymous: false,
+    inputs: [
+        { indexed: true, internalType: 'address', name: 'makerAddress', type: 'address' },
+        { indexed: true, internalType: 'address', name: 'feeRecipientAddress', type: 'address' },
+        { indexed: false, internalType: 'bytes', name: 'makerAssetData', type: 'bytes' },
+        { indexed: false, internalType: 'bytes', name: 'takerAssetData', type: 'bytes' },
+        { indexed: false, internalType: 'address', name: 'senderAddress', type: 'address' },
+        { indexed: true, internalType: 'bytes32', name: 'orderHash', type: 'bytes32' },
+    ],
+    name: 'Cancel',
+    type: 'event',
+};
+
+export const V3_CANCEL_UP_TO_ABI = {
+    anonymous: false,
+    inputs: [
+        { indexed: true, internalType: 'address', name: 'makerAddress', type: 'address' },
+        { indexed: true, internalType: 'address', name: 'orderSenderAddress', type: 'address' },
+        { indexed: false, internalType: 'uint256', name: 'orderEpoch', type: 'uint256' },
+    ],
+    name: 'CancelUpTo',
+    type: 'event',
+};
+
 export const V4_CANCEL_ABI = {
     anonymous: false,
     inputs: [

diff --git a/src/config.ts b/src/config.ts
@@ -24,6 +24,7 @@ import {
     DEFAULT_FEAT_UNISWAP_V3_POOL_CREATED_EVENT,
     DEFAULT_FEAT_UNISWAP_V3_SWAP_EVENT,
     DEFAULT_FEAT_UNISWAP_V3_VIP_SWAP_EVENT,
+    DEFAULT_FEAT_V3_CANCEL_EVENTS,
     DEFAULT_FEAT_V3_FILL_EVENT,
     DEFAULT_FEAT_V3_NATIVE_FILL,
     DEFAULT_FEAT_WRAP_UNWRAP_NATIVE_EVENT,
@@ -68,6 +69,8 @@ interface BridgeContract {
     startingBlock: number;
 }
 
+export const RESCRAPE_BLOCKS = 10; //TODO: delete me
+
 const bridgeContracts = [
     { contract: '0x1c29670f7a77f1052d30813a0a4f632c78a02610', startingBlock: 9613431 },
     { contract: '0x991c745401d5b5e469b8c3e2cb02c748f08754f1', startingBlock: 9613441 },
@@ -269,6 +272,7 @@ validateAddress(
 );
 
 export const FEAT_V3_FILL_EVENT = getBoolConfig('FEAT_V3_FILL_EVENT', DEFAULT_FEAT_V3_FILL_EVENT);
+export const FEAT_V3_CANCEL_EVENTS = getBoolConfig('FEAT_V3_CANCEL_EVENTS', DEFAULT_FEAT_V3_CANCEL_EVENTS);
 
 export const FEAT_OTC_ORDERS = getBoolConfig('FEAT_OTC_ORDERS', DEFAULT_FEAT_OTC_ORDERS);
 
@@ -370,15 +374,6 @@ export const FEAT_TOKENS_FROM_TRANSFERS = getBoolConfig(
     DEFAULT_FEAT_TOKENS_FROM_TRANSFERS,
 );
 
-export const TOKENS_FROM_TRANSFERS_START_BLOCK = getIntConfig('TOKENS_FROM_TRANSFERS_START_BLOCK', -1);
-
-validateStartBlock(
-    'TOKENS_FROM_TRANSFERS_START_BLOCK',
-    TOKENS_FROM_TRANSFERS_START_BLOCK,
-    'FEAT_TOKENS_FROM_TRANSFERS',
-    FEAT_TOKENS_FROM_TRANSFERS,
-);
-
 export const FEAT_META_TRANSACTION_EXECUTED_EVENT = getBoolConfig(
     'FEAT_META_TRANSACTION_EXECUTED_EVENT',
     DEFAULT_FEAT_META_TRANSACTION_EXECUTED_EVENT,

diff --git a/src/constants.ts b/src/constants.ts
@@ -33,6 +33,7 @@ export const DEFAULT_FEAT_UNISWAP_V2_VIP_SWAP_EVENT = false;
 export const DEFAULT_FEAT_UNISWAP_V3_VIP_SWAP_EVENT = false;
 export const DEFAULT_FEAT_UNISWAP_V3_SWAP_EVENT = false;
 export const DEFAULT_FEAT_UNISWAP_V3_POOL_CREATED_EVENT = false;
+export const DEFAULT_FEAT_V3_CANCEL_EVENTS = false;
 export const DEFAULT_FEAT_V3_FILL_EVENT = false;
 export const DEFAULT_FEAT_V3_NATIVE_FILL = false;
 export const DEFAULT_FEAT_VIP_SWAP_EVENT = false;
@@ -59,6 +60,7 @@ export const UNISWAP_V2_PAIR_CREATED_TOPIC = ['0x0d3648bd0f6ba80134a33ba9275ac58
 export const UNISWAP_V2_SYNC_TOPIC = ['0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1'];
 
 export const V3_EXCHANGE_ADDRESS = '0x61935cbdd02287b511119ddb11aeb42f1593b7ef';
+export const V3_DEPLOYMENT_BLOCK = 8952139;
 
 export const UNISWAP_V2_SWAP_EVENT_TOPIC_0 = '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822';
 export const UNISWAP_V3_SWAP_EVENT_TOPIC_0 = '0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67';
@@ -81,8 +83,6 @@ export const ERC1155_ORDER_CANCELLED_EVENT_TOPIC = [
     '0x81b6de71b4c5058b59a7b56dc73297dd4820029a7229cf7b8e9680d73ff9bab0',
 ];
 
-export const TOKEN_TRANSFER_EVENT_TOPIC = ['0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'];
-
 export const ERC165_SUPPORTS_INTERFACE_SELECTOR = '01ffc9a7';
 export const ERC165_ERC721_INTERFACE = '80ac58cd';
 export const ERC165_ERC1155_INTERFACE = 'd9b67a26';
@@ -113,3 +113,6 @@ export const UNWRAP_NATIVE_EVENT_TOPIC = ['0x7fcf532c15f0a6db0bd6d0e038bea71d30d
 export const TRANSFER_EVENT_TOPIC_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef';
 
 export const ZEROEX_API_AFFILIATE_SELECTOR = '869584cd';
+
+export const V3_CANCEL_EVENT_TOPIC = ['0x02c310a9a43963ff31a754a4099cc435ed498049687539d72d7818d9b093415c'];
+export const V3_CANCEL_UP_TO_EVENT_TOPIC = ['0x82af639571738f4ebd4268fb0363d8957ebe1bbb9e78dba5ebd69eed39b154f0'];
diff --git a/src/data_sources/events/web3.ts b/src/data_sources/events/web3.ts
@@ -16,7 +16,7 @@ const helpers = require('web3-core-helpers');
 const formatter = helpers.formatters;
 
 export interface LogPullInfo {
-    address: string;
+    address: string | null;
     fromBlock: number;
     toBlock: number;
     topics: (string | null)[];
@@ -62,18 +62,36 @@ export class Web3Source {
         });
     }
 
+    public async getBatchBlockInfoForRangeAsync<B extends boolean>(
+        startBlock: number,
+        endBlock: number,
+        includeTransactions: B,
+    ): Promise<B extends true ? BlockWithTransactionData1559[] : BlockWithoutTransactionData1559[]>;
+
     public async getBatchBlockInfoForRangeAsync(
         startBlock: number,
         endBlock: number,
         includeTransactions: boolean,
-    ): Promise<any[]> {
-        const iter = Array.from(Array(endBlock - startBlock + 1).keys());
+    ): Promise<(BlockWithoutTransactionData | BlockWithTransactionData1559)[]> {
+        const blockNumbers = Array.from(Array(endBlock - startBlock + 1).keys()).map((i) => i + startBlock);
+        return this.getBatchBlockInfoAsync(blockNumbers, includeTransactions);
+    }
+
+    public async getBatchBlockInfoAsync<B extends boolean>(
+        blockNumbers: number[],
+        includeTransactions: B,
+    ): Promise<B extends true ? BlockWithTransactionData1559[] : BlockWithoutTransactionData1559[]>;
+
+    public async getBatchBlockInfoAsync(
+        blockNumbers: number[],
+        includeTransactions: boolean,
+    ): Promise<(BlockWithoutTransactionData | BlockWithTransactionData1559)[]> {
         const batch = new this._web3.BatchRequest();
 
-        const promises = iter.map((i) => {
-            return new Promise((resolve, reject) => {
+        const promises = blockNumbers.map((blockNumber) => {
+            return new Promise<BlockWithoutTransactionData | BlockWithTransactionData1559>((resolve, reject) => {
                 const req = this._web3.eth.getBlock.request(
-                    i + startBlock,
+                    blockNumber,
                     includeTransactions,
                     (err: any, data: BlockWithTransactionData1559) => {
                         if (err) {
@@ -97,17 +115,20 @@ export class Web3Source {
         startBlock: number,
         endBlock: number,
     ): Promise<TransactionReceipt1559[][]> {
-        const iter = Array.from(Array(endBlock - startBlock + 1).keys());
+        const blockNumbers = Array.from(Array(endBlock - startBlock + 1).keys()).map((i) => i + startBlock);
+        return this.getBatchBlockReceiptsAsync(blockNumbers);
+    }
 
-        const promises = iter.map((i) => {
-            return this._web3.eth.getBlockReceipts(i + startBlock).catch((err: any) => {
+    public async getBatchBlockReceiptsAsync(blockNumbers: number[]): Promise<TransactionReceipt1559[][]> {
+        const promises = blockNumbers.map((blockNumber) => {
+            return this._web3.eth.getBlockReceipts(blockNumber).catch((err: any) => {
                 logger.error(`Blocks error: ${err}`);
             });
         });
 
         const blocks = await Promise.all(promises);
 
-        return blocks as TransactionReceipt1559[][];
+        return blocks;
     }
 
     public async getBatchTxInfoAsync(hashes: string[]): Promise<any[]> {

diff --git a/src/entities/index.ts b/src/entities/index.ts
@@ -35,7 +35,6 @@ export { StakingPoolMetadata } from './staking_pool_metadata';
 export { StakingProxyDeployment } from './staking_proxy_deployment';
 export { TokenMetadata } from './token_metadata';
 export { TokenRegistry } from './token_registry';
-export { TokenTransferEvent } from './token_transfer_event';
 export { Transaction } from './transaction';
 export { TransactionExecutionEvent } from './transaction_execution_event';
 export { TransactionLogs } from './transaction_log';

diff --git a/src/entities/token_transfer_event.ts b/src/entities/token_transfer_event.ts