[IND-481] Remove market cache. (#818)

* [IND-481] Remove market cache. More clean-up in preparation of migration to SQL based block processor.
dydxprotocol · Nov 29, 2023 · 1973bc2 · 1973bc2
1 parent f9b92e5
commit 1973bc2
Show file tree

Hide file tree

Showing 15 changed files with 7 additions and 151 deletions.
diff --git a/indexer/packages/postgres/__tests__/loops/market-refresher.test.ts b/indexer/packages/postgres/__tests__/loops/market-refresher.test.ts
diff --git a/indexer/packages/postgres/src/index.ts b/indexer/packages/postgres/src/index.ts
@@ -36,7 +36,6 @@ export * as ComplianceTable from './stores/compliance-table';
 
 export * as perpetualMarketRefresher from './loops/perpetual-market-refresher';
 export * as assetRefresher from './loops/asset-refresher';
-export * as marketRefresher from './loops/market-refresher';
 export * as liquidityTierRefresher from './loops/liquidity-tier-refresher';
 
 export * as uuid from './helpers/uuid';

diff --git a/indexer/packages/postgres/src/loops/market-refresher.ts b/indexer/packages/postgres/src/loops/market-refresher.ts
diff --git a/indexer/services/ender/__tests__/caches/block-cache.test.ts b/indexer/services/ender/__tests__/caches/block-cache.test.ts
@@ -4,7 +4,6 @@ import {
   OraclePriceTable,
   assetRefresher,
   dbHelpers,
-  marketRefresher,
   perpetualMarketRefresher,
   testConstants,
   testMocks,
@@ -35,7 +34,6 @@ describe('block-cache', () => {
     clearCandlesMap();
     perpetualMarketRefresher.clear();
     assetRefresher.clear();
-    marketRefresher.clear();
   });
 
   afterAll(async () => {
@@ -80,7 +78,6 @@ describe('block-cache', () => {
         expect(getCandlesMap()).not.toEqual({});
         expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({});
         expect(assetRefresher.getAssetsMap()).not.toEqual({});
-        expect(marketRefresher.getMarketsMap()).not.toEqual({});
       }
     });
   });
@@ -92,7 +89,6 @@ describe('block-cache', () => {
       expect(getCandlesMap()).toEqual({});
       expect(perpetualMarketRefresher.getPerpetualMarketsMap()).toEqual({});
       expect(assetRefresher.getAssetsMap()).toEqual({});
-      expect(marketRefresher.getMarketsMap()).toEqual({});
 
       await initializeAllCaches();
 
@@ -101,7 +97,6 @@ describe('block-cache', () => {
       expect(getCandlesMap()).not.toEqual({});
       expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({});
       expect(assetRefresher.getAssetsMap()).not.toEqual({});
-      expect(marketRefresher.getMarketsMap()).not.toEqual({});
     });
   });
 });
diff --git a/indexer/services/ender/__tests__/handlers/asset-handler.test.ts b/indexer/services/ender/__tests__/handlers/asset-handler.test.ts
@@ -12,7 +12,6 @@ import {
   MarketTable,
   Ordering,
   testConstants,
-  marketRefresher,
   BlockTable,
   TendermintEventTable,
   assetRefresher,
@@ -112,7 +111,6 @@ describe('assetHandler', () => {
 
   it('creates new asset', async () => {
     await MarketTable.create(testConstants.defaultMarket);
-    await marketRefresher.updateMarkets();
     const transactionIndex: number = 0;
 
     const assetEvent: AssetCreateEventV1 = defaultAssetCreateEvent;

diff --git a/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts b/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts
@@ -12,7 +12,6 @@ import {
   MarketTable,
   Ordering,
   testConstants,
-  marketRefresher,
   BlockTable,
   TendermintEventTable,
   perpetualMarketRefresher,
@@ -115,7 +114,6 @@ describe('perpetualMarketHandler', () => {
 
   it('fails when liquidity tier doesnt exist for perpetual market', async () => {
     await MarketTable.create(testConstants.defaultMarket);
-    await marketRefresher.updateMarkets();
     const transactionIndex: number = 0;
     const kafkaMessage: KafkaMessage = createKafkaMessageFromPerpetualMarketEvent({
       perpetualMarketEvent: defaultPerpetualMarketCreateEvent,
@@ -134,7 +132,6 @@ describe('perpetualMarketHandler', () => {
       LiquidityTiersTable.create(testConstants.defaultLiquidityTier),
     ]);
     await liquidityTierRefresher.updateLiquidityTiers();
-    await marketRefresher.updateMarkets();
 
     const transactionIndex: number = 0;
 

diff --git a/indexer/services/ender/__tests__/lib/on-message.test.ts b/indexer/services/ender/__tests__/lib/on-message.test.ts
@@ -6,7 +6,6 @@ import {
   dbHelpers,
   IsoString,
   LiquidityTiersTable,
-  marketRefresher,
   MarketTable,
   perpetualMarketRefresher,
   PerpetualMarketTable,
@@ -893,13 +892,11 @@ describe('on-message', () => {
     // Initialize assetRefresher
     await assetRefresher.updateAssets();
     await perpetualMarketRefresher.updatePerpetualMarkets();
-    await marketRefresher.updateMarkets();
     (SubaccountUpdateHandler as jest.Mock).mockReturnValue({
       handle: () => {
         // clear cache so we can confirm that the cache is updated after the error
         assetRefresher.clear();
         perpetualMarketRefresher.clear();
-        marketRefresher.clear();
         throw new Error();
       },
       validate: () => null,
@@ -909,7 +906,6 @@ describe('on-message', () => {
 
     expect(assetRefresher.getAssetsMap()).not.toEqual({});
     expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({});
-    expect(marketRefresher.getMarketsMap()).not.toEqual({});
   });
 });
 

diff --git a/indexer/services/ender/__tests__/validators/asset-validator.test.ts b/indexer/services/ender/__tests__/validators/asset-validator.test.ts
@@ -1,6 +1,6 @@
 import { logger } from '@dydxprotocol-indexer/base';
 import { AssetCreateEventV1, IndexerTendermintBlock, IndexerTendermintEvent } from '@dydxprotocol-indexer/v4-protos';
-import { dbHelpers, marketRefresher, testMocks } from '@dydxprotocol-indexer/postgres';
+import { dbHelpers, testMocks } from '@dydxprotocol-indexer/postgres';
 import { DydxIndexerSubtypes } from '../../src/lib/types';
 import {
   defaultAssetCreateEvent, defaultHeight, defaultTime, defaultTxHash,
@@ -15,7 +15,6 @@ import { AssetValidator } from '../../src/validators/asset-validator';
 describe('asset-validator', () => {
   beforeEach(async () => {
     await testMocks.seedData();
-    await marketRefresher.updateMarkets();
     jest.spyOn(logger, 'error');
   });
 

diff --git a/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts b/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts
@@ -1,7 +1,7 @@
 import { logger, ParseMessageError } from '@dydxprotocol-indexer/base';
 import { PerpetualMarketCreateEventV1, IndexerTendermintBlock, IndexerTendermintEvent } from '@dydxprotocol-indexer/v4-protos';
 import {
-  dbHelpers, marketRefresher, testMocks, perpetualMarketRefresher,
+  dbHelpers, testMocks, perpetualMarketRefresher,
 } from '@dydxprotocol-indexer/postgres';
 import { DydxIndexerSubtypes } from '../../src/lib/types';
 import {
@@ -18,7 +18,6 @@ import Long from 'long';
 describe('perpetual-market-validator', () => {
   beforeEach(async () => {
     await testMocks.seedData();
-    await marketRefresher.updateMarkets();
     jest.spyOn(logger, 'error');
   });
 

diff --git a/indexer/services/ender/src/caches/block-cache.ts b/indexer/services/ender/src/caches/block-cache.ts
@@ -5,7 +5,6 @@ import {
   IsolationLevel,
   Transaction,
   assetRefresher,
-  marketRefresher,
   perpetualMarketRefresher,
 } from '@dydxprotocol-indexer/postgres';
 import Big from 'big.js';
@@ -111,7 +110,6 @@ export async function initializeAllCaches(): Promise<void> {
     startCandleCache(txId),
     perpetualMarketRefresher.updatePerpetualMarkets({ txId }),
     assetRefresher.updateAssets({ txId }),
-    marketRefresher.updateMarkets({ txId }),
   ]);
 
   await Transaction.rollback(txId);

diff --git a/indexer/services/ender/src/handlers/markets/market-create-handler.ts b/indexer/services/ender/src/handlers/markets/market-create-handler.ts
@@ -1,12 +1,8 @@
 import { logger } from '@dydxprotocol-indexer/base';
 import {
-  MarketFromDatabase,
-  MarketModel,
-  marketRefresher,
   storeHelpers,
 } from '@dydxprotocol-indexer/postgres';
 import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos';
-import * as pg from 'pg';
 
 import { ConsolidatedKafkaEvent, MarketCreateEventMessage } from '../../lib/types';
 import { Handler } from '../handler';
@@ -27,7 +23,7 @@ export class MarketCreateHandler extends Handler<MarketEventV1> {
     });
 
     const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes;
-    const result: pg.QueryResult = await storeHelpers.rawQuery(
+    await storeHelpers.rawQuery(
       `SELECT dydx_market_create_handler(
         '${JSON.stringify(MarketEventV1.decode(eventDataBinary))}'
       ) AS result;`,
@@ -50,9 +46,6 @@ export class MarketCreateHandler extends Handler<MarketEventV1> {
       throw error;
     });
 
-    const market: MarketFromDatabase = MarketModel.fromJson(
-      result.rows[0].result.market) as MarketFromDatabase;
-    marketRefresher.updateMarket(market);
     return [];
   }
 }
diff --git a/indexer/services/ender/src/handlers/markets/market-modify-handler.ts b/indexer/services/ender/src/handlers/markets/market-modify-handler.ts
@@ -1,9 +1,6 @@
 import { logger } from '@dydxprotocol-indexer/base';
-import {
-  MarketFromDatabase, marketRefresher, storeHelpers, MarketModel,
-} from '@dydxprotocol-indexer/postgres';
+import { storeHelpers } from '@dydxprotocol-indexer/postgres';
 import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos';
-import * as pg from 'pg';
 
 import { ConsolidatedKafkaEvent, MarketModifyEventMessage } from '../../lib/types';
 import { Handler } from '../handler';
@@ -24,7 +21,7 @@ export class MarketModifyHandler extends Handler<MarketEventV1> {
     });
 
     const eventDataBinary: Uint8Array = this.indexerTendermintEvent.dataBytes;
-    const result: pg.QueryResult = await storeHelpers.rawQuery(
+    await storeHelpers.rawQuery(
       `SELECT dydx_market_modify_handler(
         '${JSON.stringify(MarketEventV1.decode(eventDataBinary))}' 
       ) AS result;`,
@@ -52,9 +49,6 @@ export class MarketModifyHandler extends Handler<MarketEventV1> {
       );
     });
 
-    const market: MarketFromDatabase = MarketModel.fromJson(
-      result.rows[0].result.market) as MarketFromDatabase;
-    marketRefresher.updateMarket(market);
     return [];
   }
 }
diff --git a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts
@@ -3,7 +3,7 @@ import {
   MarketFromDatabase,
   OraclePriceFromDatabase,
   OraclePriceModel,
-  MarketMessageContents, storeHelpers, MarketModel, marketRefresher,
+  MarketMessageContents, storeHelpers, MarketModel,
 } from '@dydxprotocol-indexer/postgres';
 import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos';
 import * as pg from 'pg';
@@ -62,8 +62,6 @@ export class MarketPriceUpdateHandler extends Handler<MarketEventV1> {
     const oraclePrice: OraclePriceFromDatabase = OraclePriceModel.fromJson(
       result.rows[0].result.oracle_price) as OraclePriceFromDatabase;
 
-    marketRefresher.updateMarket(market);
-
     return [
       this.generateKafkaEvent(
         oraclePrice, market.pair,

diff --git a/indexer/services/ender/src/index.ts b/indexer/services/ender/src/index.ts
@@ -1,7 +1,7 @@
 import { logger, startBugsnag, wrapBackgroundTask } from '@dydxprotocol-indexer/base';
 import { stopConsumer, startConsumer } from '@dydxprotocol-indexer/kafka';
 import {
-  assetRefresher, perpetualMarketRefresher, marketRefresher, liquidityTierRefresher,
+  assetRefresher, perpetualMarketRefresher, liquidityTierRefresher,
 } from '@dydxprotocol-indexer/postgres';
 
 import { initializeAllCaches } from './caches/block-cache';
@@ -24,7 +24,6 @@ async function startKafka(): Promise<void> {
   await Promise.all([
     perpetualMarketRefresher.updatePerpetualMarkets(),
     assetRefresher.updateAssets(),
-    marketRefresher.updateMarkets(),
     liquidityTierRefresher.updateLiquidityTiers(),
   ]);
   // Ender does not need to refresh its caches in a loop because Ender is the only service that

diff --git a/indexer/services/ender/src/lib/cache-manager.ts b/indexer/services/ender/src/lib/cache-manager.ts
@@ -2,7 +2,6 @@ import {
   IsolationLevel,
   Transaction,
   assetRefresher,
-  marketRefresher,
   perpetualMarketRefresher,
 } from '@dydxprotocol-indexer/postgres';
 
@@ -13,7 +12,6 @@ export async function refreshDataCaches(): Promise<void> {
   await Promise.all([
     perpetualMarketRefresher.updatePerpetualMarkets({ txId, readReplica: true }),
     assetRefresher.updateAssets({ txId, readReplica: true }),
-    marketRefresher.updateMarkets({ txId, readReplica: true }),
   ]);
 
   await Transaction.rollback(txId);