From 3f5d93778d87079b7d9a9454d4155605ea7fcfb4 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik <126621805+lcwik@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:37:32 -0800 Subject: [PATCH] [IND-481] Delete price cache in ender. (#815) More clean-up before migration to SQL based block processor. --- .../__tests__/caches/block-cache.test.ts | 5 - .../__tests__/caches/price-cache.test.ts | 133 ------------------ .../handlers/funding-handler.test.ts | 2 - .../market-price-update-handler.test.ts | 5 - .../services/ender/src/caches/block-cache.ts | 3 - .../services/ender/src/caches/price-cache.ts | 60 -------- .../markets/market-price-update-handler.ts | 2 - 7 files changed, 210 deletions(-) delete mode 100644 indexer/services/ender/__tests__/caches/price-cache.test.ts delete mode 100644 indexer/services/ender/src/caches/price-cache.ts diff --git a/indexer/services/ender/__tests__/caches/block-cache.test.ts b/indexer/services/ender/__tests__/caches/block-cache.test.ts index 303b61541b..3157a0f807 100644 --- a/indexer/services/ender/__tests__/caches/block-cache.test.ts +++ b/indexer/services/ender/__tests__/caches/block-cache.test.ts @@ -17,7 +17,6 @@ import { shouldSkipBlock, } from '../../src/caches/block-cache'; import { clearCandlesMap, getCandlesMap } from '../../src/caches/candle-cache'; -import { clearPriceMap, getPriceMap } from '../../src/caches/price-cache'; describe('block-cache', () => { beforeAll(async () => { @@ -34,7 +33,6 @@ describe('block-cache', () => { await dbHelpers.clearData(); resetBlockCache(); clearCandlesMap(); - clearPriceMap(); perpetualMarketRefresher.clear(); assetRefresher.clear(); marketRefresher.clear(); @@ -80,7 +78,6 @@ describe('block-cache', () => { // validate that block, candles, and price cache are updated expect(getCurrentBlockHeight()).toEqual('3'); expect(getCandlesMap()).not.toEqual({}); - expect(getPriceMap()).not.toEqual({}); expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({}); expect(assetRefresher.getAssetsMap()).not.toEqual({}); expect(marketRefresher.getMarketsMap()).not.toEqual({}); @@ -93,7 +90,6 @@ describe('block-cache', () => { // Validate that caches are empty expect(getCurrentBlockHeight()).toEqual('-1'); expect(getCandlesMap()).toEqual({}); - expect(getPriceMap()).toEqual({}); expect(perpetualMarketRefresher.getPerpetualMarketsMap()).toEqual({}); expect(assetRefresher.getAssetsMap()).toEqual({}); expect(marketRefresher.getMarketsMap()).toEqual({}); @@ -103,7 +99,6 @@ describe('block-cache', () => { // Validate that caches are populated expect(getCurrentBlockHeight()).toEqual('2'); expect(getCandlesMap()).not.toEqual({}); - expect(getPriceMap()).not.toEqual({}); expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({}); expect(assetRefresher.getAssetsMap()).not.toEqual({}); expect(marketRefresher.getMarketsMap()).not.toEqual({}); diff --git a/indexer/services/ender/__tests__/caches/price-cache.test.ts b/indexer/services/ender/__tests__/caches/price-cache.test.ts deleted file mode 100644 index 53141a58be..0000000000 --- a/indexer/services/ender/__tests__/caches/price-cache.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { - dbHelpers, - MarketTable, - OraclePriceFromDatabase, - OraclePriceTable, - PriceMap, - testConstants, - testMocks, - MarketCreateObject, -} from '@dydxprotocol-indexer/postgres'; -import _ from 'lodash'; -import { - clearPriceMap, - getPrice, - getPriceMap, - startPriceCache, - updatePriceCacheWithPrice, -} from '../../src/caches/price-cache'; - -describe('priceCache', () => { - beforeAll(async () => { - await dbHelpers.migrate(); - }); - - afterEach(async () => { - await dbHelpers.clearData(); - clearPriceMap(); - }); - - afterAll(async () => { - await dbHelpers.teardown(); - }); - - const priceFromDatabase: OraclePriceFromDatabase = { - ...testConstants.defaultOraclePrice, - id: testConstants.defaultOraclePriceId, - price: '5000', - }; - - it('uses both Markets table and Oracle prices to populate price cache', async () => { - await testMocks.seedData(); - const marketWithOraclePrice: MarketCreateObject = { - id: 127, - pair: 'NEAR-USD', - exponent: -9, - minPriceChangePpm: 1000, - oraclePrice: '30', - }; - const marketWithoutOraclePrice: MarketCreateObject = { - id: 128, - pair: 'UNI-USD', - exponent: -9, - minPriceChangePpm: 1000, - }; - await Promise.all([ - MarketTable.create(marketWithOraclePrice), - MarketTable.create(marketWithoutOraclePrice), - OraclePriceTable.create(testConstants.defaultOraclePrice), - OraclePriceTable.create(testConstants.defaultOraclePrice2), - ]); - await startPriceCache(testConstants.defaultBlock2.blockHeight); - - const map:PriceMap = getPriceMap(); - expect(_.size(map)).toEqual(4); - expect( - getPrice(0), - ).toEqual( - testConstants.defaultOraclePrice.price, - ); - expect( - getPrice(1), - ).toEqual( - testConstants.defaultOraclePrice2.price, - ); - expect( - getPrice(marketWithOraclePrice.id), - ).toEqual( - marketWithOraclePrice.oraclePrice, - ); - await expect(() => getPrice(marketWithoutOraclePrice.id)).toThrow( - new Error(`price not found for marketId ${marketWithoutOraclePrice.id} in price cache`), - ); - }); - - it('getPrice throws error on empty price cache', async () => { - await startPriceCache(testConstants.defaultBlock2.blockHeight); - - const map: PriceMap = getPriceMap(); - expect(_.size(map)).toEqual(0); - const marketId: number = 0; - await expect(() => getPrice(marketId)).toThrow( - new Error(`price not found for marketId ${marketId} in price cache`), - ); - }); - - it('successfully populates price cache', async () => { - await testMocks.seedData(); - await Promise.all([ - OraclePriceTable.create(testConstants.defaultOraclePrice), - OraclePriceTable.create(testConstants.defaultOraclePrice2), - ]); - await startPriceCache(testConstants.defaultBlock2.blockHeight); - - expect( - getPrice(0), - ).toEqual( - testConstants.defaultOraclePrice.price, - ); - expect( - getPrice(1), - ).toEqual( - testConstants.defaultOraclePrice2.price, - ); - }); - - it('successfully updates price cache', async () => { - await testMocks.seedData(); - await Promise.all([ - OraclePriceTable.create(testConstants.defaultOraclePrice), - OraclePriceTable.create(testConstants.defaultOraclePrice2), - ]); - await startPriceCache(testConstants.defaultBlock2.blockHeight); - updatePriceCacheWithPrice(priceFromDatabase); - - const map: PriceMap = getPriceMap(); - expect(_.size(map)).toEqual(3); - expect( - getPrice(0), - ).toEqual( - priceFromDatabase.price, - ); - }); -}); diff --git a/indexer/services/ender/__tests__/handlers/funding-handler.test.ts b/indexer/services/ender/__tests__/handlers/funding-handler.test.ts index d2404a59fd..8a5bb2883a 100644 --- a/indexer/services/ender/__tests__/handlers/funding-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/funding-handler.test.ts @@ -39,7 +39,6 @@ import { redis } from '@dydxprotocol-indexer/redis'; import Big from 'big.js'; import { redisClient } from '../../src/helpers/redis/redis-controller'; import { bigIntToBytes } from '@dydxprotocol-indexer/v4-proto-parser'; -import { startPriceCache } from '../../src/caches/price-cache'; import { createPostgresFunctions } from '../../src/helpers/postgres/postgres-functions'; describe('fundingHandler', () => { @@ -57,7 +56,6 @@ describe('fundingHandler', () => { await perpetualMarketRefresher.updatePerpetualMarkets(); await assetRefresher.updateAssets(); updateBlockCache(defaultPreviousHeight); - await startPriceCache(defaultPreviousHeight); }); afterEach(async () => { diff --git a/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts b/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts index 9a56111afd..77eb916dbe 100644 --- a/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/markets/market-price-update-handler.test.ts @@ -31,7 +31,6 @@ import { updateBlockCache } from '../../../src/caches/block-cache'; import { MarketEventV1, IndexerTendermintBlock, IndexerTendermintEvent } from '@dydxprotocol-indexer/v4-protos'; import { MarketPriceUpdateHandler } from '../../../src/handlers/markets/market-price-update-handler'; import Long from 'long'; -import { getPrice } from '../../../src/caches/price-cache'; import { createPostgresFunctions } from '../../../src/helpers/postgres/postgres-functions'; describe('marketPriceUpdateHandler', () => { @@ -148,8 +147,6 @@ describe('marketPriceUpdateHandler', () => { defaultHeight, ); - expect(getPrice(oraclePrice.marketId)).toEqual(oraclePrice.price); - const contents: MarketMessageContents = generateOraclePriceContents( oraclePrice, market.pair, @@ -202,8 +199,6 @@ describe('marketPriceUpdateHandler', () => { defaultHeight, ); - expect(getPrice(oraclePrice.marketId)).toEqual(oraclePrice.price); - const contents: MarketMessageContents = generateOraclePriceContents( oraclePrice, market.pair, diff --git a/indexer/services/ender/src/caches/block-cache.ts b/indexer/services/ender/src/caches/block-cache.ts index 27bcbab224..2690c2486e 100644 --- a/indexer/services/ender/src/caches/block-cache.ts +++ b/indexer/services/ender/src/caches/block-cache.ts @@ -12,7 +12,6 @@ import Big from 'big.js'; import config from '../config'; import { startCandleCache } from './candle-cache'; -import { startPriceCache } from './price-cache'; const INITIAL_BLOCK_HEIGHT: string = '-1'; @@ -114,8 +113,6 @@ export async function initializeAllCaches(): Promise { assetRefresher.updateAssets({ txId }), marketRefresher.updateMarkets({ txId }), ]); - // Must be run after startBlockCache() because it uses the block cache. - await startPriceCache(getCurrentBlockHeight(), txId); await Transaction.rollback(txId); } diff --git a/indexer/services/ender/src/caches/price-cache.ts b/indexer/services/ender/src/caches/price-cache.ts deleted file mode 100644 index 8b252a166a..0000000000 --- a/indexer/services/ender/src/caches/price-cache.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { logger } from '@dydxprotocol-indexer/base'; -import { - Transaction, - OraclePriceFromDatabase, - OraclePriceTable, - PriceMap, - MarketTable, - MarketFromDatabase, -} from '@dydxprotocol-indexer/postgres'; - -let priceMap: PriceMap = {}; - -// PriceCache should be started after BlockCache is started. -export async function startPriceCache(blockHeight: string, txId?: number): Promise { - const markets: MarketFromDatabase[] = await MarketTable.findAll({}, [], { txId }); - priceMap = await OraclePriceTable.findLatestPrices( - blockHeight, - Transaction.get(txId), - ); - markets.forEach((market: MarketFromDatabase) => { - if ( - priceMap[market.id] === undefined && - market.oraclePrice !== null && - market.oraclePrice !== undefined - ) { - priceMap[market.id] = market.oraclePrice; - } - }); -} - -export function getPrice( - marketId: number, -): string { - if (marketId in priceMap) { - return priceMap[marketId]; - } else { - logger.crit({ - at: 'PriceCache.getPrice', - message: 'price not found', - marketId, - }); - throw Error(`price not found for marketId ${marketId} in price cache`); - } -} - -export function updatePriceCacheWithPrice(oraclePrice: OraclePriceFromDatabase): void { - priceMap[oraclePrice.marketId] = oraclePrice.price; -} - -export function getPriceMap(): PriceMap { - return priceMap; -} - -export function clearPriceMap(): void { - if (process.env.NODE_ENV !== 'test') { - throw Error('cannot clear price map outside of test environment'); - } - - priceMap = {}; -} diff --git a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts index eea8a07d4f..03ac35cd76 100644 --- a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts @@ -8,7 +8,6 @@ import { import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; -import { updatePriceCacheWithPrice } from '../../caches/price-cache'; import { generateOraclePriceContents } from '../../helpers/kafka-helper'; import { ConsolidatedKafkaEvent, @@ -64,7 +63,6 @@ export class MarketPriceUpdateHandler extends Handler { result.rows[0].result.oracle_price) as OraclePriceFromDatabase; marketRefresher.updateMarket(market); - updatePriceCacheWithPrice(oraclePrice); return [ this.generateKafkaEvent(