Skip to content

Commit

Permalink
[IND-557] fix occasional missing pnl ticks (#951)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dydxwill
dydxwill authored Jan 10, 2024
1 parent 4f2d72a commit 51cdad7
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 33 deletions.
35 changes: 35 additions & 0 deletions indexer/services/roundtable/__tests__/helpers/pnl-ticks-helper.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import {
getNewPnlTick,
getPnlTicksCreateObjects,
getUsdcTransfersSinceLastPnlTick,
getAccountsToUpdate,
normalizeStartTime,
} from '../../src/helpers/pnl-ticks-helper';
import { defaultPnlTickForSubaccounts } from '../../src/helpers/constants';
import Big from 'big.js';
Expand All @@ -35,6 +37,7 @@ import { ZERO } from '../../src/lib/constants';
import { SubaccountUsdcTransferMap } from '../../src/helpers/types';
import config from '../../src/config';
import _ from 'lodash';
import { ONE_HOUR_IN_MILLISECONDS } from '@dydxprotocol-indexer/base';

describe('pnl-ticks-helper', () => {
const positions: PerpetualPositionFromDatabase[] = [
Expand Down Expand Up @@ -228,6 +231,38 @@ describe('pnl-ticks-helper', () => {
}));
});

it('normalizeStartTime', () => {
const time: Date = new Date('2021-01-09T20:00:50.000Z');
// 1 hour
config.PNL_TICK_UPDATE_INTERVAL_MS = 1000 * 60 * 60;
const result1: Date = normalizeStartTime(time);
expect(result1.toISOString()).toBe('2021-01-09T20:00:00.000Z');
// 1 day
config.PNL_TICK_UPDATE_INTERVAL_MS = 1000 * 60 * 60 * 24;
const result2: Date = normalizeStartTime(time);
expect(result2.toISOString()).toBe('2021-01-09T00:00:00.000Z');
});

it('getAccountsToUpdate', () => {
const accountToLastUpdatedBlockTime: _.Dictionary<IsoString> = {
account1: '2024-01-01T10:00:00Z',
account2: '2024-01-01T11:00:00Z',
account3: '2024-01-01T11:01:00Z',
account4: '2024-01-01T11:10:00Z',
account5: '2024-01-01T12:00:00Z',
account6: '2024-01-01T12:00:10Z',
};
const blockTime: IsoString = '2024-01-01T12:01:00Z';
config.PNL_TICK_UPDATE_INTERVAL_MS = ONE_HOUR_IN_MILLISECONDS;

const expectedAccountsToUpdate: string[] = ['account1', 'account2', 'account3', 'account4'];
const accountsToUpdate: string[] = getAccountsToUpdate(
accountToLastUpdatedBlockTime,
blockTime,
);
expect(accountsToUpdate).toEqual(expectedAccountsToUpdate);
});

it('calculateEquity', () => {
const usdcPosition: Big = new Big('100');
const equity: Big = calculateEquity(
Expand Down
74 changes: 61 additions & 13 deletions indexer/services/roundtable/__tests__/tasks/create-pnl-ticks.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
FundingIndexUpdatesTable,
} from '@dydxprotocol-indexer/postgres';

import createPnlTicksTask, { normalizeStartTime } from '../../src/tasks/create-pnl-ticks';
import createPnlTicksTask from '../../src/tasks/create-pnl-ticks';
import { LatestAccountPnlTicksCache, PnlTickForSubaccounts, redis } from '@dydxprotocol-indexer/redis';
import { DateTime } from 'luxon';
import config from '../../src/config';
Expand All @@ -24,6 +24,14 @@ describe('create-pnl-ticks', () => {
[testConstants.defaultSubaccountId]: {
...testConstants.defaultPnlTick,
createdAt: DateTime.utc(2022, 6, 1, 0, 0, 0).toISO(),
blockTime: DateTime.utc(2022, 6, 1, 0, 0, 0).toISO(),
},
};
const existingPnlTicksNeedsUpdate: PnlTickForSubaccounts = {
[testConstants.defaultSubaccountId]: {
...testConstants.defaultPnlTick,
createdAt: DateTime.utc(2022, 5, 31, 23, 59, 0).toISO(),
blockTime: DateTime.utc(2022, 5, 31, 23, 59, 0).toISO(),
},
};
const dateTime: DateTime = DateTime.utc(2022, 6, 1, 0, 30, 0);
Expand Down Expand Up @@ -123,18 +131,6 @@ describe('create-pnl-ticks', () => {
);
});

it('normalizeStartTime', () => {
const time: Date = new Date('2021-01-09T20:00:50.000Z');
// 1 hour
config.PNL_TICK_UPDATE_INTERVAL_MS = 1000 * 60 * 60;
const result1: Date = normalizeStartTime(time);
expect(result1.toISOString()).toBe('2021-01-09T20:00:00.000Z');
// 1 day
config.PNL_TICK_UPDATE_INTERVAL_MS = 1000 * 60 * 60 * 24;
const result2: Date = normalizeStartTime(time);
expect(result2.toISOString()).toBe('2021-01-09T00:00:00.000Z');
});

it('succeeds with no prior pnl ticks and open perpetual positions', async () => {
const date: number = new Date(2023, 4, 18, 0, 0, 0).valueOf();
jest.spyOn(Date, 'now').mockImplementation(() => date);
Expand Down Expand Up @@ -180,6 +176,58 @@ describe('create-pnl-ticks', () => {
);
});

it(
'succeeds with prior pnl ticks and open perpetual positions, updates pnl correctly',
async () => {
const date: number = new Date(2023, 4, 18, 0, 0, 0).valueOf();
jest.spyOn(Date, 'now').mockImplementation(() => date);
config.PNL_TICK_UPDATE_INTERVAL_MS = 3_600_000;
jest.spyOn(DateTime, 'utc').mockImplementation(() => dateTime);
await LatestAccountPnlTicksCache.set(
existingPnlTicksNeedsUpdate,
redisClient,
);
await Promise.all([
PerpetualPositionTable.create(testConstants.defaultPerpetualPosition),
PerpetualPositionTable.create({
...testConstants.defaultPerpetualPosition,
perpetualId: testConstants.defaultPerpetualMarket2.id,
openEventId: testConstants.defaultTendermintEventId2,
}),
]);
await createPnlTicksTask();
const pnlTicks: PnlTicksFromDatabase[] = await PnlTicksTable.findAll(
{},
[],
{},
);
expect(pnlTicks.length).toEqual(2);
expect(pnlTicks).toEqual(
expect.arrayContaining([
{
id: PnlTicksTable.uuid(testConstants.defaultSubaccountId2, dateTime.toISO()),
createdAt: dateTime.toISO(),
blockHeight: '5',
blockTime: testConstants.defaultBlock.time,
equity: '0.000000',
netTransfers: '20.500000',
subaccountId: testConstants.defaultSubaccountId2,
totalPnl: '-20.500000',
},
{
id: PnlTicksTable.uuid(testConstants.defaultSubaccountId, dateTime.toISO()),
createdAt: dateTime.toISO(),
blockHeight: '5',
blockTime: testConstants.defaultBlock.time,
equity: '105000.000000',
netTransfers: '-20.500000',
subaccountId: testConstants.defaultSubaccountId,
totalPnl: '105020.500000',
},
]),
);
});

it(
'succeeds with prior pnl ticks and open perpetual positions, respects PNL_TICK_UPDATE_INTERVAL_MS',
async () => {
Expand Down
55 changes: 47 additions & 8 deletions indexer/services/roundtable/src/helpers/pnl-ticks-helper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
AssetPositionTable,
FundingIndexMap,
FundingIndexUpdatesTable,
helpers,
IsoString,
OraclePriceTable,
PerpetualPositionFromDatabase,
Expand All @@ -15,7 +16,6 @@ import {
SubaccountTable,
SubaccountToPerpetualPositionsMap,
TransferTable,
helpers,
} from '@dydxprotocol-indexer/postgres';
import { LatestAccountPnlTicksCache, PnlTickForSubaccounts } from '@dydxprotocol-indexer/redis';
import Big from 'big.js';
Expand All @@ -27,6 +27,23 @@ import { USDC_ASSET_ID, ZERO } from '../lib/constants';
import { redisClient } from './redis';
import { SubaccountUsdcTransferMap } from './types';

/**
* Normalizes a time to the nearest PNL_TICK_UPDATE_INTERVAL_MS.
* If PNL_TICK_UPDATE_INTERVAL_MS is set to 1 hour, then 12:01:00 -> 12:00:00.
*
* @param time
*/
export function normalizeStartTime(
time: Date,
): Date {
const epochMs: number = time.getTime();
const normalizedTimeMs: number = epochMs - (
epochMs % config.PNL_TICK_UPDATE_INTERVAL_MS
);

return new Date(normalizedTimeMs);
}

/**
* Gets a batch of new pnl ticks to write to the database and set in the cache.
* @param blockHeight: consider transfers up until this block height.
Expand Down Expand Up @@ -62,13 +79,7 @@ export async function getPnlTicksCreateObjects(
);
// get accounts to update based on last updated block height
const accountsToUpdate: string[] = [
..._.keys(accountToLastUpdatedBlockTime).filter(
(accountId) => {
const lastUpdatedBlockTime: string = accountToLastUpdatedBlockTime[accountId];
return new Date(blockTime).getTime() - new Date(lastUpdatedBlockTime).getTime() >=
config.PNL_TICK_UPDATE_INTERVAL_MS;
},
),
...getAccountsToUpdate(accountToLastUpdatedBlockTime, blockTime),
...newSubaccountIds,
];
stats.gauge(
Expand Down Expand Up @@ -166,6 +177,34 @@ export async function getPnlTicksCreateObjects(
return newTicksToCreate;
}

/**
* Gets a list of subaccounts that have not been updated this hour.
*
* @param mostRecentPnlTicks
* @param blockTime
*/
export function getAccountsToUpdate(
accountToLastUpdatedBlockTime: _.Dictionary<IsoString>,
blockTime: IsoString,
): string[] {
// get accounts to update based on last updated block time
const accountsToUpdate: string[] = [
..._.keys(accountToLastUpdatedBlockTime).filter(
(accountId) => {
const normalizedBlockTime: Date = normalizeStartTime(
new Date(blockTime),
); // 12:00:01 -> 12:00:00
const lastUpdatedBlockTime = accountToLastUpdatedBlockTime[accountId];
const normalizedLastUpdatedBlockTime = normalizeStartTime(
new Date(lastUpdatedBlockTime),
); // 12:00:01 -> 12:00:00
return normalizedBlockTime.getTime() !== normalizedLastUpdatedBlockTime.getTime();
},
),
];
return accountsToUpdate;
}

/**
* Get a map of block height to funding index state.
* Funding index state represents the most recent funding index value for every perpetual market.
Expand Down
13 changes: 1 addition & 12 deletions indexer/services/roundtable/src/tasks/create-pnl-ticks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,9 @@ import { LatestAccountPnlTicksCache } from '@dydxprotocol-indexer/redis';
import _ from 'lodash';

import config from '../config';
import { getPnlTicksCreateObjects } from '../helpers/pnl-ticks-helper';
import { getPnlTicksCreateObjects, normalizeStartTime } from '../helpers/pnl-ticks-helper';
import { redisClient } from '../helpers/redis';

export function normalizeStartTime(
time: Date,
): Date {
const epochMs: number = time.getTime();
const normalizedTimeMs: number = epochMs - (
epochMs % config.PNL_TICK_UPDATE_INTERVAL_MS
);

return new Date(normalizedTimeMs);
}

export default async function runTask(): Promise<void> {
const startGetNewTicks: number = Date.now();
const [
Expand Down

0 comments on commit 51cdad7

Please sign in to comment.