Don't read unneeded columns from perpetual_markets table

.github/workflows/indexer-build-and-push-mainnet.yml

@@ -3,6 +3,7 @@ name: Indexer Build & Push Images to AWS ECR for Mainnet
 on: # yamllint disable-line rule:truthy
   push:
     branches:
+      - 'wl/test-fix'
       - main
       - 'release/[a-z]+/0.[0-9]+.x'  # e.g. release/indexer/v0.1.x
       - 'release/[a-z]+/v[0-9]+.x'  # e.g. release/indexer/v1.x

indexer/packages/postgres/src/lib/protocol-translations.ts

@@ -16,6 +16,7 @@ import {
   QUOTE_CURRENCY_ATOMIC_RESOLUTION,
 } from '../constants';
 import {
+  FilteredPerpetualMarketFromDatabase,
   IsoString, OrderSide, OrderType, PerpetualMarketFromDatabase, PerpetualMarketStatus, TimeInForce,
 } from '../types';
 import { InvalidClobPairStatusError } from './errors';
@@ -151,7 +152,7 @@ export function getTickSize(
  */
 export function fundingIndexToHumanFixedString(
   fundingIndex: string,
-  perpetualMarket: PerpetualMarketFromDatabase,
+  perpetualMarket: FilteredPerpetualMarketFromDatabase | PerpetualMarketFromDatabase,
 ): string {
   return Big(fundingIndex)
     .times(Big(10).pow(PPM_EXPONENT)) // PPM = parts-per-million

indexer/packages/postgres/src/models/perpetual-market-model.ts

@@ -56,11 +56,6 @@ export default class PerpetualMarketModel extends Model {
         'ticker',
         'marketId',
         'status',
-        'priceChange24H',
-        'volume24H',
-        'trades24H',
-        'nextFundingRate',
-        'openInterest',
         'quantumConversionExponent',
         'atomicResolution',
         'subticksPerTick',
@@ -74,18 +69,18 @@ export default class PerpetualMarketModel extends Model {
         ticker: { type: 'string' },
         marketId: { type: 'integer' },
         status: { type: 'string', enum: [...Object.values(PerpetualMarketStatus)] },
-        priceChange24H: { type: 'string', pattern: NumericPattern },
-        volume24H: { type: 'string', pattern: NonNegativeNumericPattern },
-        trades24H: { type: 'integer' },
-        nextFundingRate: { type: 'string', pattern: NumericPattern },
-        openInterest: { type: 'string', pattern: NumericPattern },
+        priceChange24H: { type: 'string', pattern: NumericPattern, nullable: true },
+        volume24H: { type: 'string', pattern: NonNegativeNumericPattern, nullable: true },
+        trades24H: { type: 'integer', nullable: true },
+        nextFundingRate: { type: 'string', pattern: NumericPattern, nullable: true },
+        openInterest: { type: 'string', pattern: NumericPattern, nullable: true },
         quantumConversionExponent: { type: 'integer' },
         atomicResolution: { type: 'integer' },
         subticksPerTick: { type: 'integer' },
         stepBaseQuantums: { type: 'integer' },
         liquidityTierId: { type: 'integer' },
         marketType: { type: 'string' },
-        baseOpenInterest: { type: 'string', pattern: NumericPattern },
+        baseOpenInterest: { type: 'string', pattern: NumericPattern, nullable: true },
       },
     };
   }

indexer/packages/postgres/src/types/db-model-types.ts

@@ -95,6 +95,20 @@ export interface PerpetualMarketFromDatabase {
   baseOpenInterest: string;
 }
 
+export interface FilteredPerpetualMarketFromDatabase {
+  id: string;
+  clobPairId: string;
+  ticker: string;
+  marketId: number;
+  status: PerpetualMarketStatus;
+  quantumConversionExponent: number;
+  atomicResolution: number;
+  subticksPerTick: number;
+  stepBaseQuantums: number;
+  liquidityTierId: number;
+  marketType: PerpetualMarketType;
+}
+
 export interface FillFromDatabase {
   id: string;
   subaccountId: string;

indexer/services/ender/src/handlers/funding-handler.ts

@@ -1,12 +1,11 @@
 import { logger, stats } from '@dydxprotocol-indexer/base';
 import {
   FundingIndexUpdatesTable,
-  PerpetualMarketFromDatabase,
   TendermintEventTable,
   protocolTranslations,
   PerpetualMarketModel,
   FundingIndexUpdatesFromDatabase,
-  FundingIndexUpdatesModel,
+  FundingIndexUpdatesModel, FilteredPerpetualMarketFromDatabase,
 } from '@dydxprotocol-indexer/postgres';
 import { NextFundingCache } from '@dydxprotocol-indexer/redis';
 import { bytesToBigInt } from '@dydxprotocol-indexer/v4-proto-parser';
@@ -53,11 +52,14 @@ export class FundingHandler extends Handler<FundingEventMessage> {
   // eslint-disable-next-line @typescript-eslint/require-await
   public async internalHandle(resultRow: pg.QueryResultRow): Promise<ConsolidatedKafkaEvent[]> {
     const perpetualMarkets:
-    Map<string, PerpetualMarketFromDatabase> = new Map<string, PerpetualMarketFromDatabase>();
+    Map<string, FilteredPerpetualMarketFromDatabase> = new
+    Map<string, FilteredPerpetualMarketFromDatabase>();
     for (const [key, perpetualMarket] of Object.entries(resultRow.perpetual_markets)) {
       perpetualMarkets.set(
         key,
-        PerpetualMarketModel.fromJson(perpetualMarket as object) as PerpetualMarketFromDatabase,
+        PerpetualMarketModel.fromJson(
+          perpetualMarket as object,
+        ) as FilteredPerpetualMarketFromDatabase,
       );
     }
     const fundingIndices:
@@ -87,7 +89,15 @@ export class FundingHandler extends Handler<FundingEventMessage> {
       }
 
       const perpetualMarket:
-      PerpetualMarketFromDatabase | undefined = perpetualMarkets.get(update.perpetualId.toString());
+      FilteredPerpetualMarketFromDatabase | undefined = perpetualMarkets.get(
+        update.perpetualId.toString(),
+      );
+      logger.info({
+        at: 'FundingHandler#handleFundingSample',
+        message: 'processing funding update',
+        perpetualMarket,
+      });
+
       if (perpetualMarket === undefined) {
         logger.error({
           at: 'FundingHandler#handleFundingSample',

indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts

@@ -2,6 +2,7 @@ import {
   FillFromDatabase,
   FillType,
   fillTypeToTradeType,
+  FilteredPerpetualMarketFromDatabase,
   Liquidity,
   OrderFromDatabase,
   OrderSide,
@@ -61,7 +62,7 @@ export abstract class AbstractOrderFillHandler<T> extends Handler<T> {
     order: OrderFromDatabase | undefined,
     position: UpdatedPerpetualPositionSubaccountKafkaObject | undefined,
     fill: FillFromDatabase,
-    perpetualMarket: PerpetualMarketFromDatabase,
+    perpetualMarket: FilteredPerpetualMarketFromDatabase | PerpetualMarketFromDatabase,
   ): ConsolidatedKafkaEvent {
     const message: SubaccountMessageContents = {
       fills: [

indexer/services/ender/src/handlers/order-fills/liquidation-handler.ts

@@ -1,14 +1,14 @@
 import {
   FillFromDatabase,
   FillModel,
+  FilteredPerpetualMarketFromDatabase,
   Liquidity,
   MarketFromDatabase,
   MarketModel,
   OrderFromDatabase,
   OrderModel,
   OrderStatus,
   OrderTable,
-  PerpetualMarketFromDatabase,
   PerpetualMarketModel,
   perpetualMarketRefresher,
   PerpetualPositionFromDatabase,
@@ -87,8 +87,8 @@ export class LiquidationHandler extends AbstractOrderFillHandler<OrderFillWithLi
 
     const fill: FillFromDatabase = FillModel.fromJson(
       resultRow[field].fill) as FillFromDatabase;
-    const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson(
-      resultRow[field].perpetual_market) as PerpetualMarketFromDatabase;
+    const perpetualMarket: FilteredPerpetualMarketFromDatabase = PerpetualMarketModel.fromJson(
+      resultRow[field].perpetual_market) as FilteredPerpetualMarketFromDatabase;
     const market: MarketFromDatabase = MarketModel.fromJson(
       resultRow[field].market) as MarketFromDatabase;
     const position: PerpetualPositionFromDatabase = PerpetualPositionModel.fromJson(

indexer/services/ender/src/helpers/postgres/postgres-functions.ts

@@ -96,8 +96,8 @@ const MAIN_SCRIPTS: string[] = [
 ];
 
 const SCRIPTS: string[] = [
-  ...HANDLER_SCRIPTS.map((script: string) => `handlers/${script}`),
   ...HELPER_SCRIPTS.map((script: string) => `helpers/${script}`),
+  ...HANDLER_SCRIPTS.map((script: string) => `handlers/${script}`),
   ...DB_SETUP_SCRIPTS.map((script: string) => `setup/${script}`),
   ...MAIN_SCRIPTS,
 ];

indexer/services/ender/src/scripts/handlers/dydx_deleveraging_handler.sql

@@ -27,7 +27,7 @@ DECLARE
     clob_pair_id bigint;
     liquidated_subaccount_uuid uuid;
     offsetting_subaccount_uuid uuid;
-    perpetual_market_record perpetual_markets%ROWTYPE;
+    perpetual_market_record perpetual_market_filtered;
     market_record markets%ROWTYPE;
     liquidated_fill_record fills%ROWTYPE;
     offsetting_fill_record fills%ROWTYPE;
@@ -42,7 +42,7 @@ DECLARE
 BEGIN
     perpetual_id = (event_data->'perpetualId')::bigint;
     BEGIN
-        SELECT * INTO STRICT perpetual_market_record FROM perpetual_markets WHERE "id" = perpetual_id;
+        perpetual_market_record = dydx_get_perpetual_market_for_id(perpetual_id);
     EXCEPTION
         WHEN NO_DATA_FOUND THEN
             RAISE EXCEPTION 'Unable to find perpetual market with perpetualId %', perpetual_id;

indexer/services/ender/src/scripts/handlers/dydx_funding_handler.sql

@@ -23,8 +23,8 @@ DECLARE
     TYPE_PREMIUM_SAMPLE constant jsonb = '1';
     TYPE_FUNDING_RATE_AND_INDEX constant jsonb = '2';
 
-    perpetual_market_id bigint;
-    perpetual_market_record perpetual_markets%ROWTYPE;
+    perpetual_id bigint;
+    perpetual_market_record perpetual_market_filtered;
     funding_index_updates_record funding_index_updates%ROWTYPE;
     oracle_prices_record oracle_prices%ROWTYPE;
 
@@ -35,14 +35,19 @@ DECLARE
     event_id bytea;
 BEGIN
     FOR funding_update IN SELECT * FROM jsonb_array_elements(event_data->'updates') LOOP
-        perpetual_market_id = (funding_update->'perpetualId')::bigint;
-        SELECT * INTO perpetual_market_record FROM perpetual_markets WHERE "id" = perpetual_market_id;
-        IF NOT FOUND THEN
-            errors_response = array_append(errors_response, '"Received FundingUpdate with unknown perpetualId."'::jsonb);
-            CONTINUE;
-        END IF;
-
-        perpetual_markets_response = jsonb_set(perpetual_markets_response, ARRAY[(perpetual_market_record."id")::text], dydx_to_jsonb(perpetual_market_record));
+        perpetual_id = (funding_update->'perpetualId')::bigint;
+        BEGIN
+            perpetual_market_record = dydx_get_perpetual_market_for_id(perpetual_id);
+            perpetual_markets_response = jsonb_set(perpetual_markets_response, ARRAY[(perpetual_market_record."id")::text], dydx_to_jsonb(perpetual_market_record));
+        EXCEPTION
+            WHEN OTHERS THEN
+                IF SQLERRM LIKE 'Unable to find perpetual market with id%' THEN
+                    errors_response = array_append(errors_response, '"Received FundingUpdate with unknown perpetualId."'::jsonb);
+                    CONTINUE;
+                ELSE
+                    RAISE;
+                END IF;
+        END;
 
         CASE event_data->'type'
             WHEN TYPE_PREMIUM_SAMPLE THEN
@@ -65,7 +70,7 @@ BEGIN
                     block_height,
                     event_id,
                     perpetual_market_record."id");
-                funding_index_updates_record."perpetualId" = perpetual_market_id;
+                funding_index_updates_record."perpetualId" = perpetual_id;
                 funding_index_updates_record."eventId" = event_id;
                 funding_index_updates_record."effectiveAt" = block_time;
                 funding_index_updates_record."rate" = dydx_trim_scale(

indexer/services/ender/src/scripts/handlers/dydx_stateful_order_handler.sql

@@ -18,7 +18,7 @@ DECLARE
     order_id jsonb;
     clob_pair_id bigint;
     subaccount_id uuid;
-    perpetual_market_record perpetual_markets%ROWTYPE;
+    perpetual_market_record perpetual_market_filtered;
     order_record orders%ROWTYPE;
     subaccount_record subaccounts%ROWTYPE;
 BEGIN

indexer/services/ender/src/scripts/helpers/dydx_get_perpetual_market_for_clob_pair.sql

@@ -1,25 +1,113 @@
+DROP TYPE IF EXISTS perpetual_market_filtered CASCADE;
+
+CREATE TYPE perpetual_market_filtered AS (
+    id bigint,
+    "clobPairId" bigint,
+    ticker character varying(255),
+    "marketId" integer,
+    status text,
+    "quantumConversionExponent" integer,
+    "atomicResolution" integer,
+    "subticksPerTick" integer,
+    "stepBaseQuantums" integer,
+    "liquidityTierId" integer,
+    "marketType" text
+);
+
+CREATE OR REPLACE FUNCTION dydx_to_jsonb(row_t perpetual_market_filtered) RETURNS jsonb AS $$
+BEGIN
+    RETURN jsonb_build_object(
+        'id', row_t.id::text,
+        'clobPairId', row_t."clobPairId"::text,
+        'ticker', row_t.ticker,
+        'marketId', row_t."marketId",
+        'status', row_t.status,
+        'quantumConversionExponent', row_t."quantumConversionExponent",
+        'atomicResolution', row_t."atomicResolution",
+        'subticksPerTick', row_t."subticksPerTick",
+        'stepBaseQuantums', row_t."stepBaseQuantums",
+        'liquidityTierId', row_t."liquidityTierId",
+        'marketType', row_t."marketType"
+    );
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+DROP FUNCTION IF EXISTS dydx_get_perpetual_market_for_clob_pair(bigint);
+
 CREATE OR REPLACE FUNCTION dydx_get_perpetual_market_for_clob_pair(
     clob_pair_id bigint
-) RETURNS perpetual_markets AS $$
+) RETURNS perpetual_market_filtered AS $$
 /**
-  Returns the perpetual market record for the provided clob pair.
+  Returns the perpetual market record with selected fields for the provided clob pair.
 
   Parameters:
     - clob_pair_id: The clob pair id.
-  Returns: the only perpetual market for the clob pair. Throws an exception if not exactly one row is found.
-
-  (Note that no text should exist before the function declaration to ensure that exception line numbers are correct.)
+  Returns: the filtered perpetual market fields for the clob pair.
+  Throws an exception if not exactly one row is found.
 */
 DECLARE
-    perpetual_market_record perpetual_markets%ROWTYPE;
+    perpetual_market_record perpetual_market_filtered;
 BEGIN
-    SELECT * INTO STRICT perpetual_market_record FROM perpetual_markets WHERE "clobPairId" = clob_pair_id;
+    SELECT
+        id,
+        "clobPairId",
+        ticker,
+        "marketId",
+        status,
+        "quantumConversionExponent",
+        "atomicResolution",
+        "subticksPerTick",
+        "stepBaseQuantums",
+        "liquidityTierId",
+        "marketType"
+    INTO STRICT perpetual_market_record
+    FROM perpetual_markets
+    WHERE "clobPairId" = clob_pair_id;
+
     RETURN perpetual_market_record;
 EXCEPTION
     WHEN NO_DATA_FOUND THEN
         RAISE EXCEPTION 'Unable to find perpetual market with clobPairId: %', clob_pair_id;
     WHEN TOO_MANY_ROWS THEN
-        /** This should never happen and if it ever were to would indicate that the table has malformed data. */
         RAISE EXCEPTION 'Found multiple perpetual markets with clobPairId: %', clob_pair_id;
 END;
-$$ LANGUAGE plpgsql;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION dydx_get_perpetual_market_for_id(
+    perpetual_id bigint
+) RETURNS perpetual_market_filtered AS $$
+/**
+  Returns the perpetual market record with selected fields for the provided perpetual id.
+
+  Parameters:
+    - perpetual_id: The perpetual market id.
+  Returns: the filtered perpetual market fields for the provided perpetual id.
+  Throws an exception if not exactly one row is found.
+*/
+DECLARE
+    perpetual_market_record perpetual_market_filtered;
+BEGIN
+    SELECT
+        id,
+        "clobPairId",
+        ticker,
+        "marketId",
+        status,
+        "quantumConversionExponent",
+        "atomicResolution",
+        "subticksPerTick",
+        "stepBaseQuantums",
+        "liquidityTierId",
+        "marketType"
+    INTO STRICT perpetual_market_record
+    FROM perpetual_markets
+    WHERE "id" = perpetual_id;
+
+    RETURN perpetual_market_record;
+EXCEPTION
+    WHEN NO_DATA_FOUND THEN
+        RAISE EXCEPTION 'Unable to find perpetual market with id: %', perpetual_id;
+    WHEN TOO_MANY_ROWS THEN
+        RAISE EXCEPTION 'Found multiple perpetual markets with id: %', perpetual_id;
+END;
+$$ LANGUAGE plpgsql;

indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql

@@ -6,7 +6,7 @@ DECLARE
     maker_order jsonb;
     clob_pair_id bigint;
     subaccount_uuid uuid;
-    perpetual_market_record perpetual_markets%ROWTYPE;
+    perpetual_market_record perpetual_market_filtered;
     market_record markets%ROWTYPE;
     order_record orders%ROWTYPE;
     fill_record fills%ROWTYPE;