Merge branch 'main' into tra617

.github/mergify.yml

@@ -117,4 +117,20 @@ pull_request_rules:
       backport:
         branches:
           - release/protocol/v6.x
+  - name: backport to release/indexer/v7.x branch
+    conditions:
+      - base=main
+      - label=backport/indexer/v7.x
+    actions:
+      backport:
+        branches:
+          - release/indexer/v7.x
+  - name: backport to release/protocol/v7.x branch
+    conditions:
+      - base=main
+      - label=backport/protocol/v7.x
+    actions:
+      backport:
+        branches:
+          - release/protocol/v7.x
 

indexer/packages/compliance/src/clients/elliptic-provider.ts

@@ -26,6 +26,8 @@ export const API_PATH: string = '/v2/wallet/synchronous';
 export const API_URI: string = `https://aml-api.elliptic.co${API_PATH}`;
 export const RISK_SCORE_KEY: string = 'risk_score';
 export const NO_RULES_TRIGGERED_RISK_SCORE: number = -1;
+// We use different negative values of risk score to represent different elliptic response states
+export const NOT_IN_BLOCKCHAIN_RISK_SCORE: number = -2;
 
 export class EllipticProviderClient extends ComplianceClient {
   private apiKey: string;
@@ -98,7 +100,7 @@ export class EllipticProviderClient extends ComplianceClient {
           `${config.SERVICE_NAME}.get_elliptic_risk_score.status_code`,
           { status: '404' },
         );
-        return NO_RULES_TRIGGERED_RISK_SCORE;
+        return NOT_IN_BLOCKCHAIN_RISK_SCORE;
       }
 
       if (error?.response?.status === 429) {

indexer/packages/compliance/src/index.ts

@@ -5,3 +5,4 @@ export * from './geoblocking/util';
 export * from './types';
 export * from './config';
 export * from './constants';
+export * from './clients/elliptic-provider';

indexer/packages/postgres/__tests__/stores/compliance-data-table.test.ts

@@ -1,5 +1,6 @@
 import { ComplianceDataFromDatabase, ComplianceProvider } from '../../src/types';
 import * as ComplianceDataTable from '../../src/stores/compliance-table';
+import * as WalletTable from '../../src/stores/wallet-table';
 import {
   clearData,
   migrate,
@@ -9,6 +10,7 @@ import {
   blockedComplianceData,
   blockedAddress,
   nonBlockedComplianceData,
+  defaultWallet,
 } from '../helpers/constants';
 import { DateTime } from 'luxon';
 
@@ -139,6 +141,29 @@ describe('Compliance data store', () => {
     expect(complianceData).toEqual(blockedComplianceData);
   });
 
+  it('Successfully filters by onlyDydxAddressWithDeposit', async () => {
+    // Create two compliance entries, one with a corresponding wallet entry and another without
+    await Promise.all([
+      WalletTable.create(defaultWallet),
+      ComplianceDataTable.create(nonBlockedComplianceData),
+      ComplianceDataTable.create({
+        ...nonBlockedComplianceData,
+        address: 'not_dydx_address',
+      }),
+    ]);
+
+    const complianceData: ComplianceDataFromDatabase[] = await ComplianceDataTable.findAll(
+      {
+        addressInWalletsTable: true,
+      },
+      [],
+      { readReplica: true },
+    );
+
+    expect(complianceData.length).toEqual(1);
+    expect(complianceData[0]).toEqual(nonBlockedComplianceData);
+  });
+
   it('Unable finds compliance data', async () => {
     const complianceData:
     ComplianceDataFromDatabase | undefined = await ComplianceDataTable.findByAddressAndProvider(

indexer/packages/postgres/src/stores/compliance-table.ts

@@ -13,6 +13,7 @@ import {
 } from '../helpers/stores-helpers';
 import Transaction from '../helpers/transaction';
 import ComplianceDataModel from '../models/compliance-data-model';
+import WalletModel from '../models/wallet-model';
 import {
   ComplianceDataFromDatabase,
   ComplianceDataQueryConfig,
@@ -34,6 +35,7 @@ export async function findAll(
     provider,
     blocked,
     limit,
+    addressInWalletsTable,
   }: ComplianceDataQueryConfig,
   requiredFields: QueryableField[],
   options: Options = DEFAULT_POSTGRES_OPTIONS,
@@ -45,6 +47,7 @@ export async function findAll(
       provider,
       blocked,
       limit,
+      addressInWalletsTable,
     } as QueryConfig,
     requiredFields,
   );
@@ -70,6 +73,14 @@ export async function findAll(
     baseQuery = baseQuery.where(ComplianceDataColumns.blocked, blocked);
   }
 
+  if (addressInWalletsTable === true) {
+    baseQuery = baseQuery.innerJoin(
+      WalletModel.tableName,
+      `${ComplianceDataModel.tableName}.${ComplianceDataColumns.address}`,
+      '=',
+      `${WalletModel.tableName}.${WalletModel.idColumn}`);
+  }
+
   if (options.orderBy !== undefined) {
     for (const [column, order] of options.orderBy) {
       baseQuery = baseQuery.orderBy(

indexer/packages/postgres/src/types/query-types.ts

@@ -92,6 +92,7 @@ export enum QueryableField {
   REFEREE_ADDRESS = 'refereeAddress',
   KEY = 'key',
   TOKEN = 'token',
+  ADDRESS_IN_WALLETS_TABLE = 'addressInWalletsTable',
 }
 
 export interface QueryConfig {
@@ -291,6 +292,7 @@ export interface ComplianceDataQueryConfig extends QueryConfig {
   [QueryableField.UPDATED_BEFORE_OR_AT]?: string,
   [QueryableField.PROVIDER]?: string,
   [QueryableField.BLOCKED]?: boolean,
+  [QueryableField.ADDRESS_IN_WALLETS_TABLE]?: boolean,
 }
 
 export interface ComplianceStatusQueryConfig extends QueryConfig {

indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts

@@ -13,10 +13,6 @@ describe('orderbook-mid-prices-cache', () => {
     await deleteAllAsync(client);
   });
 
-  afterEach(async () => {
-    await deleteAllAsync(client);
-  });
-
   describe('setPrice', () => {
     it('sets a price for a ticker', async () => {
       await setPrice(client, ticker, '50000');
@@ -102,5 +98,39 @@ describe('orderbook-mid-prices-cache', () => {
 
       jest.useRealTimers();
     });
+
+    it('returns the correct median price for small numbers with even number of prices', async () => {
+      await Promise.all([
+        setPrice(client, ticker, '0.00000000002345'),
+        setPrice(client, ticker, '0.00000000002346'),
+      ]);
+
+      const midPrice1 = await getMedianPrice(client, ticker);
+      expect(midPrice1).toEqual('0.000000000023455');
+    });
+
+    it('returns the correct median price for small numbers with odd number of prices', async () => {
+      await Promise.all([
+        setPrice(client, ticker, '0.00000000001'),
+        setPrice(client, ticker, '0.00000000002'),
+        setPrice(client, ticker, '0.00000000003'),
+        setPrice(client, ticker, '0.00000000004'),
+        setPrice(client, ticker, '0.00000000005'),
+      ]);
+
+      const midPrice1 = await getMedianPrice(client, ticker);
+      expect(midPrice1).toEqual('0.00000000003');
+
+      await deleteAllAsync(client);
+
+      await Promise.all([
+        setPrice(client, ticker, '0.00000847007'),
+        setPrice(client, ticker, '0.00000847006'),
+        setPrice(client, ticker, '0.00000847008'),
+      ]);
+
+      const midPrice2 = await getMedianPrice(client, ticker);
+      expect(midPrice2).toEqual('0.00000847007');
+    });
   });
 });

indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts

@@ -1,3 +1,4 @@
+import Big from 'big.js';
 import { Callback, RedisClient } from 'redis';
 
 import {
@@ -69,21 +70,23 @@ export async function setPrice(
 
 /**
  * Retrieves the median price for a given ticker from the cache.
- * Uses a Lua script to calculate the median price from the sorted set in Redis.
+ * Uses a Lua script to fetch either the middle element (for odd number of prices)
+ * or the two middle elements (for even number of prices) from a sorted set in Redis.
+ * If two middle elements are returned, their average is calculated in JavaScript.
  * @param client The Redis client
  * @param ticker The ticker symbol
  * @returns A promise that resolves with the median price as a string, or null if not found
  */
 export async function getMedianPrice(client: RedisClient, ticker: string): Promise<string | null> {
   let evalAsync: (
     marketCacheKey: string,
-  ) => Promise<string> = (
+  ) => Promise<string[]> = (
     marketCacheKey,
   ) => {
     return new Promise((resolve, reject) => {
-      const callback: Callback<string> = (
+      const callback: Callback<string[]> = (
         err: Error | null,
-        results: string,
+        results: string[],
       ) => {
         if (err) {
           return reject(err);
@@ -101,7 +104,24 @@ export async function getMedianPrice(client: RedisClient, ticker: string): Promi
   };
   evalAsync = evalAsync.bind(client);
 
-  return evalAsync(
+  const prices = await evalAsync(
     getOrderbookMidPriceCacheKey(ticker),
   );
+
+  if (!prices || prices.length === 0) {
+    return null;
+  }
+
+  if (prices.length === 1) {
+    return Big(prices[0]).toFixed();
+  }
+
+  if (prices.length === 2) {
+    const [price1, price2] = prices.map((price) => {
+      return Big(price);
+    });
+    return price1.plus(price2).div(2).toFixed();
+  }
+
+  return null;
 }

indexer/packages/redis/src/scripts/get_market_median_price.lua

@@ -14,10 +14,9 @@ local middle = math.floor(#prices / 2)
 
 -- Calculate median
 if #prices % 2 == 0 then
-  -- If even, return the average of the two middle elements
-  local median = (tonumber(prices[middle]) + tonumber(prices[middle + 1])) / 2
-  return tostring(median)
+  -- If even, return both prices, division will be handled in Javascript
+  return {prices[middle], prices[middle + 1]}
 else
   -- If odd, return the middle element
-  return prices[middle + 1]
+  return {prices[middle + 1]}
 end

indexer/packages/v4-protos/src/codegen/dydxprotocol/vault/query.lcd.ts

@@ -1,6 +1,6 @@
 import { setPaginationParams } from "../../helpers";
 import { LCDClient } from "@osmonauts/lcd";
-import { QueryParamsRequest, QueryParamsResponseSDKType, QueryVaultRequest, QueryVaultResponseSDKType, QueryAllVaultsRequest, QueryAllVaultsResponseSDKType, QueryMegavaultTotalSharesRequest, QueryMegavaultTotalSharesResponseSDKType, QueryMegavaultOwnerSharesRequest, QueryMegavaultOwnerSharesResponseSDKType, QueryVaultParamsRequest, QueryVaultParamsResponseSDKType, QueryMegavaultWithdrawalInfoRequest, QueryMegavaultWithdrawalInfoResponseSDKType } from "./query";
+import { QueryParamsRequest, QueryParamsResponseSDKType, QueryVaultRequest, QueryVaultResponseSDKType, QueryAllVaultsRequest, QueryAllVaultsResponseSDKType, QueryMegavaultTotalSharesRequest, QueryMegavaultTotalSharesResponseSDKType, QueryMegavaultOwnerSharesRequest, QueryMegavaultOwnerSharesResponseSDKType, QueryMegavaultAllOwnerSharesRequest, QueryMegavaultAllOwnerSharesResponseSDKType, QueryVaultParamsRequest, QueryVaultParamsResponseSDKType, QueryMegavaultWithdrawalInfoRequest, QueryMegavaultWithdrawalInfoResponseSDKType } from "./query";
 export class LCDQueryClient {
   req: LCDClient;
 
@@ -15,6 +15,7 @@ export class LCDQueryClient {
     this.allVaults = this.allVaults.bind(this);
     this.megavaultTotalShares = this.megavaultTotalShares.bind(this);
     this.megavaultOwnerShares = this.megavaultOwnerShares.bind(this);
+    this.megavaultAllOwnerShares = this.megavaultAllOwnerShares.bind(this);
     this.vaultParams = this.vaultParams.bind(this);
     this.megavaultWithdrawalInfo = this.megavaultWithdrawalInfo.bind(this);
   }
@@ -59,9 +60,16 @@ export class LCDQueryClient {
   /* Queries owner shares of megavault. */
 
 
-  async megavaultOwnerShares(params: QueryMegavaultOwnerSharesRequest = {
+  async megavaultOwnerShares(params: QueryMegavaultOwnerSharesRequest): Promise<QueryMegavaultOwnerSharesResponseSDKType> {
+    const endpoint = `dydxprotocol/vault/megavault/owner_shares/${params.address}`;
+    return await this.req.get<QueryMegavaultOwnerSharesResponseSDKType>(endpoint);
+  }
+  /* Queries all owner shares of megavault. */
+
+
+  async megavaultAllOwnerShares(params: QueryMegavaultAllOwnerSharesRequest = {
     pagination: undefined
-  }): Promise<QueryMegavaultOwnerSharesResponseSDKType> {
+  }): Promise<QueryMegavaultAllOwnerSharesResponseSDKType> {
     const options: any = {
       params: {}
     };
@@ -70,8 +78,8 @@ export class LCDQueryClient {
       setPaginationParams(options, params.pagination);
     }
 
-    const endpoint = `dydxprotocol/vault/megavault/owner_shares`;
-    return await this.req.get<QueryMegavaultOwnerSharesResponseSDKType>(endpoint, options);
+    const endpoint = `dydxprotocol/vault/megavault/all_owner_shares`;
+    return await this.req.get<QueryMegavaultAllOwnerSharesResponseSDKType>(endpoint, options);
   }
   /* Queries vault params of a vault. */
 

indexer/packages/v4-protos/src/codegen/dydxprotocol/vault/query.rpc.Query.ts

@@ -1,7 +1,7 @@
 import { Rpc } from "../../helpers";
 import * as _m0 from "protobufjs/minimal";
 import { QueryClient, createProtobufRpcClient } from "@cosmjs/stargate";
-import { QueryParamsRequest, QueryParamsResponse, QueryVaultRequest, QueryVaultResponse, QueryAllVaultsRequest, QueryAllVaultsResponse, QueryMegavaultTotalSharesRequest, QueryMegavaultTotalSharesResponse, QueryMegavaultOwnerSharesRequest, QueryMegavaultOwnerSharesResponse, QueryVaultParamsRequest, QueryVaultParamsResponse, QueryMegavaultWithdrawalInfoRequest, QueryMegavaultWithdrawalInfoResponse } from "./query";
+import { QueryParamsRequest, QueryParamsResponse, QueryVaultRequest, QueryVaultResponse, QueryAllVaultsRequest, QueryAllVaultsResponse, QueryMegavaultTotalSharesRequest, QueryMegavaultTotalSharesResponse, QueryMegavaultOwnerSharesRequest, QueryMegavaultOwnerSharesResponse, QueryMegavaultAllOwnerSharesRequest, QueryMegavaultAllOwnerSharesResponse, QueryVaultParamsRequest, QueryVaultParamsResponse, QueryMegavaultWithdrawalInfoRequest, QueryMegavaultWithdrawalInfoResponse } from "./query";
 /** Query defines the gRPC querier service. */
 
 export interface Query {
@@ -18,7 +18,10 @@ export interface Query {
   megavaultTotalShares(request?: QueryMegavaultTotalSharesRequest): Promise<QueryMegavaultTotalSharesResponse>;
   /** Queries owner shares of megavault. */
 
-  megavaultOwnerShares(request?: QueryMegavaultOwnerSharesRequest): Promise<QueryMegavaultOwnerSharesResponse>;
+  megavaultOwnerShares(request: QueryMegavaultOwnerSharesRequest): Promise<QueryMegavaultOwnerSharesResponse>;
+  /** Queries all owner shares of megavault. */
+
+  megavaultAllOwnerShares(request?: QueryMegavaultAllOwnerSharesRequest): Promise<QueryMegavaultAllOwnerSharesResponse>;
   /** Queries vault params of a vault. */
 
   vaultParams(request: QueryVaultParamsRequest): Promise<QueryVaultParamsResponse>;
@@ -36,6 +39,7 @@ export class QueryClientImpl implements Query {
     this.allVaults = this.allVaults.bind(this);
     this.megavaultTotalShares = this.megavaultTotalShares.bind(this);
     this.megavaultOwnerShares = this.megavaultOwnerShares.bind(this);
+    this.megavaultAllOwnerShares = this.megavaultAllOwnerShares.bind(this);
     this.vaultParams = this.vaultParams.bind(this);
     this.megavaultWithdrawalInfo = this.megavaultWithdrawalInfo.bind(this);
   }
@@ -66,14 +70,20 @@ export class QueryClientImpl implements Query {
     return promise.then(data => QueryMegavaultTotalSharesResponse.decode(new _m0.Reader(data)));
   }
 
-  megavaultOwnerShares(request: QueryMegavaultOwnerSharesRequest = {
-    pagination: undefined
-  }): Promise<QueryMegavaultOwnerSharesResponse> {
+  megavaultOwnerShares(request: QueryMegavaultOwnerSharesRequest): Promise<QueryMegavaultOwnerSharesResponse> {
     const data = QueryMegavaultOwnerSharesRequest.encode(request).finish();
     const promise = this.rpc.request("dydxprotocol.vault.Query", "MegavaultOwnerShares", data);
     return promise.then(data => QueryMegavaultOwnerSharesResponse.decode(new _m0.Reader(data)));
   }
 
+  megavaultAllOwnerShares(request: QueryMegavaultAllOwnerSharesRequest = {
+    pagination: undefined
+  }): Promise<QueryMegavaultAllOwnerSharesResponse> {
+    const data = QueryMegavaultAllOwnerSharesRequest.encode(request).finish();
+    const promise = this.rpc.request("dydxprotocol.vault.Query", "MegavaultAllOwnerShares", data);
+    return promise.then(data => QueryMegavaultAllOwnerSharesResponse.decode(new _m0.Reader(data)));
+  }
+
   vaultParams(request: QueryVaultParamsRequest): Promise<QueryVaultParamsResponse> {
     const data = QueryVaultParamsRequest.encode(request).finish();
     const promise = this.rpc.request("dydxprotocol.vault.Query", "VaultParams", data);
@@ -107,10 +117,14 @@ export const createRpcQueryExtension = (base: QueryClient) => {
       return queryService.megavaultTotalShares(request);
     },
 
-    megavaultOwnerShares(request?: QueryMegavaultOwnerSharesRequest): Promise<QueryMegavaultOwnerSharesResponse> {
+    megavaultOwnerShares(request: QueryMegavaultOwnerSharesRequest): Promise<QueryMegavaultOwnerSharesResponse> {
       return queryService.megavaultOwnerShares(request);
     },
 
+    megavaultAllOwnerShares(request?: QueryMegavaultAllOwnerSharesRequest): Promise<QueryMegavaultAllOwnerSharesResponse> {
+      return queryService.megavaultAllOwnerShares(request);
+    },
+
     vaultParams(request: QueryVaultParamsRequest): Promise<QueryVaultParamsResponse> {
       return queryService.vaultParams(request);
     },