diff --git a/protocol/lib/metrics/constants.go b/protocol/lib/metrics/constants.go index 431704df85..7eff351d8f 100644 --- a/protocol/lib/metrics/constants.go +++ b/protocol/lib/metrics/constants.go @@ -282,6 +282,7 @@ const ( NumSubaccountsIterated = "num_subaccounts_iterated" NotEnoughPositionToFullyOffset = "not_enough_position_to_fully_offset" NonOverlappingBankruptcyPrices = "non_overlapping_bankruptcy_prices" + NoOpenPositionOnOppositeSide = "no_open_position_on_opposite_side" // Pricefeed Daemon. Exchange = "exchange" diff --git a/protocol/x/clob/flags/flags.go b/protocol/x/clob/flags/flags.go index 541afece21..146121c277 100644 --- a/protocol/x/clob/flags/flags.go +++ b/protocol/x/clob/flags/flags.go @@ -9,8 +9,9 @@ import ( // A struct containing the values of all flags. type ClobFlags struct { - MaxLiquidationOrdersPerBlock uint32 - MaxDeleveragingAttemptsPerBlock uint32 + MaxLiquidationOrdersPerBlock uint32 + MaxDeleveragingAttemptsPerBlock uint32 + MaxDeleveragingSubaccountsToIterate uint32 MevTelemetryEnabled bool MevTelemetryHost string @@ -20,8 +21,9 @@ type ClobFlags struct { // List of CLI flags. const ( // Liquidations and deleveraging. - MaxLiquidationOrdersPerBlock = "max-liquidation-orders-per-block" - MaxDeleveragingAttemptsPerBlock = "max-deleveraging-attempts-per-block" + MaxLiquidationOrdersPerBlock = "max-liquidation-orders-per-block" + MaxDeleveragingAttemptsPerBlock = "max-deleveraging-attempts-per-block" + MaxDeleveragingSubaccountsToIterate = "max-deleveraging-subaccounts-to-iterate" // Mev. MevTelemetryEnabled = "mev-telemetry-enabled" @@ -31,8 +33,9 @@ const ( // Default values. const ( - DefaultMaxLiquidationOrdersPerBlock = 20 - DefaultMaxDeleveragingAttemptsPerBlock = 5 + DefaultMaxLiquidationOrdersPerBlock = 20 + DefaultMaxDeleveragingAttemptsPerBlock = 5 + DefaultMaxDeleveragingSubaccountsToIterate = 500 DefaultMevTelemetryEnabled = false DefaultMevTelemetryHost = "" @@ -59,6 +62,14 @@ func AddClobFlagsToCmd(cmd *cobra.Command) { DefaultMaxDeleveragingAttemptsPerBlock, ), ) + cmd.Flags().Uint32( + MaxDeleveragingSubaccountsToIterate, + DefaultMaxDeleveragingSubaccountsToIterate, + fmt.Sprintf( + "Sets the maximum number of subaccounts iterated for each deleveraging event. Default = %d", + DefaultMaxDeleveragingSubaccountsToIterate, + ), + ) cmd.Flags().Bool( MevTelemetryEnabled, DefaultMevTelemetryEnabled, @@ -78,11 +89,12 @@ func AddClobFlagsToCmd(cmd *cobra.Command) { func GetDefaultClobFlags() ClobFlags { return ClobFlags{ - MaxLiquidationOrdersPerBlock: DefaultMaxLiquidationOrdersPerBlock, - MaxDeleveragingAttemptsPerBlock: DefaultMaxDeleveragingAttemptsPerBlock, - MevTelemetryEnabled: DefaultMevTelemetryEnabled, - MevTelemetryHost: DefaultMevTelemetryHost, - MevTelemetryIdentifier: DefaultMevTelemetryIdentifier, + MaxLiquidationOrdersPerBlock: DefaultMaxLiquidationOrdersPerBlock, + MaxDeleveragingAttemptsPerBlock: DefaultMaxDeleveragingAttemptsPerBlock, + MaxDeleveragingSubaccountsToIterate: DefaultMaxDeleveragingSubaccountsToIterate, + MevTelemetryEnabled: DefaultMevTelemetryEnabled, + MevTelemetryHost: DefaultMevTelemetryHost, + MevTelemetryIdentifier: DefaultMevTelemetryIdentifier, } } @@ -115,5 +127,9 @@ func GetClobFlagValuesFromOptions( result.MaxDeleveragingAttemptsPerBlock = v } + if v, ok := appOpts.Get(MaxDeleveragingSubaccountsToIterate).(uint32); ok { + result.MaxDeleveragingSubaccountsToIterate = v + } + return result } diff --git a/protocol/x/clob/flags/flags_test.go b/protocol/x/clob/flags/flags_test.go index 9e8d9368c8..863b9ba334 100644 --- a/protocol/x/clob/flags/flags_test.go +++ b/protocol/x/clob/flags/flags_test.go @@ -44,28 +44,32 @@ func TestGetFlagValuesFromOptions(t *testing.T) { optsMap map[string]any // Expectations. - expectedMaxLiquidationOrdersPerBlock uint32 - expectedMaxDeleveragingAttemptsPerBlock uint32 - expectedMevTelemetryHost string - expectedMevTelemetryIdentifier string + expectedMaxLiquidationOrdersPerBlock uint32 + expectedMaxDeleveragingAttemptsPerBlock uint32 + expectedMaxDeleveragingSubaccountsToIterate uint32 + expectedMevTelemetryHost string + expectedMevTelemetryIdentifier string }{ "Sets to default if unset": { - expectedMaxLiquidationOrdersPerBlock: flags.DefaultMaxLiquidationOrdersPerBlock, - expectedMaxDeleveragingAttemptsPerBlock: flags.DefaultMaxDeleveragingAttemptsPerBlock, - expectedMevTelemetryHost: flags.DefaultMevTelemetryHost, - expectedMevTelemetryIdentifier: flags.DefaultMevTelemetryIdentifier, + expectedMaxLiquidationOrdersPerBlock: flags.DefaultMaxLiquidationOrdersPerBlock, + expectedMaxDeleveragingAttemptsPerBlock: flags.DefaultMaxDeleveragingAttemptsPerBlock, + expectedMaxDeleveragingSubaccountsToIterate: flags.DefaultMaxDeleveragingSubaccountsToIterate, + expectedMevTelemetryHost: flags.DefaultMevTelemetryHost, + expectedMevTelemetryIdentifier: flags.DefaultMevTelemetryIdentifier, }, "Sets values from options": { optsMap: map[string]any{ - flags.MaxLiquidationOrdersPerBlock: uint32(50), - flags.MaxDeleveragingAttemptsPerBlock: uint32(25), - flags.MevTelemetryHost: "https://localhost:13137", - flags.MevTelemetryIdentifier: "node-agent-01", + flags.MaxLiquidationOrdersPerBlock: uint32(50), + flags.MaxDeleveragingAttemptsPerBlock: uint32(25), + flags.MaxDeleveragingSubaccountsToIterate: uint32(100), + flags.MevTelemetryHost: "https://localhost:13137", + flags.MevTelemetryIdentifier: "node-agent-01", }, - expectedMaxLiquidationOrdersPerBlock: uint32(50), - expectedMaxDeleveragingAttemptsPerBlock: uint32(25), - expectedMevTelemetryHost: "https://localhost:13137", - expectedMevTelemetryIdentifier: "node-agent-01", + expectedMaxLiquidationOrdersPerBlock: uint32(50), + expectedMaxDeleveragingAttemptsPerBlock: uint32(25), + expectedMaxDeleveragingSubaccountsToIterate: uint32(100), + expectedMevTelemetryHost: "https://localhost:13137", + expectedMevTelemetryIdentifier: "node-agent-01", }, } @@ -98,6 +102,11 @@ func TestGetFlagValuesFromOptions(t *testing.T) { tc.expectedMaxDeleveragingAttemptsPerBlock, flags.MaxDeleveragingAttemptsPerBlock, ) + require.Equal( + t, + tc.expectedMaxDeleveragingSubaccountsToIterate, + flags.MaxDeleveragingSubaccountsToIterate, + ) }) } } diff --git a/protocol/x/clob/keeper/deleveraging.go b/protocol/x/clob/keeper/deleveraging.go index b370c9489c..1c311c15e0 100644 --- a/protocol/x/clob/keeper/deleveraging.go +++ b/protocol/x/clob/keeper/deleveraging.go @@ -160,18 +160,26 @@ func (k Keeper) OffsetSubaccountPerpetualPosition( ) numSubaccountsIterated := 0 + numSubaccountsWithNonOverlappingBankruptcyPrices := 0 + numSubaccountsWithNoOpenPositionOnOppositeSide := 0 deltaQuantumsRemaining = new(big.Int).Set(deltaQuantumsTotal) fills = make([]types.MatchPerpetualDeleveraging_Fill, 0) k.subaccountsKeeper.ForEachSubaccountRandomStart( ctx, func(offsettingSubaccount satypes.Subaccount) (finished bool) { + // Iterate at most `MaxDeleveragingSubaccountsToIterate` subaccounts. + if numSubaccountsIterated >= int(k.MaxDeleveragingSubaccountsToIterate) { + return true + } + numSubaccountsIterated++ offsettingPosition, _ := offsettingSubaccount.GetPerpetualPositionForId(perpetualId) bigOffsettingPositionQuantums := offsettingPosition.GetBigQuantums() // Skip subaccounts that do not have a position in the opposite direction as the liquidated subaccount. if deltaQuantumsRemaining.Sign() != bigOffsettingPositionQuantums.Sign() { + numSubaccountsWithNoOpenPositionOnOppositeSide++ return false } @@ -275,7 +283,7 @@ func (k Keeper) OffsetSubaccountPerpetualPosition( return false } - k.Logger(ctx).Info( + k.Logger(ctx).Debug( "Encountered error when processing deleveraging", "error", err, "blockHeight", ctx.BlockHeight(), @@ -289,17 +297,25 @@ func (k Keeper) OffsetSubaccountPerpetualPosition( "offsettingBankruptcyPriceQuoteQuantums", offsettingBankruptcyPrice, "offsettingTnc", offsettingTnc, ) - telemetry.IncrCounter( - 1, - types.ModuleName, metrics.Deleveraging, metrics.NonOverlappingBankruptcyPrices, metrics.Count, - ) + numSubaccountsWithNonOverlappingBankruptcyPrices++ } return deltaQuantumsRemaining.Sign() == 0 }, k.GetPseudoRand(ctx), ) - telemetry.SetGauge(float32(numSubaccountsIterated), metrics.NumSubaccountsIterated, metrics.Count) + telemetry.IncrCounter( + float32(numSubaccountsIterated), + types.ModuleName, metrics.Deleveraging, metrics.NumSubaccountsIterated, metrics.Count, + ) + telemetry.IncrCounter( + float32(numSubaccountsWithNonOverlappingBankruptcyPrices), + types.ModuleName, metrics.Deleveraging, metrics.NonOverlappingBankruptcyPrices, metrics.Count, + ) + telemetry.IncrCounter( + float32(numSubaccountsWithNoOpenPositionOnOppositeSide), + types.ModuleName, metrics.Deleveraging, metrics.NoOpenPositionOnOppositeSide, metrics.Count, + ) if deltaQuantumsRemaining.Sign() == 0 { // Deleveraging was successful. diff --git a/protocol/x/clob/keeper/keeper.go b/protocol/x/clob/keeper/keeper.go index 299f95e6ca..0295da9c57 100644 --- a/protocol/x/clob/keeper/keeper.go +++ b/protocol/x/clob/keeper/keeper.go @@ -44,8 +44,9 @@ type ( memStoreInitialized *atomic.Bool - MaxLiquidationOrdersPerBlock uint32 - MaxDeleveragingAttemptsPerBlock uint32 + MaxLiquidationOrdersPerBlock uint32 + MaxDeleveragingAttemptsPerBlock uint32 + MaxDeleveragingSubaccountsToIterate uint32 mevTelemetryConfig MevTelemetryConfig @@ -108,10 +109,11 @@ func NewKeeper( Host: clobFlags.MevTelemetryHost, Identifier: clobFlags.MevTelemetryIdentifier, }, - MaxLiquidationOrdersPerBlock: clobFlags.MaxLiquidationOrdersPerBlock, - MaxDeleveragingAttemptsPerBlock: clobFlags.MaxDeleveragingAttemptsPerBlock, - placeOrderRateLimiter: placeOrderRateLimiter, - cancelOrderRateLimiter: cancelOrderRateLimiter, + MaxLiquidationOrdersPerBlock: clobFlags.MaxLiquidationOrdersPerBlock, + MaxDeleveragingAttemptsPerBlock: clobFlags.MaxDeleveragingAttemptsPerBlock, + MaxDeleveragingSubaccountsToIterate: clobFlags.MaxDeleveragingSubaccountsToIterate, + placeOrderRateLimiter: placeOrderRateLimiter, + cancelOrderRateLimiter: cancelOrderRateLimiter, } // Provide the keeper to the MemClob. diff --git a/protocol/x/clob/keeper/liquidations.go b/protocol/x/clob/keeper/liquidations.go index 21b208c540..1413ee7d58 100644 --- a/protocol/x/clob/keeper/liquidations.go +++ b/protocol/x/clob/keeper/liquidations.go @@ -979,7 +979,7 @@ func (k Keeper) validateMatchedLiquidation( // Validate that processing the liquidation fill does not leave insufficient funds // in the insurance fund (such that the liquidation couldn't have possibly continued). if !k.IsValidInsuranceFundDelta(ctx, insuranceFundDelta) { - k.Logger(ctx).Info("ProcessMatches: insurance fund has insufficient balance to process the liquidation.") + k.Logger(ctx).Debug("ProcessMatches: insurance fund has insufficient balance to process the liquidation.") return nil, errorsmod.Wrapf( types.ErrInsuranceFundHasInsufficientFunds, "Liquidation order %v, insurance fund delta %v",