[feat][protocol] halt withdrawals if chain outage seen within the las…

…t 5 minutes

proto/dydxprotocol/blocktime/query.proto

@@ -23,6 +23,7 @@ service Query {
   // Queries all recorded downtime info.
   rpc AllDowntimeInfo(QueryAllDowntimeInfoRequest)
       returns (QueryAllDowntimeInfoResponse);
+  option (google.api.http).get = "/dydxprotocol/v4/blocktime/all_downtime_info";
 }
 
 // QueryDowntimeParamsRequest is a request type for the DowntimeParams

protocol/app/app.go

@@ -903,6 +903,7 @@ func New(
 		app.AssetsKeeper,
 		app.BankKeeper,
 		app.PerpetualsKeeper,
+		app.BlockTimeKeeper,
 		app.IndexerEventManager,
 	)
 	subaccountsModule := subaccountsmodule.NewAppModule(

protocol/lib/metrics/metric_keys.go

@@ -21,6 +21,7 @@ const (
 	LiquidationsLiquidationMatchNegativeTNC            = "liquidations_liquidation_match_negative_tnc"
 	ClobMevErrorCount                                  = "clob_mev_error_count"
 	SubaccountsNegativeTncSubaccountSeen               = "negative_tnc_subaccount_seen"
+	SubaccountsChainOutageSeen                         = "chain_outage_seen"
 
 	// Gauges
 	InsuranceFundBalance = "insurance_fund_balance"

protocol/testutil/keeper/subaccounts.go

@@ -1,10 +1,11 @@
 package keeper
 
 import (
-	dbm "github.com/cosmos/cosmos-db"
 	"math/big"
 	"testing"
 
+	dbm "github.com/cosmos/cosmos-db"
+
 	"github.com/dydxprotocol/v4-chain/protocol/testutil/constants"
 
 	"github.com/dydxprotocol/v4-chain/protocol/indexer/common"
@@ -21,6 +22,7 @@ import (
 	"github.com/dydxprotocol/v4-chain/protocol/dtypes"
 	asskeeper "github.com/dydxprotocol/v4-chain/protocol/x/assets/keeper"
 	assettypes "github.com/dydxprotocol/v4-chain/protocol/x/assets/types"
+	blocktimekeeper "github.com/dydxprotocol/v4-chain/protocol/x/blocktime/keeper"
 	perpskeeper "github.com/dydxprotocol/v4-chain/protocol/x/perpetuals/keeper"
 	priceskeeper "github.com/dydxprotocol/v4-chain/protocol/x/prices/keeper"
 	"github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/keeper"
@@ -84,6 +86,7 @@ func createSubaccountsKeeper(
 	ak *asskeeper.Keeper,
 	bk types.BankKeeper,
 	pk *perpskeeper.Keeper,
+	btk *blocktimekeeper.Keeper,
 	transientStoreKey storetypes.StoreKey,
 	msgSenderEnabled bool,
 ) (*keeper.Keeper, storetypes.StoreKey) {
@@ -101,6 +104,7 @@ func createSubaccountsKeeper(
 		ak,
 		bk,
 		pk,
+		btk,
 		mockIndexerEventsManager,
 	)
 

protocol/x/subaccounts/keeper/keeper.go

@@ -18,6 +18,7 @@ type (
 		assetsKeeper        types.AssetsKeeper
 		bankKeeper          types.BankKeeper
 		perpetualsKeeper    types.PerpetualsKeeper
+		blocktimeKeeper     types.BlocktimeKeeper
 		indexerEventManager indexer_manager.IndexerEventManager
 	}
 )
@@ -28,6 +29,7 @@ func NewKeeper(
 	assetsKeeper types.AssetsKeeper,
 	bankKeeper types.BankKeeper,
 	perpetualsKeeper types.PerpetualsKeeper,
+	blocktimeKeeper types.BlocktimeKeeper,
 	indexerEventManager indexer_manager.IndexerEventManager,
 ) *Keeper {
 	return &Keeper{

protocol/x/subaccounts/keeper/subaccount.go

@@ -1,13 +1,14 @@
 package keeper
 
 import (
-	storetypes "cosmossdk.io/store/types"
 	"errors"
 	"fmt"
 	"math/big"
 	"math/rand"
 	"time"
 
+	storetypes "cosmossdk.io/store/types"
+
 	errorsmod "cosmossdk.io/errors"
 	"cosmossdk.io/store/prefix"
 	"github.com/cosmos/cosmos-sdk/telemetry"
@@ -483,8 +484,10 @@ func (k Keeper) internalCanUpdateSubaccounts(
 	success = true
 	successPerUpdate = make([]types.UpdateResult, len(settledUpdates))
 
-	// Block all withdrawals and transfers if there was a negative TNC subaccount seen within the
-	// last `WITHDRAWAL_AND_TRANSFERS_BLOCKED_AFTER_NEGATIVE_TNC_SUBACCOUNT_SEEN_BLOCKS`.
+	// Block all withdrawals and transfers if either of the following is true within the last
+	// `WITHDRAWAL_AND_TRANSFERS_BLOCKED_AFTER_NEGATIVE_TNC_SUBACCOUNT_SEEN_BLOCKS`:
+	// - There was a negative TNC subaccount seen.
+	// - There was a chain outage that lasted at least five minutes.
 	if updateType == types.Withdrawal || updateType == types.Transfer {
 		lastBlockNegativeTncSubaccountSeen, exists := k.GetNegativeTncSubaccountSeenAtBlock(ctx)
 		currentBlock := uint32(ctx.BlockHeight())
@@ -501,8 +504,26 @@ func (k Keeper) internalCanUpdateSubaccounts(
 			)
 		}
 
-		if exists && currentBlock-lastBlockNegativeTncSubaccountSeen <
-			types.WITHDRAWAL_AND_TRANSFERS_BLOCKED_AFTER_NEGATIVE_TNC_SUBACCOUNT_SEEN_BLOCKS {
+		// Panic if the current block is less than the last block a chain outage was seen.
+		downtimeInfo := k.blocktimeKeeper.GetDowntimeInfoFor(ctx, 5*time.Minute)
+		if currentBlock < downtimeInfo.BlockInfo.Height {
+			panic(
+				fmt.Sprintf(
+					"internalCanUpdateSubaccounts: current block (%d) is less than the last "+
+						"block a chain outage was seen (%d)",
+					currentBlock,
+					downtimeInfo.BlockInfo.Height,
+				),
+			)
+		}
+
+		negativeTncSubaccountSeen := exists && currentBlock-lastBlockNegativeTncSubaccountSeen <
+			types.WITHDRAWAL_AND_TRANSFERS_BLOCKED_AFTER_NEGATIVE_TNC_SUBACCOUNT_SEEN_BLOCKS
+		// TODO: update this to check for existence.
+		chainOutageSeen := currentBlock-downtimeInfo.BlockInfo.Height <
+			types.WITHDRAWAL_AND_TRANSFERS_BLOCKED_AFTER_NEGATIVE_TNC_SUBACCOUNT_SEEN_BLOCKS
+
+		if negativeTncSubaccountSeen || chainOutageSeen {
 			success = false
 			for i := range settledUpdates {
 				successPerUpdate[i] = types.WithdrawalsAndTransfersBlocked
@@ -511,6 +532,8 @@ func (k Keeper) internalCanUpdateSubaccounts(
 				metrics.SubaccountWithdrawalsAndTransfersBlocked,
 				1,
 				metrics.GetLabelForStringValue(metrics.UpdateType, updateType.String()),
+				metrics.GetLabelForBoolValue(metrics.SubaccountsNegativeTncSubaccountSeen, negativeTncSubaccountSeen),
+				metrics.GetLabelForBoolValue(metrics.SubaccountsChainOutageSeen, chainOutageSeen),
 			)
 			return success, successPerUpdate, nil
 		}

protocol/x/subaccounts/types/expected_keepers.go

@@ -3,8 +3,10 @@ package types
 import (
 	"context"
 	"math/big"
+	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	blocktimetypes "github.com/dydxprotocol/v4-chain/protocol/x/blocktime/types"
 	perptypes "github.com/dydxprotocol/v4-chain/protocol/x/perpetuals/types"
 )
 
@@ -87,3 +89,7 @@ type BankKeeper interface {
 	) error
 	SendCoinsFromModuleToModule(ctx context.Context, senderModule, recipientModule string, amt sdk.Coins) error
 }
+
+type BlocktimeKeeper interface {
+	GetDowntimeInfoFor(ctx sdk.Context, duration time.Duration) blocktimetypes.AllDowntimeInfo_DowntimeInfo
+}