feat: refactor for sp exit and bucket migrate (#328)

CHANGELOG.md

@@ -10,6 +10,7 @@ Features
 * [#326](https://github.com/bnb-chain/greenfield/pull/326) feat: add bls verification
 * [#336](https://github.com/bnb-chain/greenfield/pull/336) feat: add tendermint to sdk
 * [#341](https://github.com/bnb-chain/greenfield/pull/341) feat: support cross chain for multiple blockchains
+* [#328](https://github.com/bnb-chain/greenfield/pull/328) feat: refactor for sp exit and bucket migrate 
 
 Bugfixes
 * [#307](https://github.com/bnb-chain/greenfield/pull/307) fix DefaultMaxPayloadSize from 2GB to 64GB 

Makefile

@@ -34,6 +34,9 @@ proto-format-check:
 build:
 	go build -o build/bin/gnfd -ldflags="$(ldflags)" ./cmd/gnfd/main.go
 
+mock-gen:
+	sh ./scripts/mockgen.sh
+
 docker-image:
 	go mod vendor # temporary, should be removed after open source
 	docker build . -t ${IMAGE_NAME}

app/ante/utils_test.go

@@ -11,7 +11,6 @@ import (
 	sdkmath "cosmossdk.io/math"
 	dbm "github.com/cometbft/cometbft-db"
 	abci "github.com/cometbft/cometbft/abci/types"
-	"github.com/cometbft/cometbft/crypto/tmhash"
 	"github.com/cometbft/cometbft/libs/log"
 	tmproto "github.com/cometbft/cometbft/proto/tendermint/types"
 	tmtypes "github.com/cometbft/cometbft/types"
@@ -36,7 +35,6 @@ import (
 	govtypes "github.com/cosmos/cosmos-sdk/x/gov/types/v1beta1"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/bnb-chain/greenfield/app"
@@ -45,6 +43,7 @@ import (
 	"github.com/bnb-chain/greenfield/e2e/core"
 	"github.com/bnb-chain/greenfield/sdk/client/test"
 	"github.com/bnb-chain/greenfield/sdk/keys"
+	"github.com/bnb-chain/greenfield/testutil/sample"
 )
 
 type AnteTestSuite struct {
@@ -97,10 +96,7 @@ func (suite *AnteTestSuite) CreateTestEIP712TxBuilderMsgDelegate(from sdk.AccAdd
 
 func (suite *AnteTestSuite) CreateTestEIP712MsgCreateValidator(from sdk.AccAddress, priv keys.KeyManager, chainId string, gas uint64, gasAmount sdk.Coins) client.TxBuilder {
 	privEd := ed25519.GenPrivKey()
-	blsSecretKey, _ := bls.RandKey()
-	blsPubkey := hex.EncodeToString(blsSecretKey.PublicKey().Marshal())
-	blsProofBuf := blsSecretKey.Sign(tmhash.Sum(blsSecretKey.PublicKey().Marshal()))
-	blsProof := hex.EncodeToString(blsProofBuf.Marshal())
+	blsPubKey, blsProof := sample.RandBlsPubKeyAndBlsProof()
 	msgCreate, err := stakingtypes.NewMsgCreateValidator(
 		from,
 		privEd.PubKey(),
@@ -112,7 +108,7 @@ func (suite *AnteTestSuite) CreateTestEIP712MsgCreateValidator(from sdk.AccAddre
 		from,
 		from,
 		from,
-		blsPubkey,
+		blsPubKey,
 		blsProof,
 	)
 	suite.Require().NoError(err)
@@ -144,18 +140,15 @@ func (suite *AnteTestSuite) CreateTestEIP712GrantAllowance(from sdk.AccAddress,
 func (suite *AnteTestSuite) CreateTestEIP712MsgEditValidator(from sdk.AccAddress, priv keys.KeyManager, chainId string, gas uint64, gasAmount sdk.Coins) client.TxBuilder {
 	newRelayerAddr := core.GenRandomAddr()
 	newChallengerAddr := core.GenRandomAddr()
-	blsSecretKey, _ := bls.RandKey()
-	blsPk := hex.EncodeToString(blsSecretKey.PublicKey().Marshal())
-	blsProofBuf := blsSecretKey.Sign(tmhash.Sum(blsSecretKey.PublicKey().Marshal()))
-	blsProof := hex.EncodeToString(blsProofBuf.Marshal())
+	blsPubKey, blsProof := sample.RandBlsPubKeyAndBlsProof()
 	msgEdit := stakingtypes.NewMsgEditValidator(
 		from,
 		stakingtypes.NewDescription("moniker", "identity", "website", "security_contract", "details"),
 		nil,
 		nil,
 		newRelayerAddr,
 		newChallengerAddr,
-		blsPk,
+		blsPubKey,
 		blsProof,
 	)
 	return suite.CreateTestEIP712CosmosTxBuilder(from, priv, chainId, gas, gasAmount, msgEdit)
@@ -176,10 +169,7 @@ func (suite *AnteTestSuite) CreateTestEIP712MsgSubmitEvidence(from sdk.AccAddres
 
 func (suite *AnteTestSuite) CreateTestEIP712TxBuilderMsgSubmitProposalV1(from sdk.AccAddress, priv keys.KeyManager, chainId string, gas uint64, gasAmount sdk.Coins) client.TxBuilder {
 	privEd := ed25519.GenPrivKey()
-	blsSecretKey, _ := bls.RandKey()
-	blsPubkey := hex.EncodeToString(blsSecretKey.PublicKey().Marshal())
-	blsProofBuf := blsSecretKey.Sign(tmhash.Sum(blsSecretKey.PublicKey().Marshal()))
-	blsProof := hex.EncodeToString(blsProofBuf.Marshal())
+	blsPubKey, blsProof := sample.RandBlsPubKeyAndBlsProof()
 	msgCreate, err := stakingtypes.NewMsgCreateValidator(
 		from,
 		privEd.PubKey(),
@@ -191,7 +181,7 @@ func (suite *AnteTestSuite) CreateTestEIP712TxBuilderMsgSubmitProposalV1(from sd
 		from,
 		from,
 		from,
-		blsPubkey,
+		blsPubKey,
 		blsProof,
 	)
 	suite.Require().NoError(err)

app/app.go

@@ -107,6 +107,9 @@ import (
 	storagemodule "github.com/bnb-chain/greenfield/x/storage"
 	storagemodulekeeper "github.com/bnb-chain/greenfield/x/storage/keeper"
 	storagemoduletypes "github.com/bnb-chain/greenfield/x/storage/types"
+	virtualgroupmodule "github.com/bnb-chain/greenfield/x/virtualgroup"
+	virtualgroupmodulekeeper "github.com/bnb-chain/greenfield/x/virtualgroup/keeper"
+	virtualgroupmoduletypes "github.com/bnb-chain/greenfield/x/virtualgroup/types"
 )
 
 const (
@@ -158,22 +161,24 @@ var (
 		spmodule.AppModuleBasic{},
 		paymentmodule.AppModuleBasic{},
 		permissionmodule.AppModuleBasic{},
+		virtualgroupmodule.AppModuleBasic{},
 		storagemodule.AppModuleBasic{},
 		challengemodule.AppModuleBasic{},
 	)
 
 	// module account permissions
 	maccPerms = map[string][]string{
-		authtypes.FeeCollectorName:       nil,
-		distrtypes.ModuleName:            nil,
-		stakingtypes.BondedPoolName:      {authtypes.Burner, authtypes.Staking},
-		stakingtypes.NotBondedPoolName:   {authtypes.Burner, authtypes.Staking},
-		govtypes.ModuleName:              {authtypes.Burner},
-		paymentmoduletypes.ModuleName:    {authtypes.Minter, authtypes.Burner, authtypes.Staking},
-		crosschaintypes.ModuleName:       {authtypes.Minter},
-		permissionmoduletypes.ModuleName: nil,
-		bridgemoduletypes.ModuleName:     nil,
-		spmoduletypes.ModuleName:         {authtypes.Staking},
+		authtypes.FeeCollectorName:         nil,
+		distrtypes.ModuleName:              nil,
+		stakingtypes.BondedPoolName:        {authtypes.Burner, authtypes.Staking},
+		stakingtypes.NotBondedPoolName:     {authtypes.Burner, authtypes.Staking},
+		govtypes.ModuleName:                {authtypes.Burner},
+		paymentmoduletypes.ModuleName:      {authtypes.Minter, authtypes.Burner, authtypes.Staking},
+		crosschaintypes.ModuleName:         {authtypes.Minter},
+		permissionmoduletypes.ModuleName:   nil,
+		bridgemoduletypes.ModuleName:       nil,
+		spmoduletypes.ModuleName:           {authtypes.Staking},
+		virtualgroupmoduletypes.ModuleName: nil,
 	}
 )
 
@@ -230,6 +235,7 @@ type App struct {
 	PaymentKeeper          paymentmodulekeeper.Keeper
 	ChallengeKeeper        challengemodulekeeper.Keeper
 	PermissionmoduleKeeper permissionmodulekeeper.Keeper
+	VirtualgroupKeeper     virtualgroupmodulekeeper.Keeper
 	StorageKeeper          storagemodulekeeper.Keeper
 
 	// mm is the module manager
@@ -284,6 +290,7 @@ func New(
 		bridgemoduletypes.StoreKey,
 		gashubtypes.StoreKey,
 		spmoduletypes.StoreKey,
+		virtualgroupmoduletypes.StoreKey,
 		paymentmoduletypes.StoreKey,
 		permissionmoduletypes.StoreKey,
 		storagemoduletypes.StoreKey,
@@ -457,6 +464,18 @@ func New(
 	)
 	paymentModule := paymentmodule.NewAppModule(appCodec, app.PaymentKeeper, app.AccountKeeper, app.BankKeeper)
 
+	app.VirtualgroupKeeper = *virtualgroupmodulekeeper.NewKeeper(
+		appCodec,
+		keys[virtualgroupmoduletypes.StoreKey],
+		tKeys[virtualgroupmoduletypes.TStoreKey],
+		authtypes.NewModuleAddress(virtualgroupmoduletypes.ModuleName).String(),
+		app.SpKeeper,
+		app.AccountKeeper,
+		app.BankKeeper,
+		app.PaymentKeeper,
+	)
+	virtualgroupModule := virtualgroupmodule.NewAppModule(appCodec, app.VirtualgroupKeeper, app.SpKeeper)
+
 	app.PermissionmoduleKeeper = *permissionmodulekeeper.NewKeeper(
 		appCodec,
 		keys[permissionmoduletypes.StoreKey],
@@ -474,6 +493,7 @@ func New(
 		app.PaymentKeeper,
 		app.PermissionmoduleKeeper,
 		app.CrossChainKeeper,
+		app.VirtualgroupKeeper,
 		authtypes.NewModuleAddress(govtypes.ModuleName).String(),
 	)
 	storageModule := storagemodule.NewAppModule(appCodec, app.StorageKeeper, app.AccountKeeper, app.BankKeeper, app.SpKeeper)
@@ -521,6 +541,7 @@ func New(
 		bridgeModule,
 		gashubModule,
 		spModule,
+		virtualgroupModule,
 		paymentModule,
 		permissionModule,
 		storageModule,
@@ -549,6 +570,7 @@ func New(
 		bridgemoduletypes.ModuleName,
 		gashubtypes.ModuleName,
 		spmoduletypes.ModuleName,
+		virtualgroupmoduletypes.ModuleName,
 		paymentmoduletypes.ModuleName,
 		permissionmoduletypes.ModuleName,
 		storagemoduletypes.ModuleName,
@@ -573,6 +595,7 @@ func New(
 		bridgemoduletypes.ModuleName,
 		gashubtypes.ModuleName,
 		spmoduletypes.ModuleName,
+		virtualgroupmoduletypes.ModuleName,
 		paymentmoduletypes.ModuleName,
 		permissionmoduletypes.ModuleName,
 		storagemoduletypes.ModuleName,
@@ -602,6 +625,7 @@ func New(
 		oracletypes.ModuleName,
 		bridgemoduletypes.ModuleName,
 		spmoduletypes.ModuleName,
+		virtualgroupmoduletypes.ModuleName,
 		paymentmoduletypes.ModuleName,
 		permissionmoduletypes.ModuleName,
 		storagemoduletypes.ModuleName,
@@ -684,6 +708,11 @@ func New(
 		tmos.Exit("cannot convert bank store to ival store")
 	}
 	bankIavl.EnableDiff()
+	paymentIavl, ok := ms.GetCommitStore(keys[paymentmoduletypes.StoreKey]).(*iavl.Store)
+	if !ok {
+		tmos.Exit("cannot convert payment store to ival store")
+	}
+	paymentIavl.EnableDiff()
 
 	app.initModules(ctx)
 
@@ -727,7 +756,8 @@ func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.R
 func (app *App) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock {
 	resp := app.mm.EndBlock(ctx, req)
 	bankIavl, _ := app.CommitMultiStore().GetCommitStore(sdk.NewKVStoreKey(banktypes.StoreKey)).(*iavl.Store)
-	app.reconBalance(ctx, bankIavl)
+	paymentIavl, _ := app.CommitMultiStore().GetCommitStore(sdk.NewKVStoreKey(paymentmoduletypes.StoreKey)).(*iavl.Store)
+	app.reconcile(ctx, bankIavl, paymentIavl)
 	return resp
 }
 

app/reconciliation.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"cosmossdk.io/math"
+	paymenttypes "github.com/bnb-chain/greenfield/x/payment/types"
 	"github.com/cosmos/cosmos-sdk/store/iavl"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
@@ -20,10 +21,12 @@ var (
 	SupplyKey          = banktypes.SupplyKey
 	DenomAddressPrefix = banktypes.DenomAddressPrefix
 	BalancesPrefix     = banktypes.BalancesPrefix
+
+	StreamRecordKeyPrefix = paymenttypes.StreamRecordKeyPrefix
 )
 
-// reconBalance will do reconciliation for accounts balances.
-func (app *App) reconBalance(ctx sdk.Context, bankIavl *iavl.Store) {
+// reconcile will do reconciliation for accounts balances.
+func (app *App) reconcile(ctx sdk.Context, bankIavl *iavl.Store, paymentIavl *iavl.Store) {
 	if ctx.BlockHeight() <= 2 {
 		return
 	}
@@ -33,14 +36,20 @@ func (app *App) reconBalance(ctx sdk.Context, bankIavl *iavl.Store) {
 		panic(fmt.Sprintf("unbalanced state at block height %d, please use hardfork to bypass it", height))
 	}
 
-	balanced := app.reconBankChanges(ctx, bankIavl)
+	bankBalanced := app.reconBankChanges(ctx, bankIavl)
 	bankIavl.ResetDiff()
 
-	if !balanced {
+	paymentBalanced := app.reconPaymentChanges(ctx, paymentIavl)
+	paymentIavl.ResetDiff()
+
+	if !bankBalanced || !paymentBalanced {
+		ctx.Logger().Error("reconciliation unbalanced", "bank", bankBalanced, "payment", paymentBalanced,
+			"height", ctx.BlockHeight())
 		app.saveUnbalancedBlockHeight(ctx)
 	}
 }
 
+// reconBankChanges will reconcile bank balance changes
 func (app *App) reconBankChanges(ctx sdk.Context, bankIavl *iavl.Store) bool {
 	supplyPre := sdk.Coins{}
 	balancePre := sdk.Coins{}
@@ -54,15 +63,15 @@ func (app *App) reconBankChanges(ctx sdk.Context, bankIavl *iavl.Store) bool {
 		kBz := []byte(k)
 		denom := ""
 		isSupply := false
-		if bytes.HasPrefix([]byte(k), SupplyKey) {
+		if bytes.HasPrefix(kBz, SupplyKey) {
 			isSupply = true
 			denom = parseDenomFromSupplyKey(kBz)
 			amount := math.ZeroInt()
 			if vBz := bankIavl.Get(kBz); vBz != nil {
 				amount = parseAmountFromValue(vBz)
 			}
 			supplyCurrent = supplyCurrent.Add(sdk.NewCoin(denom, amount))
-		} else if bytes.HasPrefix([]byte(k), BalancesPrefix) {
+		} else if bytes.HasPrefix(kBz, BalancesPrefix) {
 			denom = parseDenomFromBalanceKey(kBz)
 			amount := math.ZeroInt()
 			if vBz := bankIavl.Get(kBz); vBz != nil {
@@ -97,6 +106,49 @@ func (app *App) reconBankChanges(ctx sdk.Context, bankIavl *iavl.Store) bool {
 	return supplyChanges.IsEqual(balanceChanges)
 }
 
+// reconPaymentChanges will reconcile payment flow rate changes
+func (app *App) reconPaymentChanges(ctx sdk.Context, paymentIavl *iavl.Store) bool {
+	flowCurrent := sdk.ZeroInt()
+	flowPre := sdk.ZeroInt()
+
+	diff := paymentIavl.GetDiff()
+	version := ctx.BlockHeight() - 2
+	ctx.Logger().Debug("reconciliation payment changes", "height", ctx.BlockHeight(), "version", version)
+	for k := range diff {
+		kBz := []byte(k)
+		if bytes.HasPrefix(kBz, StreamRecordKeyPrefix) {
+			if vBz := paymentIavl.Get(kBz); vBz != nil {
+				var sr paymenttypes.StreamRecord
+				err := app.cdc.Unmarshal(vBz, &sr)
+				if err != nil {
+					ctx.Logger().Error("fail to unmarshal stream record", "err", err.Error())
+				} else {
+					flowCurrent = flowCurrent.Add(sr.NetflowRate)
+				}
+			}
+
+			preStore, err := paymentIavl.GetImmutable(version)
+			if err != nil {
+				panic(fmt.Sprintf("fail to find store at version %d", version))
+			}
+			vBz := preStore.Get(kBz)
+			if vBz != nil {
+				var sr paymenttypes.StreamRecord
+				err = app.cdc.Unmarshal(vBz, &sr)
+				if err != nil {
+					ctx.Logger().Error("fail to unmarshal stream record", "err", err.Error())
+				} else {
+					flowPre = flowPre.Add(sr.NetflowRate)
+				}
+			}
+		}
+	}
+
+	ctx.Logger().Debug("reconciliation payment details", "flowCurrent", flowCurrent.String(), "flowPre", flowPre.String(),
+		"height", ctx.BlockHeight(), "version", version)
+	return flowCurrent.Equal(flowPre)
+}
+
 func (app *App) saveUnbalancedBlockHeight(ctx sdk.Context) {
 	reconStore := app.CommitMultiStore().GetCommitStore(sdk.NewKVStoreKey(reconStoreKey)).(*iavl.Store)
 	bz := make([]byte, 8)

deployment/localup/create_sp.json

@@ -22,7 +22,9 @@
       "read_price": "0.087",
       "store_price": "0.0048",
       "free_read_quota": 10000000000,
-      "creator":"0x7b5Fe22B5446f7C62Ea27B8BD71CeF94e03f3dF2"
+      "creator":"0x7b5Fe22B5446f7C62Ea27B8BD71CeF94e03f3dF2",
+      "bls_key": "89f5e82d1bbb7c751b063d6d69d30c5812a088cabb9ed015b1741939a45a896d83b5435d63257b954450c26d857de4e1",
+      "bls_proof": "987406f0dd2e5f5f3e9e3e447f832dbf73809479ea552fe9b23e06e65651c01a5893e92a797d12078ef9b0c9eb72b8d90faaee65f738e222793d94b80a58cd0f329375ee04e8460f12f4772cf42859d30dd6444ed3350c3335aedf1dbde3bb68"
     }
   ],
   "title": "create sp",