From a9fb658ef3056736a575774ca95aaa6fe58a252f Mon Sep 17 00:00:00 2001 From: huangweichang Date: Tue, 7 Jul 2020 11:32:05 +0800 Subject: [PATCH 1/6] deprecate balance, delegation, unbondingDelegation of account update in sync --- service/handler/account.go | 12 +++---- service/handler/block.go | 4 +-- service/handler/tx.go | 4 +-- store/document/account.go | 60 +++++++++++++++++++--------------- store/document/account_test.go | 12 +++---- 5 files changed, 49 insertions(+), 43 deletions(-) diff --git a/service/handler/account.go b/service/handler/account.go index fbbdbc5..cceb5df 100644 --- a/service/handler/account.go +++ b/service/handler/account.go @@ -9,7 +9,7 @@ import ( ) // TODO: sync only save account address, let app to update balance, delegation, unbondingDelegation info -func SaveOrUpdateAccountBalanceInfo(accounts []string, height, timestamp int64) { +func SaveAccountBalanceInfo(accounts []string, height, timestamp int64) { var ( accountModel document.Account ) @@ -21,14 +21,14 @@ func SaveOrUpdateAccountBalanceInfo(accounts []string, height, timestamp int64) coins, accountNumber := helper.QueryAccountInfo(v) coinIris := getCoinIrisFromCoins(coins) - if err := accountModel.UpsertBalanceInfo(v, coinIris, accountNumber, height, timestamp); err != nil { + if err := accountModel.SaveBalanceInfo(v, coinIris, accountNumber, height, timestamp); err != nil { logger.Error("update account balance info fail", logger.Int64("height", height), logger.String("address", v), logger.String("err", err.Error())) } } } -func SaveOrUpdateAccountDelegationInfo(docTx document.CommonTx) { +func SaveAccountDelegationInfo(docTx document.CommonTx) { var ( delegator string accountModel document.Account @@ -47,13 +47,13 @@ func SaveOrUpdateAccountDelegationInfo(docTx document.CommonTx) { Amount: helper.CalculateDelegatorDelegationTokens(delegations), } - if err := accountModel.UpsertDelegationInfo(delegator, delegation, docTx.Height, docTx.Time.Unix()); err != nil { + if err := accountModel.SaveDelegationInfo(delegator, delegation, docTx.Height, docTx.Time.Unix()); err != nil { logger.Error("update account delegation info fail", logger.Int64("height", docTx.Height), logger.String("address", delegator), logger.String("err", err.Error())) } } -func SaveOrUpdateAccountUnbondingDelegationInfo(accounts []string, height, timestamp int64) { +func SaveAccountUnbondingDelegationInfo(accounts []string, height, timestamp int64) { var ( accountModel document.Account ) @@ -68,7 +68,7 @@ func SaveOrUpdateAccountUnbondingDelegationInfo(accounts []string, height, times Amount: helper.CalculateDelegatorUnbondingDelegationTokens(unbondingDelegations), } - if err := accountModel.UpsertUnbondingDelegationInfo(v, unbondingDelegation, height, timestamp); err != nil { + if err := accountModel.SaveUnbondingDelegationInfo(v, unbondingDelegation, height, timestamp); err != nil { logger.Error("update account unbonding delegation info fail", logger.Int64("height", height), logger.String("address", v), logger.String("err", err.Error())) } diff --git a/service/handler/block.go b/service/handler/block.go index 11dcc05..2760ef1 100644 --- a/service/handler/block.go +++ b/service/handler/block.go @@ -144,9 +144,9 @@ func ParseBlock(meta *types.BlockMeta, block *types.Block, validators []*types.V docBlock.Result.EndBlock.Tags, docBlock.Height) accsBalanceNeedUpdated = helper.DistinctStringSlice(append(accsBalanceNeedUpdated, accsBalanceNeedUpdatedByParseTxs...)) - SaveOrUpdateAccountBalanceInfo(accsBalanceNeedUpdated, docBlock.Height, docBlock.Time.Unix()) + SaveAccountBalanceInfo(accsBalanceNeedUpdated, docBlock.Height, docBlock.Time.Unix()) - SaveOrUpdateAccountUnbondingDelegationInfo(accsUnbondingDelegationNeedUpdated, docBlock.Height, docBlock.Time.Unix()) + SaveAccountUnbondingDelegationInfo(accsUnbondingDelegationNeedUpdated, docBlock.Height, docBlock.Time.Unix()) return docBlock } diff --git a/service/handler/tx.go b/service/handler/tx.go index 133acd6..40e5bf7 100644 --- a/service/handler/tx.go +++ b/service/handler/tx.go @@ -38,11 +38,11 @@ func HandleTx(block *types.Block) ([]string, error) { //handleTokenFlow(blockWithTags, tx, &batch) // save or update account delegations info and unbonding delegation info - SaveOrUpdateAccountDelegationInfo(tx) + SaveAccountDelegationInfo(tx) switch tx.Type { case constant.TxTypeStakeBeginUnbonding: accounts := []string{tx.From} - SaveOrUpdateAccountUnbondingDelegationInfo(accounts, tx.Height, tx.Time.Unix()) + SaveAccountUnbondingDelegationInfo(accounts, tx.Height, tx.Time.Unix()) } // get accounts which balance need updated by parse tx diff --git a/store/document/account.go b/store/document/account.go index 63ef4ed..3bdc6e6 100644 --- a/store/document/account.go +++ b/store/document/account.go @@ -67,8 +67,8 @@ func (d Account) getAccountByPK() (Account, error) { return res, nil } -// save or update account balance info -func (d Account) UpsertBalanceInfo(address string, balance store.Coin, accountNumber uint64, height, timestamp int64) error { +// save account balance info +func (d Account) SaveBalanceInfo(address string, balance store.Coin, accountNumber uint64, height, timestamp int64) error { d.Address = address if account, err := d.getAccountByPK(); err != nil { return err @@ -85,19 +85,21 @@ func (d Account) UpsertBalanceInfo(address string, balance store.Coin, accountNu account.Address = address account.Total = balance return d.Save(account) - } else { - // record already exist - account.Total = store.Coin{ - Denom: balance.Denom, - Amount: balance.Amount + account.Delegation.Amount + account.UnbondingDelegation.Amount, - } - return store.Update(account) } + //else { + // // record already exist + // account.Total = store.Coin{ + // Denom: balance.Denom, + // Amount: balance.Amount + account.Delegation.Amount + account.UnbondingDelegation.Amount, + // } + // return store.Update(account) + //} + return nil } } -// save or update delegation info -func (d Account) UpsertDelegationInfo(address string, delegation store.Coin, height, timestamp int64) error { +// save delegation info +func (d Account) SaveDelegationInfo(address string, delegation store.Coin, height, timestamp int64) error { d.Address = address if account, err := d.getAccountByPK(); err != nil { return err @@ -112,19 +114,21 @@ func (d Account) UpsertDelegationInfo(address string, delegation store.Coin, hei account.Address = address account.Total = delegation return d.Save(account) - } else { - // record exist - account.Total = store.Coin{ - Denom: delegation.Denom, - Amount: account.CoinIris.Amount + delegation.Amount + account.UnbondingDelegation.Amount, - } - return store.Update(account) } + //else { + // // record exist + // account.Total = store.Coin{ + // Denom: delegation.Denom, + // Amount: account.CoinIris.Amount + delegation.Amount + account.UnbondingDelegation.Amount, + // } + // return store.Update(account) + //} + return nil } } -// save or update unbondingDelegation info -func (d Account) UpsertUnbondingDelegationInfo(address string, unbondingDelegation store.Coin, height, timestamp int64) error { +// save unbondingDelegation info +func (d Account) SaveUnbondingDelegationInfo(address string, unbondingDelegation store.Coin, height, timestamp int64) error { d.Address = address if account, err := d.getAccountByPK(); err != nil { return err @@ -140,13 +144,15 @@ func (d Account) UpsertUnbondingDelegationInfo(address string, unbondingDelegati account.Address = address account.Total = unbondingDelegation return d.Save(account) - } else { - // record exist - account.Total = store.Coin{ - Denom: unbondingDelegation.Denom, - Amount: account.CoinIris.Amount + account.Delegation.Amount + unbondingDelegation.Amount, - } - return store.Update(account) } + //else { + // // record exist + // account.Total = store.Coin{ + // Denom: unbondingDelegation.Denom, + // Amount: account.CoinIris.Amount + account.Delegation.Amount + unbondingDelegation.Amount, + // } + // return store.Update(account) + //} + return nil } } diff --git a/store/document/account_test.go b/store/document/account_test.go index d80899e..184003f 100644 --- a/store/document/account_test.go +++ b/store/document/account_test.go @@ -49,14 +49,14 @@ func TestAccount_UpsertBalanceInfo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { d := Account{} - if err := d.UpsertBalanceInfo(tt.args.address, tt.args.balance, tt.args.accountNumber, tt.args.height, tt.args.timestamp); err != nil { + if err := d.SaveBalanceInfo(tt.args.address, tt.args.balance, tt.args.accountNumber, tt.args.height, tt.args.timestamp); err != nil { t.Fatal(err) } }) } } -func TestAccount_UpsertDelegationInfo(t *testing.T) { +func TestAccount_SaveDelegationInfo(t *testing.T) { type fields struct { } type args struct { @@ -76,8 +76,8 @@ func TestAccount_UpsertDelegationInfo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { d := Account{} - if err := d.UpsertDelegationInfo(tt.args.address, tt.args.delegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { - t.Errorf("Account.UpsertDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) + if err := d.SaveDelegationInfo(tt.args.address, tt.args.delegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { + t.Errorf("Account.SaveDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) } }) } @@ -103,8 +103,8 @@ func TestAccount_UpsertUnbondingDelegationInfo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { d := Account{} - if err := d.UpsertUnbondingDelegationInfo(tt.args.address, tt.args.unbondingDelegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { - t.Errorf("Account.UpsertUnbondingDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) + if err := d.SaveUnbondingDelegationInfo(tt.args.address, tt.args.unbondingDelegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { + t.Errorf("Account.SaveUnbondingDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) } }) } From e738b4b386bf497c1bb40e7bf21fde006b72da2f Mon Sep 17 00:00:00 2001 From: huangweichang Date: Wed, 8 Jul 2020 10:29:53 +0800 Subject: [PATCH 2/6] improve code --- store/document/account.go | 68 +++++++++++++-------------------------- 1 file changed, 23 insertions(+), 45 deletions(-) diff --git a/store/document/account.go b/store/document/account.go index 3bdc6e6..2e03d3e 100644 --- a/store/document/account.go +++ b/store/document/account.go @@ -73,28 +73,20 @@ func (d Account) SaveBalanceInfo(address string, balance store.Coin, accountNumb if account, err := d.getAccountByPK(); err != nil { return err } else { + if account.Address != "" { + return nil + } + // record not exist account.AccountNumber = accountNumber account.CoinIris = balance account.CoinIrisUpdateHeight = height account.CoinIrisUpdateAt = timestamp account.TotalUpdateHeight = height account.TotalUpdateAt = timestamp + account.Address = address + account.Total = balance + return d.Save(account) - if account.Address == "" { - // record not exist - account.Address = address - account.Total = balance - return d.Save(account) - } - //else { - // // record already exist - // account.Total = store.Coin{ - // Denom: balance.Denom, - // Amount: balance.Amount + account.Delegation.Amount + account.UnbondingDelegation.Amount, - // } - // return store.Update(account) - //} - return nil } } @@ -104,26 +96,19 @@ func (d Account) SaveDelegationInfo(address string, delegation store.Coin, heigh if account, err := d.getAccountByPK(); err != nil { return err } else { + if account.Address != "" { + return nil + } + // record not exist account.Delegation = delegation account.DelegationUpdateHeight = height account.DelegationUpdateAt = timestamp account.TotalUpdateHeight = height account.TotalUpdateAt = timestamp - if account.Address == "" { - // record not exist - account.Address = address - account.Total = delegation - return d.Save(account) - } - //else { - // // record exist - // account.Total = store.Coin{ - // Denom: delegation.Denom, - // Amount: account.CoinIris.Amount + delegation.Amount + account.UnbondingDelegation.Amount, - // } - // return store.Update(account) - //} - return nil + account.Address = address + account.Total = delegation + return d.Save(account) + } } @@ -134,25 +119,18 @@ func (d Account) SaveUnbondingDelegationInfo(address string, unbondingDelegation return err } else { + if account.Address != "" { + return nil + } + // record not exist account.UnbondingDelegation = unbondingDelegation account.UnbondingDelegationUpdateHeight = height account.UnbondingDelegationUpdateAt = timestamp account.TotalUpdateHeight = height account.TotalUpdateAt = timestamp - if account.Address == "" { - // record not exist - account.Address = address - account.Total = unbondingDelegation - return d.Save(account) - } - //else { - // // record exist - // account.Total = store.Coin{ - // Denom: unbondingDelegation.Denom, - // Amount: account.CoinIris.Amount + account.Delegation.Amount + unbondingDelegation.Amount, - // } - // return store.Update(account) - //} - return nil + account.Address = address + account.Total = unbondingDelegation + return d.Save(account) + } } From bbbd8dd3ea12f8d682ca757d079fbc6981fce2ce Mon Sep 17 00:00:00 2001 From: huangweichang Date: Thu, 9 Jul 2020 14:40:29 +0800 Subject: [PATCH 3/6] improve code --- conf/server/types.go | 2 +- go.mod | 4 +- go.sum | 2 + service/handler/account.go | 99 ++++------------ service/handler/block.go | 12 +- service/handler/block_test.go | 2 +- service/handler/tx.go | 27 +---- service/task/task_execute.go | 5 +- service/task/task_execute_test.go | 2 +- store/document/account.go | 80 ++----------- store/document/account_test.go | 182 +++++++++++++++--------------- 11 files changed, 139 insertions(+), 278 deletions(-) diff --git a/conf/server/types.go b/conf/server/types.go index 9576613..b8c0596 100644 --- a/conf/server/types.go +++ b/conf/server/types.go @@ -10,7 +10,7 @@ import ( ) var ( - BlockChainMonitorUrl = []string{"tcp://35.201.147.145:30657"} + BlockChainMonitorUrl = []string{"http://192.168.150.32:26657"} WorkerNumCreateTask = 1 WorkerNumExecuteTask = 60 diff --git a/go.mod b/go.mod index 934eb59..01846c3 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,8 @@ module github.com/irisnet/irishub-sync require ( github.com/go-kit/kit v0.9.0 - github.com/irisnet/irishub v0.16.0-rc0 + github.com/irisnet/irishub v0.16.0 github.com/jolestar/go-commons-pool v2.0.0+incompatible - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.2.1 github.com/robfig/cron v1.2.0 diff --git a/go.sum b/go.sum index b08eb2d..b73a4f7 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/irisnet/iavl v0.12.3 h1:AYqI1q/EVAOCWznYx4FtEZIq8x9yPBs9ZA/Rk3zsTDo= github.com/irisnet/iavl v0.12.3/go.mod h1:qofGh9236iFgVez+gPk7bC+ef5wCw7aIMvAVBK3lT84= github.com/irisnet/irishub v0.16.0-rc0 h1:A+8FOKOZDVeCzUVaiP0HFMiHs0uYpkW3nlvmEPZxGdk= github.com/irisnet/irishub v0.16.0-rc0/go.mod h1:qE/9fLjRS6yUquLtfWmj2P66yPvTI6d8qCaF4BF7TNc= +github.com/irisnet/irishub v0.16.0 h1:Xuvynxz2TsrTsBym3HAcbleogqR4GXP+NUxW25JJcmk= +github.com/irisnet/irishub v0.16.0/go.mod h1:S6Y1vyjutENBFt7mS8UTh6L+91TpenYtmfu0KtND9I8= github.com/irisnet/tendermint v0.32.0 h1:+NFxoUzVq67LLnAtM8hIkeqTRji3hJOUip+T3/ZHU6A= github.com/irisnet/tendermint v0.32.0/go.mod h1:36RAXv6V/Ar3H6ofEtmtxU2u80tYRdQh1z51gxoG9Nk= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/service/handler/account.go b/service/handler/account.go index cceb5df..53e5c85 100644 --- a/service/handler/account.go +++ b/service/handler/account.go @@ -1,90 +1,39 @@ package handler import ( - "github.com/irisnet/irishub-sync/logger" - "github.com/irisnet/irishub-sync/store" "github.com/irisnet/irishub-sync/store/document" "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/logger" + "github.com/irisnet/irishub-sync/types/msg" "github.com/irisnet/irishub-sync/util/helper" + "encoding/json" ) -// TODO: sync only save account address, let app to update balance, delegation, unbondingDelegation info -func SaveAccountBalanceInfo(accounts []string, height, timestamp int64) { - var ( - accountModel document.Account - ) - if len(accounts) == 0 { - return - } - - for _, v := range accounts { - coins, accountNumber := helper.QueryAccountInfo(v) - coinIris := getCoinIrisFromCoins(coins) - - if err := accountModel.SaveBalanceInfo(v, coinIris, accountNumber, height, timestamp); err != nil { - logger.Error("update account balance info fail", logger.Int64("height", height), - logger.String("address", v), logger.String("err", err.Error())) +func saveNewAccount(tx document.CommonTx) { + var accountModel document.Account + switch tx.Type { + case constant.TxTypeTransfer: + accountModel.Address = tx.To + case constant.TxTypeAddTrustee: + if len(tx.Msgs) > 0 { + msgData := msg.DocTxMsgAddTrustee{} + if err := json.Unmarshal([]byte(helper.ToJson(tx.Msgs[0].Msg)), &msgData); err == nil { + accountModel.Address = msgData.Address + } } - } -} - -func SaveAccountDelegationInfo(docTx document.CommonTx) { - var ( - delegator string - accountModel document.Account - ) - switch docTx.Type { - case constant.TxTypeStakeDelegate, constant.TxTypeStakeBeginUnbonding, constant.TxTypeBeginRedelegate, - constant.TxTypeStakeCreateValidator: - delegator = docTx.From - } - if delegator == "" { - return - } - delegations := helper.GetDelegations(delegator) - delegation := store.Coin{ - Denom: constant.IrisAttoUnit, - Amount: helper.CalculateDelegatorDelegationTokens(delegations), - } - if err := accountModel.SaveDelegationInfo(delegator, delegation, docTx.Height, docTx.Time.Unix()); err != nil { - logger.Error("update account delegation info fail", logger.Int64("height", docTx.Height), - logger.String("address", delegator), logger.String("err", err.Error())) + case constant.TxTypeSetWithdrawAddress: + if len(tx.Msgs) > 0 { + msgData := msg.DocTxMsgSetWithdrawAddress{} + if err := json.Unmarshal([]byte(helper.ToJson(tx.Msgs[0].Msg)), &msgData); err == nil { + accountModel.Address = msgData.WithdrawAddr + } + } } -} - -func SaveAccountUnbondingDelegationInfo(accounts []string, height, timestamp int64) { - var ( - accountModel document.Account - ) - - if len(accounts) == 0 { + if accountModel.Address == "" { return } - for _, v := range accounts { - unbondingDelegations := helper.GetUnbondingDelegations(v) - unbondingDelegation := store.Coin{ - Denom: constant.IrisAttoUnit, - Amount: helper.CalculateDelegatorUnbondingDelegationTokens(unbondingDelegations), - } - - if err := accountModel.SaveUnbondingDelegationInfo(v, unbondingDelegation, height, timestamp); err != nil { - logger.Error("update account unbonding delegation info fail", logger.Int64("height", height), - logger.String("address", v), logger.String("err", err.Error())) - } - } -} - -func getCoinIrisFromCoins(coins store.Coins) store.Coin { - if len(coins) > 0 { - for _, v := range coins { - if v.Denom == constant.IrisAttoUnit { - return store.Coin{ - Denom: v.Denom, - Amount: v.Amount, - } - } - } + if err := accountModel.SaveAddress(accountModel.Address); err != nil { + logger.Warn("Save new account address failed", logger.String("err", err.Error())) } - return store.Coin{} } diff --git a/service/handler/block.go b/service/handler/block.go index 2760ef1..08e3573 100644 --- a/service/handler/block.go +++ b/service/handler/block.go @@ -28,8 +28,7 @@ const ( unDelegationSubject = "Undelegation" ) -func ParseBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Validator, - accsBalanceNeedUpdatedByParseTxs []string) document.Block { +func ParseBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Validator) document.Block { cdc := types.GetCodec() hexFunc := func(bytes []byte) string { @@ -139,15 +138,6 @@ func ParseBlock(meta *types.BlockMeta, block *types.Block, validators []*types.V } } - // save or update account balance info and unbonding delegation info by parse block coin flow - accsBalanceNeedUpdated, accsUnbondingDelegationNeedUpdated := getAccountsFromCoinFlow( - docBlock.Result.EndBlock.Tags, docBlock.Height) - - accsBalanceNeedUpdated = helper.DistinctStringSlice(append(accsBalanceNeedUpdated, accsBalanceNeedUpdatedByParseTxs...)) - SaveAccountBalanceInfo(accsBalanceNeedUpdated, docBlock.Height, docBlock.Time.Unix()) - - SaveAccountUnbondingDelegationInfo(accsUnbondingDelegationNeedUpdated, docBlock.Height, docBlock.Time.Unix()) - return docBlock } diff --git a/service/handler/block_test.go b/service/handler/block_test.go index cd80806..1fcd007 100644 --- a/service/handler/block_test.go +++ b/service/handler/block_test.go @@ -65,7 +65,7 @@ func TestParseBlock(t *testing.T) { } else { validators = valRes.Validators } - blockDoc := ParseBlock(res.BlockMeta, res.Block, validators, nil) + blockDoc := ParseBlock(res.BlockMeta, res.Block, validators) resBytes, _ := json.MarshalIndent(blockDoc, "", "\t") t.Log(string(resBytes)) diff --git a/service/handler/tx.go b/service/handler/tx.go index 40e5bf7..964eef0 100644 --- a/service/handler/tx.go +++ b/service/handler/tx.go @@ -4,23 +4,15 @@ import ( "github.com/irisnet/irishub-sync/store" "github.com/irisnet/irishub-sync/store/document" "github.com/irisnet/irishub-sync/types" - "github.com/irisnet/irishub-sync/util/constant" "github.com/irisnet/irishub-sync/util/helper" "gopkg.in/mgo.v2/bson" "gopkg.in/mgo.v2/txn" - "strings" ) -func HandleTx(block *types.Block) ([]string, error) { +func HandleTx(block *types.Block) (error) { var ( batch []txn.Op - accsBalanceNeedUpdated []string ) - getAccsBalanceNeedUpdated := func(addr string) { - if strings.HasPrefix(addr, types.Bech32AccountAddrPrefix) { - accsBalanceNeedUpdated = append(accsBalanceNeedUpdated, addr) - } - } for _, txByte := range block.Txs { tx := helper.ParseTx(txByte, block) @@ -37,25 +29,16 @@ func HandleTx(block *types.Block) ([]string, error) { handleProposal(tx) //handleTokenFlow(blockWithTags, tx, &batch) - // save or update account delegations info and unbonding delegation info - SaveAccountDelegationInfo(tx) - switch tx.Type { - case constant.TxTypeStakeBeginUnbonding: - accounts := []string{tx.From} - SaveAccountUnbondingDelegationInfo(accounts, tx.Height, tx.Time.Unix()) - } - - // get accounts which balance need updated by parse tx - getAccsBalanceNeedUpdated(tx.From) - getAccsBalanceNeedUpdated(tx.To) + // save new account address + saveNewAccount(tx) } if len(batch) > 0 { err := store.Txn(batch) if err != nil { - return accsBalanceNeedUpdated, err + return err } } - return accsBalanceNeedUpdated, nil + return nil } diff --git a/service/task/task_execute.go b/service/task/task_execute.go index 382490b..3656157 100644 --- a/service/task/task_execute.go +++ b/service/task/task_execute.go @@ -299,8 +299,7 @@ func parseBlock(b int64, client *helper.Client) (document.Block, error) { } } - accsBalanceNeedUpdatedByParseTxs, err := handler.HandleTx(block.Block) - if err != nil { + if err := handler.HandleTx(block.Block); err != nil { return blockDoc, err } @@ -313,7 +312,7 @@ func parseBlock(b int64, client *helper.Client) (document.Block, error) { validators = res.Validators } - return handler.ParseBlock(block.BlockMeta, block.Block, validators, accsBalanceNeedUpdatedByParseTxs), nil + return handler.ParseBlock(block.BlockMeta, block.Block, validators), nil } // assert task worker unchanged diff --git a/service/task/task_execute_test.go b/service/task/task_execute_test.go index cafd8e1..4f55397 100644 --- a/service/task/task_execute_test.go +++ b/service/task/task_execute_test.go @@ -109,7 +109,7 @@ func Test_parseBlock(t *testing.T) { name: "test parse block", args: args{ client: client, - b: 1588, + b: 4379192, }, }, } diff --git a/store/document/account.go b/store/document/account.go index 2e03d3e..c6f049c 100644 --- a/store/document/account.go +++ b/store/document/account.go @@ -12,20 +12,15 @@ const ( ) type Account struct { - Address string `bson:"address"` - AccountNumber uint64 `bson:"account_number"` - CoinIris store.Coin `bson:"coin_iris"` - Delegation store.Coin `bson:"delegation"` - UnbondingDelegation store.Coin `bson:"unbonding_delegation"` - Total store.Coin `bson:"total"` - CoinIrisUpdateHeight int64 `bson:"coin_iris_update_height"` - CoinIrisUpdateAt int64 `bson:"coin_iris_update_at"` - DelegationUpdateHeight int64 `bson:"delegation_update_height"` - DelegationUpdateAt int64 `bson:"delegation_update_at"` - UnbondingDelegationUpdateHeight int64 `bson:"unbonding_delegation_update_height"` - UnbondingDelegationUpdateAt int64 `bson:"unbonding_delegation_update_at"` - TotalUpdateHeight int64 `bson:"total_update_height"` - TotalUpdateAt int64 `bson:"total_update_at"` + Address string `bson:"address"` + AccountNumber uint64 `bson:"account_number"` + Total store.Coin `bson:"total"` + CoinIris store.Coin `bson:"coin_iris"` + Delegation store.Coin `bson:"delegation"` + UnbondingDelegation store.Coin `bson:"unbonding_delegation"` + Rewards store.Coin `bson:"rewards"` + UpdateAt int64 `bson:"update_at"` + CreateAt int64 `bson:"create_at"` } func (d Account) Name() string { @@ -67,8 +62,8 @@ func (d Account) getAccountByPK() (Account, error) { return res, nil } -// save account balance info -func (d Account) SaveBalanceInfo(address string, balance store.Coin, accountNumber uint64, height, timestamp int64) error { +// save account address +func (d Account) SaveAddress(address string) error { d.Address = address if account, err := d.getAccountByPK(); err != nil { return err @@ -76,60 +71,7 @@ func (d Account) SaveBalanceInfo(address string, balance store.Coin, accountNumb if account.Address != "" { return nil } - // record not exist - account.AccountNumber = accountNumber - account.CoinIris = balance - account.CoinIrisUpdateHeight = height - account.CoinIrisUpdateAt = timestamp - account.TotalUpdateHeight = height - account.TotalUpdateAt = timestamp account.Address = address - account.Total = balance - return d.Save(account) - - } -} - -// save delegation info -func (d Account) SaveDelegationInfo(address string, delegation store.Coin, height, timestamp int64) error { - d.Address = address - if account, err := d.getAccountByPK(); err != nil { - return err - } else { - if account.Address != "" { - return nil - } - // record not exist - account.Delegation = delegation - account.DelegationUpdateHeight = height - account.DelegationUpdateAt = timestamp - account.TotalUpdateHeight = height - account.TotalUpdateAt = timestamp - account.Address = address - account.Total = delegation - return d.Save(account) - - } -} - -// save unbondingDelegation info -func (d Account) SaveUnbondingDelegationInfo(address string, unbondingDelegation store.Coin, height, timestamp int64) error { - d.Address = address - if account, err := d.getAccountByPK(); err != nil { - return err - } else { - - if account.Address != "" { - return nil - } - // record not exist - account.UnbondingDelegation = unbondingDelegation - account.UnbondingDelegationUpdateHeight = height - account.UnbondingDelegationUpdateAt = timestamp - account.TotalUpdateHeight = height - account.TotalUpdateAt = timestamp - account.Address = address - account.Total = unbondingDelegation return d.Save(account) } diff --git a/store/document/account_test.go b/store/document/account_test.go index 184003f..5bc319a 100644 --- a/store/document/account_test.go +++ b/store/document/account_test.go @@ -2,10 +2,7 @@ package document import ( "encoding/json" - "github.com/irisnet/irishub-sync/store" - "github.com/irisnet/irishub-sync/util/constant" "testing" - "time" ) func TestGetAccountByPK(t *testing.T) { @@ -20,92 +17,93 @@ func TestGetAccountByPK(t *testing.T) { t.Log(string(resBytes)) } -func TestAccount_UpsertBalanceInfo(t *testing.T) { - type args struct { - address string - balance store.Coin - accountNumber uint64 - height int64 - timestamp int64 - } - tests := []struct { - name string - args args - }{ - { - name: "TestAccount_UpsertBalanceInfo", - args: args{ - address: "12445", - balance: store.Coin{ - Denom: constant.IrisAttoUnit, - Amount: float64(1), - }, - accountNumber: 1, - height: 121, - timestamp: time.Now().Unix(), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - d := Account{} - if err := d.SaveBalanceInfo(tt.args.address, tt.args.balance, tt.args.accountNumber, tt.args.height, tt.args.timestamp); err != nil { - t.Fatal(err) - } - }) - } -} - -func TestAccount_SaveDelegationInfo(t *testing.T) { - type fields struct { - } - type args struct { - address string - delegation store.Coin - height int64 - timestamp int64 - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - d := Account{} - if err := d.SaveDelegationInfo(tt.args.address, tt.args.delegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { - t.Errorf("Account.SaveDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestAccount_UpsertUnbondingDelegationInfo(t *testing.T) { - type fields struct { - } - type args struct { - address string - unbondingDelegation store.Coin - height int64 - timestamp int64 - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - d := Account{} - if err := d.SaveUnbondingDelegationInfo(tt.args.address, tt.args.unbondingDelegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { - t.Errorf("Account.SaveUnbondingDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} +// +//func TestAccount_UpsertBalanceInfo(t *testing.T) { +// type args struct { +// address string +// balance store.Coin +// accountNumber uint64 +// height int64 +// timestamp int64 +// } +// tests := []struct { +// name string +// args args +// }{ +// { +// name: "TestAccount_UpsertBalanceInfo", +// args: args{ +// address: "12445", +// balance: store.Coin{ +// Denom: constant.IrisAttoUnit, +// Amount: float64(1), +// }, +// accountNumber: 1, +// height: 121, +// timestamp: time.Now().Unix(), +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// d := Account{} +// if err := d.SaveBalanceInfo(tt.args.address, tt.args.balance, tt.args.accountNumber, tt.args.height, tt.args.timestamp); err != nil { +// t.Fatal(err) +// } +// }) +// } +//} +// +//func TestAccount_SaveDelegationInfo(t *testing.T) { +// type fields struct { +// } +// type args struct { +// address string +// delegation store.Coin +// height int64 +// timestamp int64 +// } +// tests := []struct { +// name string +// fields fields +// args args +// wantErr bool +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// d := Account{} +// if err := d.SaveDelegationInfo(tt.args.address, tt.args.delegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { +// t.Errorf("Account.SaveDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) +// } +// }) +// } +//} +// +//func TestAccount_UpsertUnbondingDelegationInfo(t *testing.T) { +// type fields struct { +// } +// type args struct { +// address string +// unbondingDelegation store.Coin +// height int64 +// timestamp int64 +// } +// tests := []struct { +// name string +// fields fields +// args args +// wantErr bool +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// d := Account{} +// if err := d.SaveUnbondingDelegationInfo(tt.args.address, tt.args.unbondingDelegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { +// t.Errorf("Account.SaveUnbondingDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) +// } +// }) +// } +//} From f1424cbaf86edb6586f09c2d8c0bf45b3c3c55ab Mon Sep 17 00:00:00 2001 From: huangweichang Date: Thu, 9 Jul 2020 14:45:11 +0800 Subject: [PATCH 4/6] add create_at and test case update --- store/document/account.go | 2 + store/document/account_test.go | 91 +--------------------------------- 2 files changed, 3 insertions(+), 90 deletions(-) diff --git a/store/document/account.go b/store/document/account.go index c6f049c..7b3de20 100644 --- a/store/document/account.go +++ b/store/document/account.go @@ -4,6 +4,7 @@ import ( "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" + "time" ) const ( @@ -34,6 +35,7 @@ func (d Account) PkKvPair() map[string]interface{} { // override store.Save() // not to check record if exist before save document func (d Account) Save(account Account) error { + account.CreateAt = time.Now().Unix() fn := func(c *mgo.Collection) error { return c.Insert(account) } diff --git a/store/document/account_test.go b/store/document/account_test.go index 5bc319a..2c33975 100644 --- a/store/document/account_test.go +++ b/store/document/account_test.go @@ -17,93 +17,4 @@ func TestGetAccountByPK(t *testing.T) { t.Log(string(resBytes)) } -// -//func TestAccount_UpsertBalanceInfo(t *testing.T) { -// type args struct { -// address string -// balance store.Coin -// accountNumber uint64 -// height int64 -// timestamp int64 -// } -// tests := []struct { -// name string -// args args -// }{ -// { -// name: "TestAccount_UpsertBalanceInfo", -// args: args{ -// address: "12445", -// balance: store.Coin{ -// Denom: constant.IrisAttoUnit, -// Amount: float64(1), -// }, -// accountNumber: 1, -// height: 121, -// timestamp: time.Now().Unix(), -// }, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// d := Account{} -// if err := d.SaveBalanceInfo(tt.args.address, tt.args.balance, tt.args.accountNumber, tt.args.height, tt.args.timestamp); err != nil { -// t.Fatal(err) -// } -// }) -// } -//} -// -//func TestAccount_SaveDelegationInfo(t *testing.T) { -// type fields struct { -// } -// type args struct { -// address string -// delegation store.Coin -// height int64 -// timestamp int64 -// } -// tests := []struct { -// name string -// fields fields -// args args -// wantErr bool -// }{ -// // TODO: Add test cases. -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// d := Account{} -// if err := d.SaveDelegationInfo(tt.args.address, tt.args.delegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { -// t.Errorf("Account.SaveDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) -// } -// }) -// } -//} -// -//func TestAccount_UpsertUnbondingDelegationInfo(t *testing.T) { -// type fields struct { -// } -// type args struct { -// address string -// unbondingDelegation store.Coin -// height int64 -// timestamp int64 -// } -// tests := []struct { -// name string -// fields fields -// args args -// wantErr bool -// }{ -// // TODO: Add test cases. -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// d := Account{} -// if err := d.SaveUnbondingDelegationInfo(tt.args.address, tt.args.unbondingDelegation, tt.args.height, tt.args.timestamp); (err != nil) != tt.wantErr { -// t.Errorf("Account.SaveUnbondingDelegationInfo() error = %v, wantErr %v", err, tt.wantErr) -// } -// }) -// } -//} + From abd72abd7dc9ce2e8edc4d65a78b49040c097e8c Mon Sep 17 00:00:00 2001 From: huangweichang Date: Fri, 10 Jul 2020 16:58:57 +0800 Subject: [PATCH 5/6] improve code about collect address with condition added --- service/handler/account.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/handler/account.go b/service/handler/account.go index 53e5c85..9847c98 100644 --- a/service/handler/account.go +++ b/service/handler/account.go @@ -30,7 +30,8 @@ func saveNewAccount(tx document.CommonTx) { } } } - if accountModel.Address == "" { + //when new address not found or the tx is not success,this address will not be collected + if accountModel.Address == "" || document.TxStatusSuccess != tx.Status { return } if err := accountModel.SaveAddress(accountModel.Address); err != nil { From 7cc42e8b4a6423aa15c0fde60df58b01df4201b9 Mon Sep 17 00:00:00 2001 From: huangweichang Date: Fri, 10 Jul 2020 19:00:57 +0800 Subject: [PATCH 6/6] improve code --- service/handler/account.go | 9 ++++++--- service/handler/tx.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/service/handler/account.go b/service/handler/account.go index 9847c98..3ea0fcc 100644 --- a/service/handler/account.go +++ b/service/handler/account.go @@ -9,8 +9,12 @@ import ( "encoding/json" ) -func saveNewAccount(tx document.CommonTx) { +//when new address not found or the tx is not success,this address will not be collected +func saveNewAccount(tx *document.CommonTx) { var accountModel document.Account + if document.TxStatusSuccess != tx.Status { + return + } switch tx.Type { case constant.TxTypeTransfer: accountModel.Address = tx.To @@ -30,8 +34,7 @@ func saveNewAccount(tx document.CommonTx) { } } } - //when new address not found or the tx is not success,this address will not be collected - if accountModel.Address == "" || document.TxStatusSuccess != tx.Status { + if accountModel.Address == "" { return } if err := accountModel.SaveAddress(accountModel.Address); err != nil { diff --git a/service/handler/tx.go b/service/handler/tx.go index 964eef0..b0e8d35 100644 --- a/service/handler/tx.go +++ b/service/handler/tx.go @@ -30,7 +30,7 @@ func HandleTx(block *types.Block) (error) { //handleTokenFlow(blockWithTags, tx, &batch) // save new account address - saveNewAccount(tx) + saveNewAccount(&tx) } if len(batch) > 0 {