diff --git a/accountresolver/processor.go b/accountresolver/processor.go index 78e2461f..69511f4c 100644 --- a/accountresolver/processor.go +++ b/accountresolver/processor.go @@ -31,11 +31,6 @@ type Stats struct { totalBlockHandlingDuration time.Duration totalBlockReadingDuration time.Duration cacheHit int - totalBlockPushDuration time.Duration - writeDurationAfterLastPush time.Duration - lastBlockPushedAt time.Time - totalDecodingDuration time.Duration - timeToFirstDecodedBlock time.Duration totalTimeWaitingForBlock time.Duration totalAccountsResolved int totalAccountsResolvedByCache int @@ -67,21 +62,14 @@ func (s *Stats) Log(logger *zap.Logger) { zap.String("total_block_handling_duration", durafmt.Parse(s.totalBlockHandlingDuration).String()), zap.String("total_block_processing_duration", durafmt.Parse(s.totalBlockProcessingDuration).String()), zap.String("total_transaction_processing_duration", durafmt.Parse(s.totalTransactionProcessingDuration).String()), - zap.String("total_push_duration", durafmt.Parse(s.totalBlockPushDuration).String()), zap.String("total_lookup_duration", durafmt.Parse(s.totalLookupDuration).String()), zap.String("total_extend_duration", durafmt.Parse(s.totalExtendDuration).String()), zap.String("total_duration", durafmt.Parse(time.Since(s.startProcessing)).String()), zap.String("total_block_reading_duration", durafmt.Parse(s.totalBlockReadingDuration).String()), - zap.String("total_decoding_duration", durafmt.Parse(s.totalDecodingDuration).String()), zap.String("total_time_waiting_for_block", durafmt.Parse(s.totalTimeWaitingForBlock).String()), zap.String("total_time_waiting_for_block", durafmt.Parse(s.totalTimeWaitingForBlock).String()), - //zap.String("average_block_handling_duration", durafmt.Parse(s.totalBlockHandlingDuration/time.Duration(s.totalBlockCount)).String()), - //zap.String("average_block_processing_duration", durafmt.Parse(s.totalBlockProcessingDuration/time.Duration(s.totalBlockCount)).String()), - //zap.String("average_transaction_processing_duration", durafmt.Parse(s.totalTransactionProcessingDuration/time.Duration(s.transactionCount)).String()), zap.String("average_lookup_duration", durafmt.Parse(lookupAvg).String()), zap.String("average_extend_duration", durafmt.Parse(extendAvg).String()), - zap.String("write_duration_after_last_push", durafmt.Parse(time.Since(s.lastBlockPushedAt)).String()), - zap.String("time_to_first_decoded_block", durafmt.Parse(s.timeToFirstDecodedBlock).String()), ) } @@ -239,6 +227,8 @@ func (p *Processor) processMergeBlocksFiles(ctx context.Context, cursor *Cursor, if err != nil { panic(fmt.Errorf("storing cursor at block %d: %w", cursor.slotNum, err)) } + stats.Log(p.logger) + timeOfLastPush = time.Now() } return nil } diff --git a/accountresolver/processor_test.go b/accountresolver/processor_test.go index 0cf24ba8..9b3c2a90 100644 --- a/accountresolver/processor_test.go +++ b/accountresolver/processor_test.go @@ -478,3 +478,82 @@ func Test_BlockResolved(t *testing.T) { blockAccounts := NewAccounts(blockAccountsData) require.Equal(t, expectedBlockAccounts, blockAccounts) } +func Test_BlockResolved_Multiple_extend(t *testing.T) { + tableLookupAccount := accountFromBase58(t, "6pyNrJXyGdDDA3esoLEHJ2uoohcdf2xGT11acfmfyA7Q") + tableLookupToExtendIndex := byte(2) + + expectedCreatedAccounts1 := fromBase58Strings(t, + "He3iAEV5rYjv6Xf7PxKro19eVrC3QAcdic5CF2D2obPt", + ) + expectedCreatedAccounts2 := fromBase58Strings(t, + "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", + ) + solBlock := &pbsol.Block{ + PreviousBlockhash: "9RXPunwLvRcNGiLKwMBFtxmqr3d1rTxSkYYsMZPbKCct", + Blockhash: "6CqnntW5shmcB92VivDAUkKdckn6m7Dmn7nTzSvX1G6o", + ParentSlot: 157_564_919, + Transactions: []*pbsol.ConfirmedTransaction{ + { + Transaction: &pbsol.Transaction{ + Signatures: [][]byte{{0}}, + Message: &pbsol.Message{ + AccountKeys: [][]byte{ + accountFromBase58(t, "AddressLookupTab1e1111111111111111111111111"), + }, + Instructions: []*pbsol.CompiledInstruction{ + { + ProgramIdIndex: 0, + }, + { + ProgramIdIndex: 0, + }, + { + ProgramIdIndex: 0, + }, + }, + }, + }, + Meta: &pbsol.TransactionStatusMeta{ + InnerInstructions: []*pbsol.InnerInstructions{ + { + Index: 0, + Instructions: []*pbsol.InnerInstruction{ + { + ProgramIdIndex: 0, + Accounts: []byte{tableLookupToExtendIndex, 15, 0, 3}, + Data: append([]byte{2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0}, encodeAccounts(expectedCreatedAccounts1)...), + }, + }, + }, + { + Index: 2, + Instructions: []*pbsol.InnerInstruction{ + { + ProgramIdIndex: 0, + Accounts: []byte{tableLookupToExtendIndex, 15, 0, 3}, + Data: append([]byte{2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0}, encodeAccounts(expectedCreatedAccounts2)...), + }, + }, + }, + }, + }, + }, + }, + BlockHeight: &pbsol.BlockHeight{ + BlockHeight: 157_564_920, + }, + Slot: 157_564_920, + } + err := os.RemoveAll("/tmp/my-badger.db") + require.NoError(t, err) + db, err := kvstore.New("badger3:///tmp/my-badger.db") + require.NoError(t, err) + + resolver := NewKVDBAccountsResolver(db, zap.NewNop()) + p := NewProcessor("test", NewKVDBAccountsResolver(db, zap.NewNop()), zap.NewNop()) + err = p.ProcessBlock(context.Background(), &Stats{}, solBlock) + require.NoError(t, err) + + accounts, _, err := resolver.Resolve(context.Background(), 157_564_921, tableLookupAccount) + require.Equal(t, expectedCreatedAccounts1, accounts) +} diff --git a/cmd/firesol/tools_bigtable_block.go b/cmd/firesol/tools_bigtable_block.go index 0e65975a..c63d9365 100644 --- a/cmd/firesol/tools_bigtable_block.go +++ b/cmd/firesol/tools_bigtable_block.go @@ -1,12 +1,9 @@ package main import ( - "encoding/hex" "fmt" "strconv" - "github.com/golang/protobuf/proto" - "cloud.google.com/go/bigtable" "github.com/spf13/cobra" "github.com/streamingfast/cli/sflags" @@ -67,11 +64,11 @@ func bigtableBlockRunE(logger *zap.Logger, tracer logging.Tracer) firecore.Comma if err != nil { return fmt.Errorf("printing block: %w", err) } - hexData, err := proto.Marshal(block) - if err != nil { - return fmt.Errorf("marshaling block: %w", err) - } - fmt.Println("hex:", hex.EncodeToString(hexData)) + //hexData, err := proto.Marshal(block) + //if err != nil { + // return fmt.Errorf("marshaling block: %w", err) + //} + //fmt.Println("hex:", hex.EncodeToString(hexData)) return nil }); err != nil { return fmt.Errorf("failed to find block %d: %w", startBlockNum, err)