fix stats

streamingfast · Oct 25, 2023 · a9bdea9 · a9bdea9
1 parent 1c8c565
commit a9bdea9
Show file tree

Hide file tree

Showing 3 changed files with 86 additions and 20 deletions.
diff --git a/accountresolver/processor.go b/accountresolver/processor.go
@@ -31,11 +31,6 @@ type Stats struct {
 	totalBlockHandlingDuration         time.Duration
 	totalBlockReadingDuration          time.Duration
 	cacheHit                           int
-	totalBlockPushDuration             time.Duration
-	writeDurationAfterLastPush         time.Duration
-	lastBlockPushedAt                  time.Time
-	totalDecodingDuration              time.Duration
-	timeToFirstDecodedBlock            time.Duration
 	totalTimeWaitingForBlock           time.Duration
 	totalAccountsResolved              int
 	totalAccountsResolvedByCache       int
@@ -67,21 +62,14 @@ func (s *Stats) Log(logger *zap.Logger) {
 		zap.String("total_block_handling_duration", durafmt.Parse(s.totalBlockHandlingDuration).String()),
 		zap.String("total_block_processing_duration", durafmt.Parse(s.totalBlockProcessingDuration).String()),
 		zap.String("total_transaction_processing_duration", durafmt.Parse(s.totalTransactionProcessingDuration).String()),
-		zap.String("total_push_duration", durafmt.Parse(s.totalBlockPushDuration).String()),
 		zap.String("total_lookup_duration", durafmt.Parse(s.totalLookupDuration).String()),
 		zap.String("total_extend_duration", durafmt.Parse(s.totalExtendDuration).String()),
 		zap.String("total_duration", durafmt.Parse(time.Since(s.startProcessing)).String()),
 		zap.String("total_block_reading_duration", durafmt.Parse(s.totalBlockReadingDuration).String()),
-		zap.String("total_decoding_duration", durafmt.Parse(s.totalDecodingDuration).String()),
 		zap.String("total_time_waiting_for_block", durafmt.Parse(s.totalTimeWaitingForBlock).String()),
 		zap.String("total_time_waiting_for_block", durafmt.Parse(s.totalTimeWaitingForBlock).String()),
-		//zap.String("average_block_handling_duration", durafmt.Parse(s.totalBlockHandlingDuration/time.Duration(s.totalBlockCount)).String()),
-		//zap.String("average_block_processing_duration", durafmt.Parse(s.totalBlockProcessingDuration/time.Duration(s.totalBlockCount)).String()),
-		//zap.String("average_transaction_processing_duration", durafmt.Parse(s.totalTransactionProcessingDuration/time.Duration(s.transactionCount)).String()),
 		zap.String("average_lookup_duration", durafmt.Parse(lookupAvg).String()),
 		zap.String("average_extend_duration", durafmt.Parse(extendAvg).String()),
-		zap.String("write_duration_after_last_push", durafmt.Parse(time.Since(s.lastBlockPushedAt)).String()),
-		zap.String("time_to_first_decoded_block", durafmt.Parse(s.timeToFirstDecodedBlock).String()),
 	)
 }
 
@@ -239,6 +227,8 @@ func (p *Processor) processMergeBlocksFiles(ctx context.Context, cursor *Cursor,
 		if err != nil {
 			panic(fmt.Errorf("storing cursor at block %d: %w", cursor.slotNum, err))
 		}
+		stats.Log(p.logger)
+		timeOfLastPush = time.Now()
 	}
 	return nil
 }

diff --git a/accountresolver/processor_test.go b/accountresolver/processor_test.go
@@ -478,3 +478,82 @@ func Test_BlockResolved(t *testing.T) {
 	blockAccounts := NewAccounts(blockAccountsData)
 	require.Equal(t, expectedBlockAccounts, blockAccounts)
 }
+func Test_BlockResolved_Multiple_extend(t *testing.T) {
+	tableLookupAccount := accountFromBase58(t, "6pyNrJXyGdDDA3esoLEHJ2uoohcdf2xGT11acfmfyA7Q")
+	tableLookupToExtendIndex := byte(2)
+
+	expectedCreatedAccounts1 := fromBase58Strings(t,
+		"He3iAEV5rYjv6Xf7PxKro19eVrC3QAcdic5CF2D2obPt",
+	)
+	expectedCreatedAccounts2 := fromBase58Strings(t,
+		"5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1",
+	)
+	solBlock := &pbsol.Block{
+		PreviousBlockhash: "9RXPunwLvRcNGiLKwMBFtxmqr3d1rTxSkYYsMZPbKCct",
+		Blockhash:         "6CqnntW5shmcB92VivDAUkKdckn6m7Dmn7nTzSvX1G6o",
+		ParentSlot:        157_564_919,
+		Transactions: []*pbsol.ConfirmedTransaction{
+			{
+				Transaction: &pbsol.Transaction{
+					Signatures: [][]byte{{0}},
+					Message: &pbsol.Message{
+						AccountKeys: [][]byte{
+							accountFromBase58(t, "AddressLookupTab1e1111111111111111111111111"),
+						},
+						Instructions: []*pbsol.CompiledInstruction{
+							{
+								ProgramIdIndex: 0,
+							},
+							{
+								ProgramIdIndex: 0,
+							},
+							{
+								ProgramIdIndex: 0,
+							},
+						},
+					},
+				},
+				Meta: &pbsol.TransactionStatusMeta{
+					InnerInstructions: []*pbsol.InnerInstructions{
+						{
+							Index: 0,
+							Instructions: []*pbsol.InnerInstruction{
+								{
+									ProgramIdIndex: 0,
+									Accounts:       []byte{tableLookupToExtendIndex, 15, 0, 3},
+									Data:           append([]byte{2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0}, encodeAccounts(expectedCreatedAccounts1)...),
+								},
+							},
+						},
+						{
+							Index: 2,
+							Instructions: []*pbsol.InnerInstruction{
+								{
+									ProgramIdIndex: 0,
+									Accounts:       []byte{tableLookupToExtendIndex, 15, 0, 3},
+									Data:           append([]byte{2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0}, encodeAccounts(expectedCreatedAccounts2)...),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		BlockHeight: &pbsol.BlockHeight{
+			BlockHeight: 157_564_920,
+		},
+		Slot: 157_564_920,
+	}
+	err := os.RemoveAll("/tmp/my-badger.db")
+	require.NoError(t, err)
+	db, err := kvstore.New("badger3:///tmp/my-badger.db")
+	require.NoError(t, err)
+
+	resolver := NewKVDBAccountsResolver(db, zap.NewNop())
+	p := NewProcessor("test", NewKVDBAccountsResolver(db, zap.NewNop()), zap.NewNop())
+	err = p.ProcessBlock(context.Background(), &Stats{}, solBlock)
+	require.NoError(t, err)
+
+	accounts, _, err := resolver.Resolve(context.Background(), 157_564_921, tableLookupAccount)
+	require.Equal(t, expectedCreatedAccounts1, accounts)
+}
diff --git a/cmd/firesol/tools_bigtable_block.go b/cmd/firesol/tools_bigtable_block.go
@@ -1,12 +1,9 @@
 package main
 
 import (
-	"encoding/hex"
 	"fmt"
 	"strconv"
 
-	"github.com/golang/protobuf/proto"
-
 	"cloud.google.com/go/bigtable"
 	"github.com/spf13/cobra"
 	"github.com/streamingfast/cli/sflags"
@@ -67,11 +64,11 @@ func bigtableBlockRunE(logger *zap.Logger, tracer logging.Tracer) firecore.Comma
 			if err != nil {
 				return fmt.Errorf("printing block: %w", err)
 			}
-			hexData, err := proto.Marshal(block)
-			if err != nil {
-				return fmt.Errorf("marshaling block: %w", err)
-			}
-			fmt.Println("hex:", hex.EncodeToString(hexData))
+			//hexData, err := proto.Marshal(block)
+			//if err != nil {
+			//	return fmt.Errorf("marshaling block: %w", err)
+			//}
+			//fmt.Println("hex:", hex.EncodeToString(hexData))
 			return nil
 		}); err != nil {
 			return fmt.Errorf("failed to find block %d: %w", startBlockNum, err)