cmd: fix dump cli cannot work in path mode (#2160)

cmd/geth/chaincmd.go

@@ -30,6 +30,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/olekukonko/tablewriter"
+	"github.com/urfave/cli/v2"
+
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -44,7 +47,8 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p/enode"
-	"github.com/urfave/cli/v2"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
 )
 
 var (
@@ -191,6 +195,21 @@ It's deprecated, please use "geth db export" instead.
 		}, utils.DatabasePathFlags),
 		Description: `
 This command dumps out the state for a given block (or latest, if none provided).
+If you use "dump" command in path mode, please firstly use "dump-roothash" command to get all available state root hash.
+`,
+	}
+	dumpRootHashCommand = &cli.Command{
+		Action: dumpAllRootHashInPath,
+		Name:   "dump-roothash",
+		Usage:  "Dump all available state root hash in path mode",
+		Flags: flags.Merge([]cli.Flag{
+			utils.StateSchemeFlag,
+		}, utils.DatabasePathFlags),
+		Description: `
+The dump-roothash command dump all available state root hash in path mode.
+If you use "dump" command in path mode, please note that it only keeps at most 129 blocks which belongs to diffLayer or diskLayer.
+Therefore, you must specify the blockNumber or blockHash that locates in diffLayer or diskLayer.
+"geth" will print all available blockNumber and related block state root hash, and you can query block hash by block number.
 `,
 	}
 )
@@ -590,11 +609,20 @@ func exportPreimages(ctx *cli.Context) error {
 }
 
 func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
-	db := utils.MakeChainDatabase(ctx, stack, true, false)
-	var header *types.Header
 	if ctx.NArg() > 1 {
 		return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
 	}
+
+	db := utils.MakeChainDatabase(ctx, stack, true, false)
+	scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db)
+	if err != nil {
+		return nil, nil, common.Hash{}, err
+	}
+	if scheme == rawdb.PathScheme {
+		fmt.Println("You are using geth dump in path mode, please use `geth dump-roothash` command to get all available blocks.")
+	}
+
+	header := &types.Header{}
 	if ctx.NArg() == 1 {
 		arg := ctx.Args().First()
 		if hashish(arg) {
@@ -617,11 +645,22 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 		}
 	} else {
 		// Use latest
-		header = rawdb.ReadHeadHeader(db)
+		if scheme == rawdb.PathScheme {
+			triedb := trie.NewDatabase(db, &trie.Config{PathDB: pathdb.ReadOnly})
+			defer triedb.Close()
+			if stateRoot := triedb.Head(); stateRoot != (common.Hash{}) {
+				header.Root = stateRoot
+			} else {
+				return nil, nil, common.Hash{}, fmt.Errorf("no top state root hash in path db")
+			}
+		} else {
+			header = rawdb.ReadHeadHeader(db)
+		}
 	}
 	if header == nil {
 		return nil, nil, common.Hash{}, errors.New("no head block found")
 	}
+
 	startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name))
 	var start common.Hash
 	switch len(startArg) {
@@ -634,16 +673,18 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 	default:
 		return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg)
 	}
+
 	var conf = &state.DumpConfig{
 		SkipCode:          ctx.Bool(utils.ExcludeCodeFlag.Name),
 		SkipStorage:       ctx.Bool(utils.ExcludeStorageFlag.Name),
 		OnlyWithAddresses: !ctx.Bool(utils.IncludeIncompletesFlag.Name),
 		Start:             start.Bytes(),
 		Max:               ctx.Uint64(utils.DumpLimitFlag.Name),
 	}
+	conf.StateScheme = scheme
 	log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(),
-		"skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage,
-		"start", hexutil.Encode(conf.Start), "limit", conf.Max)
+		"skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, "start", hexutil.Encode(conf.Start),
+		"limit", conf.Max, "state scheme", conf.StateScheme)
 	return conf, db, header.Root, nil
 }
 
@@ -675,6 +716,29 @@ func dump(ctx *cli.Context) error {
 	return nil
 }
 
+func dumpAllRootHashInPath(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true, false)
+	defer db.Close()
+	triedb := trie.NewDatabase(db, &trie.Config{PathDB: pathdb.ReadOnly})
+	defer triedb.Close()
+
+	scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db)
+	if err != nil {
+		return err
+	}
+	if scheme == rawdb.HashScheme {
+		return errors.New("incorrect state scheme, you should use it in path mode")
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Block Number", "Block State Root Hash"})
+	table.AppendBulk(triedb.GetAllRooHash())
+	table.Render()
+	return nil
+}
+
 // hashish returns true for strings that look like hashes.
 func hashish(x string) bool {
 	_, err := strconv.Atoi(x)

cmd/geth/main.go

@@ -239,6 +239,7 @@ func init() {
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
+		dumpRootHashCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,

cmd/utils/flags.go

@@ -1884,7 +1884,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(StateHistoryFlag.Name) {
 		cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
 	}
-	scheme, err := compareCLIWithConfig(ctx)
+	scheme, err := CompareStateSchemeCLIWithConfig(ctx)
 	if err != nil {
 		Fatalf("%v", err)
 	}
@@ -2353,7 +2353,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 	if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
 	}
-	provided, err := compareCLIWithConfig(ctx)
+	provided, err := CompareStateSchemeCLIWithConfig(ctx)
 	if err != nil {
 		Fatalf("%v", err)
 	}
@@ -2425,7 +2425,7 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read
 	config := &trie.Config{
 		Preimages: preimage,
 	}
-	provided, err := compareCLIWithConfig(ctx)
+	provided, err := CompareStateSchemeCLIWithConfig(ctx)
 	if err != nil {
 		Fatalf("%v", err)
 	}
@@ -2448,7 +2448,8 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read
 	return trie.NewDatabase(disk, config)
 }
 
-func compareCLIWithConfig(ctx *cli.Context) (string, error) {
+// CompareStateSchemeCLIWithConfig compare state scheme in CLI with config whether are equal.
+func CompareStateSchemeCLIWithConfig(ctx *cli.Context) (string, error) {
 	var (
 		cfgScheme string
 		err       error

core/state/dump.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -37,6 +38,7 @@ type DumpConfig struct {
 	OnlyWithAddresses bool
 	Start             []byte
 	Max               uint64
+	StateScheme       string
 }
 
 // DumpCollector interface which the state trie calls during iteration
@@ -57,7 +59,6 @@ type DumpAccount struct {
 	Storage   map[common.Hash]string `json:"storage,omitempty"`
 	Address   *common.Address        `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
 	SecureKey hexutil.Bytes          `json:"key,omitempty"`     // If we don't have address, we can output the key
-
 }
 
 // Dump represents the full dump in a collected format, as one large map.
@@ -177,7 +178,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []
 		}
 		if !conf.SkipStorage {
 			account.Storage = make(map[common.Hash]string)
-			tr, err := obj.getTrie()
+			var tr Trie
+			if conf.StateScheme == rawdb.PathScheme {
+				tr, err = trie.NewStateTrie(trie.StorageTrieID(obj.db.originalRoot, common.BytesToHash(it.Key),
+					obj.data.Root), obj.db.db.TrieDB())
+			} else {
+				tr, err = obj.getTrie()
+			}
 			if err != nil {
 				log.Error("Failed to load storage trie", "err", err)
 				continue

eth/api_debug.go

@@ -59,6 +59,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
 		if stateDb == nil {
 			return state.Dump{}, errors.New("pending state is not available")
 		}
+		opts.StateScheme = stateDb.Database().TrieDB().Scheme()
 		return stateDb.RawDump(opts), nil
 	}
 	var header *types.Header
@@ -83,6 +84,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
 	if err != nil {
 		return state.Dump{}, err
 	}
+	opts.StateScheme = stateDb.Database().TrieDB().Scheme()
 	return stateDb.RawDump(opts), nil
 }
 
@@ -188,6 +190,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex
 		OnlyWithAddresses: !incompletes,
 		Start:             start,
 		Max:               uint64(maxResults),
+		StateScheme:       stateDb.Database().TrieDB().Scheme(),
 	}
 	if maxResults > AccountRangeMaxResults || maxResults <= 0 {
 		opts.Max = AccountRangeMaxResults

trie/database.go

@@ -355,7 +355,7 @@ func (db *Database) SetBufferSize(size int) error {
 }
 
 // Head return the top non-fork difflayer/disklayer root hash for rewinding.
-// It's only supported by path-based database and will return an error for
+// It's only supported by path-based database and will return empty hash for
 // others.
 func (db *Database) Head() common.Hash {
 	pdb, ok := db.backend.(*pathdb.Database)
@@ -364,3 +364,15 @@ func (db *Database) Head() common.Hash {
 	}
 	return pdb.Head()
 }
+
+// GetAllHash returns all MPT root hash in diffLayer and diskLayer.
+// It's only supported by path-based database and will return nil for
+// others.
+func (db *Database) GetAllRooHash() [][]string {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		log.Error("Not supported")
+		return nil
+	}
+	return pdb.GetAllRooHash()
+}

trie/triedb/pathdb/asyncnodebuffer.go

@@ -226,7 +226,7 @@ func (nc *nodecache) node(owner common.Hash, path []byte, hash common.Hash) (*tr
 	}
 	if n.Hash != hash {
 		dirtyFalseMeter.Mark(1)
-		log.Error("Unexpected trie node in node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
+		log.Error("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
 		return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob)
 	}
 	return n, nil

trie/triedb/pathdb/database.go

@@ -20,6 +20,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"sort"
+	"strconv"
 	"sync"
 	"time"
 
@@ -441,3 +443,24 @@ func (db *Database) Head() common.Hash {
 	defer db.lock.Unlock()
 	return db.tree.front()
 }
+
+// GetAllRooHash returns all diffLayer and diskLayer root hash
+func (db *Database) GetAllRooHash() [][]string {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	data := make([][]string, 0, len(db.tree.layers))
+	for _, v := range db.tree.layers {
+		if dl, ok := v.(*diffLayer); ok {
+			data = append(data, []string{fmt.Sprintf("%d", dl.block), dl.rootHash().String()})
+		}
+	}
+	sort.Slice(data, func(i, j int) bool {
+		block1, _ := strconv.Atoi(data[i][0])
+		block2, _ := strconv.Atoi(data[j][0])
+		return block1 > block2
+	})
+
+	data = append(data, []string{"-1", db.tree.bottom().rootHash().String()})
+	return data
+}