diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 144f19850a..d8b9c00bc5 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -617,9 +617,13 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth } db := utils.MakeChainDatabase(ctx, stack, true, false) - scheme, err := utils.ParseStateScheme(ctx, db) + provided, err := utils.CompareCLIWithConfig(ctx) if err != nil { - return nil, nil, common.Hash{}, fmt.Errorf("failed to parse state scheme: %v", err) + return nil, nil, common.Hash{}, err + } + scheme, err := rawdb.ParseStateScheme(provided, db) + if err != nil { + return nil, nil, common.Hash{}, err } var header *types.Header if scheme == rawdb.PathScheme { @@ -702,6 +706,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, "start", hexutil.Encode(conf.Start), "limit", conf.Max) + conf.StateScheme = scheme return conf, db, header.Root, nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a360b0f9fd..b0575ecf86 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1884,7 +1884,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } - scheme, err := compareCLIWithConfig(ctx) + scheme, err := CompareCLIWithConfig(ctx) if err != nil { Fatalf("%v", err) } @@ -2353,7 +2353,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } - provided, err := compareCLIWithConfig(ctx) + provided, err := CompareCLIWithConfig(ctx) if err != nil { Fatalf("%v", err) } @@ -2425,7 +2425,7 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read config := &trie.Config{ Preimages: preimage, } - provided, err := compareCLIWithConfig(ctx) + provided, err := CompareCLIWithConfig(ctx) if err != nil { Fatalf("%v", err) } @@ -2448,7 +2448,7 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read return trie.NewDatabase(disk, config) } -func compareCLIWithConfig(ctx *cli.Context) (string, error) { +func CompareCLIWithConfig(ctx *cli.Context) (string, error) { var ( cfgScheme string err error diff --git a/core/state/dump.go b/core/state/dump.go index 9ce6cd394b..2d6487ee1a 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -37,6 +38,7 @@ type DumpConfig struct { OnlyWithAddresses bool Start []byte Max uint64 + StateScheme string } // DumpCollector interface which the state trie calls during iteration @@ -177,7 +179,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] } if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) - tr, err := obj.getTrie() + var tr Trie + if conf.StateScheme == rawdb.PathScheme { + tr, err = trie.NewStateTrie(trie.StorageTrieID(obj.db.originalRoot, common.BytesToHash(it.Key), + obj.data.Root), obj.db.db.TrieDB()) + } else { + tr, err = obj.getTrie() + } if err != nil { log.Error("Failed to load storage trie", "err", err) continue @@ -252,3 +260,100 @@ func (s *StateDB) IteratorDump(opts *DumpConfig) IteratorDump { iterator.Next = s.DumpToCollector(iterator, opts) return *iterator } + +// DumpToCollectorInPath +func (s *StateDB) DumpToCollectorInPath(c DumpCollector, conf *DumpConfig) (nextKey []byte) { + // Sanitize the input to allow nil configs + if conf == nil { + conf = new(DumpConfig) + } + var ( + missingPreimages int + accounts uint64 + start = time.Now() + logged = time.Now() + ) + log.Info("Trie dumping started", "root", s.trie.Hash()) + c.OnRoot(s.trie.Hash()) + + trieIt, err := s.trie.NodeIterator(conf.Start) + if err != nil { + return nil + } + it := trie.NewIterator(trieIt) + for it.Next() { + var data types.StateAccount + if err := rlp.DecodeBytes(it.Value, &data); err != nil { + panic(err) + } + account := DumpAccount{ + Balance: data.Balance.String(), + Nonce: data.Nonce, + Root: data.Root[:], + CodeHash: data.CodeHash, + SecureKey: it.Key, + } + var ( + addrBytes = s.trie.GetKey(it.Key) + addr = common.BytesToAddress(addrBytes) + address *common.Address + ) + if addrBytes == nil { + // Preimage missing + missingPreimages++ + if conf.OnlyWithAddresses { + continue + } + } else { + address = &addr + } + obj := newObject(s, addr, &data) + if !conf.SkipCode { + account.Code = obj.Code() + } + if !conf.SkipStorage { + account.Storage = make(map[common.Hash]string) + tr, err := trie.NewStateTrie(trie.StorageTrieID(obj.db.originalRoot, common.BytesToHash(it.Key), + obj.data.Root), obj.db.db.TrieDB()) + // tr, err := obj.getTrie() + if err != nil { + log.Error("Failed to load storage trie", "err", err) + continue + } + trieIt, err := tr.NodeIterator(nil) + if err != nil { + log.Error("Failed to create trie iterator", "err", err) + continue + } + storageIt := trie.NewIterator(trieIt) + for storageIt.Next() { + _, content, _, err := rlp.Split(storageIt.Value) + if err != nil { + log.Error("Failed to decode the value returned by iterator", "error", err) + continue + } + account.Storage[common.BytesToHash(s.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(content) + } + } + c.OnAccount(address, account) + accounts++ + if time.Since(logged) > 8*time.Second { + log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + if conf.Max > 0 && accounts >= conf.Max { + if it.Next() { + nextKey = it.Key + } + break + } + } + if missingPreimages > 0 { + log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) + } + log.Info("Trie dumping complete", "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + + return nextKey +} diff --git a/trie/secure_trie.go b/trie/secure_trie.go index a9493c0805..ffe006c1ff 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -19,7 +19,6 @@ package trie import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -68,7 +67,6 @@ func NewStateTrie(id *ID, db *Database) (*StateTrie, error) { } trie, err := New(id, db) if err != nil { - log.Error("NewStateTrie") return nil, err } return &StateTrie{trie: *trie, preimages: db.preimages}, nil diff --git a/trie/trie.go b/trie/trie.go index 786d7c73e9..47bdb39548 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -82,7 +82,6 @@ func (t *Trie) Copy() *Trie { func New(id *ID, db *Database) (*Trie, error) { reader, err := newTrieReader(id.StateRoot, id.Owner, db) if err != nil { - log.Error("Failed to newTrieReader") return nil, err } trie := &Trie{ @@ -93,7 +92,6 @@ func New(id *ID, db *Database) (*Trie, error) { if id.Root != (common.Hash{}) && id.Root != types.EmptyRootHash { rootnode, err := trie.resolveAndTrack(id.Root[:], nil) if err != nil { - log.Error("Failed to resolveAndTrack") return nil, err } trie.root = rootnode @@ -589,7 +587,6 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) { func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) { blob, err := t.reader.node(prefix, common.BytesToHash(n)) if err != nil { - log.Error("resolveAndTrack failed to call node") return nil, err } t.tracer.onRead(prefix, blob) diff --git a/trie/trie_reader.go b/trie/trie_reader.go index c764289321..4215964559 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -54,7 +54,6 @@ func newTrieReader(stateRoot, owner common.Hash, db *Database) (*trieReader, err } reader, err := db.Reader(stateRoot) if err != nil { - log.Error("Failed to newTrieReader") return nil, &MissingNodeError{Owner: owner, NodeHash: stateRoot, err: err} } return &trieReader{owner: owner, reader: reader}, nil @@ -73,17 +72,14 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) { // Perform the logics in tests for preventing trie node access. if r.banned != nil { if _, ok := r.banned[string(path)]; ok { - log.Error("banned") return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} } } if r.reader == nil { - log.Error("trieReader reader is nil") return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} } blob, err := r.reader.Node(r.owner, path, hash) if err != nil || len(blob) == 0 { - log.Error("Failed to call trieReader Node", "length", len(blob)) return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err} } return blob, nil diff --git a/trie/triedb/pathdb/disklayer.go b/trie/triedb/pathdb/disklayer.go index 8ebceec209..f41f6b4be3 100644 --- a/trie/triedb/pathdb/disklayer.go +++ b/trie/triedb/pathdb/disklayer.go @@ -199,10 +199,10 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b ) if owner == (common.Hash{}) { nBlob, nHash = rawdb.ReadAccountTrieNode(dl.db.diskdb, path) - log.Info("owner is empty", "owner", owner, "path", path, "expect", hash, "got", nHash, "blob", nBlob) + // log.Info("owner is empty", "owner", owner, "path", path, "expect", hash, "got", nHash, "blob", nBlob) } else { nBlob, nHash = rawdb.ReadStorageTrieNode(dl.db.diskdb, owner, path) - log.Info("owner is nonempty", "owner", owner, "path", path, "expect", hash, "got", nHash, "blob", nBlob) + // log.Info("owner is nonempty", "owner", owner, "path", path, "expect", hash, "got", nHash, "blob", nBlob) } if nHash != hash { diskFalseMeter.Mark(1)