From 9a82eaef779f674a2e34aac30b61c8add78e2da4 Mon Sep 17 00:00:00 2001 From: Francesco4203 <100074926+Francesco4203@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:02:27 +0700 Subject: [PATCH] all: enable pbss (#600) * trie: enable pathdb: add path config and enable tests * core/rawdb: now also inspect the state freezer in pathdb; rename * cmd: working on cmd ronin * core: refactor; add pathbase config; fix tests - all: fix and enable tests for pathbase - blockchain: open triedb explicitly in blockchain functions and close right after use, since diskLayer inside pathdb is a skeleton - blockchain: when writeBlockWithState, pathbase will skip the explicit garbage collector, which is only needed for hashbase - genesis.go: nit: change check genesis state, ref https://github.com/ethereum/go-ethereum/commit/08bf8a60c3b1dec73c67a187093cd066021d0453 * tests: enable path tests * eth: enable path scheme - all: fix tests, enable path scheme tests - state_accessor: split function to retrieve statedb from block to hash scheme and path scheme * light, miner, les, ethclient: clean up tests * trie: refactor triereader, return err when state reader won't be created in hash and path * trie: fix failed test in iterator and sync test tie * trie,core: improve trie reader and add checking config nil when initing database * trie: statedb instance is committed, then it's not usable, a new instance must be created based on new root updated database, reference by commit 6d2aeb4 * cmd,les,eth: fixed unittest and adding flag Parrallel correctly * core, eth: fix tests * core: refactor and fix sync_test logic * tmp: disable pathbase for TestIsPeriodBlock, TestIsTrippEffective --------- Co-authored-by: Huy Ngo --- accounts/abi/bind/backends/simulated.go | 3 +- cmd/evm/internal/t8ntool/execution.go | 6 + cmd/evm/runner.go | 20 +- cmd/evm/staterunner.go | 30 +- cmd/ronin/chaincmd.go | 22 +- cmd/ronin/dbcmd.go | 11 +- cmd/ronin/genesis_test.go | 4 +- cmd/ronin/main.go | 5 +- cmd/ronin/snapshot.go | 63 +- cmd/utils/flags.go | 146 +++- cmd/utils/flags_legacy.go | 8 + consensus/clique/clique_test.go | 5 +- consensus/clique/snapshot_test.go | 3 +- consensus/consortium/v2/consortium_test.go | 77 +- core/bench_test.go | 3 +- core/block_validator_test.go | 15 +- core/blockchain.go | 278 +++--- core/blockchain_reader.go | 14 +- core/blockchain_repair_test.go | 53 +- core/blockchain_sethead_test.go | 36 +- core/blockchain_snapshot_test.go | 371 ++++---- core/blockchain_test.go | 799 +++++++++++++----- core/chain_makers.go | 20 +- core/chain_makers_test.go | 5 +- core/dao_test.go | 36 +- core/error.go | 1 + core/genesis.go | 12 +- core/genesis_test.go | 54 +- core/headerchain_test.go | 3 +- core/rawdb/accessors_trie.go | 31 +- core/rawdb/ancient_scheme.go | 10 +- core/rawdb/ancient_utils.go | 61 +- core/rawdb/database.go | 22 +- core/rawdb/schema.go | 26 +- core/rawdb/schema_test.go | 8 +- core/rlp_test.go | 3 +- core/state/database.go | 4 +- core/state/dump.go | 15 +- core/state/iterator.go | 11 +- core/state/iterator_test.go | 15 +- core/state/pruner/pruner.go | 22 +- core/state/snapshot/generate.go | 5 +- core/state/snapshot/generate_test.go | 130 ++- core/state/state_test.go | 6 +- core/state/statedb.go | 29 +- core/state/statedb_fuzz_test.go | 2 +- core/state/statedb_test.go | 101 ++- core/state/sync_test.go | 202 +++-- core/state_processor_test.go | 16 +- core/types/hashing_test.go | 8 +- core/vote/vote_pool_test.go | 7 +- eth/api.go | 19 +- eth/api_backend.go | 2 +- eth/api_test.go | 26 +- eth/backend.go | 13 +- eth/catalyst/api_test.go | 3 +- eth/downloader/downloader_test.go | 4 +- eth/downloader/statesync.go | 2 +- eth/downloader/testchain_test.go | 3 +- eth/ethconfig/config.go | 9 + eth/ethconfig/gen_config.go | 18 + eth/fetcher/block_fetcher_test.go | 2 +- eth/filters/filter_system_test.go | 5 +- eth/filters/filter_test.go | 5 +- eth/gasprice/gasprice_test.go | 5 +- eth/handler.go | 13 +- eth/handler_eth_test.go | 4 +- eth/handler_test.go | 5 +- eth/protocols/eth/handler_test.go | 2 +- eth/protocols/eth/handlers.go | 7 +- eth/protocols/snap/handler.go | 8 +- eth/protocols/snap/sync_test.go | 211 +++-- eth/state_accessor.go | 96 ++- eth/tracers/api_test.go | 2 +- .../internal/tracetest/calltrace2_test.go | 9 +- .../internal/tracetest/calltrace_test.go | 11 +- .../internal/tracetest/flat_calltrace_test.go | 3 +- .../internal/tracetest/prestate_test.go | 3 +- eth/tracers/tracers_test.go | 4 +- ethclient/ethclient_test.go | 5 +- ethclient/gethclient/gethclient_test.go | 3 +- les/client.go | 2 +- les/downloader/downloader_test.go | 2 +- les/downloader/statesync.go | 2 +- les/downloader/testchain_test.go | 3 +- les/fetcher/block_fetcher_test.go | 2 +- les/handler_test.go | 8 +- les/server_handler.go | 2 +- les/test_helper.go | 5 +- light/lightchain_test.go | 5 +- light/odr_test.go | 6 +- light/postprocess.go | 4 +- light/trie.go | 14 +- light/trie_test.go | 16 +- light/txpool_test.go | 5 +- miner/miner_test.go | 2 +- miner/worker_test.go | 5 +- tests/block_test.go | 17 +- tests/block_test_util.go | 22 +- tests/fuzzers/les/les-fuzzer.go | 6 +- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 2 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 6 +- tests/fuzzers/trie/trie-fuzzer.go | 4 +- tests/state_test.go | 62 +- tests/state_test_util.go | 53 +- trie/database.go | 115 ++- trie/database_test.go | 8 +- trie/errors.go | 6 + trie/iterator_test.go | 78 +- trie/proof.go | 3 + trie/proof_test.go | 20 +- trie/secure_trie.go | 8 +- trie/secure_trie_test.go | 4 +- trie/stacktrie_test.go | 12 +- trie/sync.go | 11 +- trie/sync_test.go | 97 ++- trie/tracer_test.go | 18 +- trie/trie.go | 56 +- trie/trie_reader.go | 35 +- trie/trie_test.go | 70 +- trie/triedb/hashdb/database.go | 48 +- trie/triedb/pathdb/database.go | 80 +- trie/triedb/pathdb/database_test.go | 9 +- trie/triedb/pathdb/difflayer_test.go | 2 +- trie/triedb/pathdb/disklayer.go | 20 +- trie/triedb/pathdb/history_test.go | 2 +- trie/triedb/pathdb/testutils.go | 10 +- trie/triestate/state.go | 37 +- 128 files changed, 3013 insertions(+), 1330 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 96e85a9fa5..146b209f7b 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -43,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) // This nil assignment ensures at compile time that SimulatedBackend implements bind.ContractBackend. @@ -77,7 +78,7 @@ type SimulatedBackend struct { // A simulated backend always uses chainID 1337. func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} - genesis.MustCommit(database) + genesis.MustCommit(database, trie.NewDatabase(database, trie.HashDefaults)) blockchain, _ := core.NewBlockChain(database, nil, &genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) backend := &SimulatedBackend{ diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index ea59c48427..b3773b9fa2 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -258,6 +258,12 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, Difficulty: (*math.HexOrDecimal256)(vmContext.Difficulty), GasUsed: (math.HexOrDecimal64)(gasUsed), } + // Re-create statedb instance with new root upon the updated database + // for accessing latest states. + statedb, err = state.New(root, statedb.Database(), nil) + if err != nil { + return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err)) + } return statedb, execRs, nil } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 67aaa03ea9..4a060f3e11 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" "github.com/ethereum/go-ethereum/cmd/utils" @@ -37,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/runtime" "github.com/ethereum/go-ethereum/log" @@ -139,11 +142,22 @@ func runCmd(ctx *cli.Context) error { gen := readGenesis(ctx.String(GenesisFlag.Name)) genesisConfig = gen db := rawdb.NewMemoryDatabase() - genesis := gen.MustCommit(db) - statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil) + triedb := trie.NewDatabase(db, &trie.Config{ + HashDB: hashdb.Defaults, + }) + defer triedb.Close() + genesis := gen.MustCommit(db, triedb) + sdb := state.NewDatabaseWithNodeDB(db, triedb) + statedb, _ = state.New(genesis.Root(), sdb, nil) chainConfig = gen.Config } else { - statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + db := rawdb.NewMemoryDatabase() + triedb := trie.NewDatabase(db, &trie.Config{ + HashDB: hashdb.Defaults, + }) + defer triedb.Close() + sdb := state.NewDatabaseWithNodeDB(db, triedb) + statedb, _ = state.New(types.EmptyRootHash, sdb, nil) genesisConfig = new(core.Genesis) } if ctx.String(SenderFlag.Name) != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 0e05237343..506f789a8a 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -20,11 +20,14 @@ import ( "encoding/json" "errors" "fmt" - "github.com/ethereum/go-ethereum/eth/tracers/logger" "io/ioutil" "os" + "github.com/ethereum/go-ethereum/eth/tracers/logger" + + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests" @@ -96,21 +99,22 @@ func stateTestCmd(ctx *cli.Context) error { results := make([]StatetestResult, 0, len(tests)) for key, test := range tests { for _, st := range test.Subtests() { + dump := ctx.Bool(DumpFlag.Name) // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - _, s, err := test.Run(st, cfg, false) - // print state root for evmlab tracing - if ctx.Bool(MachineFlag.Name) && s != nil { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false)) - } - if err != nil { - // Test failed, mark as so and dump any state to aid debugging - result.Pass, result.Error = false, err.Error() - if ctx.Bool(DumpFlag.Name) && s != nil { - dump := s.RawDump(nil) - result.State = &dump + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + if err != nil { + // Test failed, mark as so and dump any state to aid debugging + result.Pass, result.Error = false, err.Error() + if dump { + dump := state.RawDump(nil) + result.State = &dump + } } - } + if ctx.Bool(MachineFlag.Name) && state != nil { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) + } + }) results = append(results, *result) diff --git a/cmd/ronin/chaincmd.go b/cmd/ronin/chaincmd.go index 937c5b3d68..07475ef219 100644 --- a/cmd/ronin/chaincmd.go +++ b/cmd/ronin/chaincmd.go @@ -38,7 +38,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/trie" "github.com/urfave/cli/v2" ) @@ -52,6 +51,8 @@ var ( utils.DataDirFlag, utils.DBEngineFlag, utils.ForceOverrideChainConfigFlag, + utils.CachePreimagesFlag, + utils.StateSchemeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -106,6 +107,9 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, + utils.TransactionHistoryFlag, + utils.StateSchemeFlag, + utils.StateHistoryFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -125,6 +129,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`, utils.DBEngineFlag, utils.CacheFlag, utils.SyncModeFlag, + utils.StateSchemeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -226,14 +231,13 @@ func initGenesis(ctx *cli.Context) error { utils.Fatalf("Failed to open database: %v", err) } // Create triedb firstly - triedb := trie.NewDatabaseWithConfig(chaindb, &trie.Config{ - Preimages: ctx.Bool(utils.CachePreimagesFlag.Name), - }) + + triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false) + defer chaindb.Close() _, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis, overrideChainConfig) if err != nil { utils.Fatalf("Failed to write genesis block: %v", err) } - chaindb.Close() log.Info("Successfully wrote genesis state", "database", name, "hash", hash) } return nil @@ -471,10 +475,10 @@ func dump(ctx *cli.Context) error { if err != nil { return err } - config := &trie.Config{ - Preimages: true, // always enable preimage lookup - } - state, err := state.New(root, state.NewDatabaseWithConfig(db, config), nil) + triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup + defer triedb.Close() + state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil) + if err != nil { return err } diff --git a/cmd/ronin/dbcmd.go b/cmd/ronin/dbcmd.go index dc8ce61742..53a83b5031 100644 --- a/cmd/ronin/dbcmd.go +++ b/cmd/ronin/dbcmd.go @@ -190,6 +190,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.SepoliaFlag, utils.RinkebyFlag, utils.GoerliFlag, + utils.StateSchemeFlag, }, Description: "This command looks up the specified database key from the database.", } @@ -476,6 +477,8 @@ func dbDumpTrie(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() + triedb := utils.MakeTrieDatabase(ctx, db, false, true) + defer triedb.Close() var ( state []byte storage []byte @@ -509,12 +512,16 @@ func dbDumpTrie(ctx *cli.Context) error { } } id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage)) - theTrie, err := trie.New(id, trie.NewDatabase(db)) + theTrie, err := trie.New(id, triedb) + if err != nil { + return err + } + trieIt, err := theTrie.NodeIterator(start) if err != nil { return err } var count int64 - it := trie.NewIterator(theTrie.NodeIterator(start)) + it := trie.NewIterator(trieIt) for it.Next() { if max > 0 && count == max { fmt.Printf("Exiting after %d values\n", count) diff --git a/cmd/ronin/genesis_test.go b/cmd/ronin/genesis_test.go index 2d4cb6a3ac..41b50b45d7 100644 --- a/cmd/ronin/genesis_test.go +++ b/cmd/ronin/genesis_test.go @@ -173,12 +173,12 @@ func TestCustomBackend(t *testing.T) { { // Can't start pebble on top of leveldb initArgs: []string{"--db.engine", "leveldb"}, execArgs: []string{"--db.engine", "pebble"}, - execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, + execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, }, { // Can't start leveldb on top of pebble initArgs: []string{"--db.engine", "pebble"}, execArgs: []string{"--db.engine", "leveldb"}, - execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, + execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, }, { // Reject invalid backend choice initArgs: []string{"--db.engine", "mssql"}, diff --git a/cmd/ronin/main.go b/cmd/ronin/main.go index 3e94098ab3..057080de40 100644 --- a/cmd/ronin/main.go +++ b/cmd/ronin/main.go @@ -101,7 +101,10 @@ var ( utils.ExitWhenSyncedFlag, utils.GCModeFlag, utils.SnapshotFlag, - utils.TxLookupLimitFlag, + utils.TxLookupLimitFlag, // deprecated + utils.TransactionHistoryFlag, + utils.StateSchemeFlag, + utils.StateHistoryFlag, utils.TriesInMemoryFlag, utils.LightServeFlag, utils.LightIngressFlag, diff --git a/cmd/ronin/snapshot.go b/cmd/ronin/snapshot.go index 0a756ed6da..23fd7d19dc 100644 --- a/cmd/ronin/snapshot.go +++ b/cmd/ronin/snapshot.go @@ -77,10 +77,7 @@ two version states are available: genesis and the specific one. The default pruning target is the HEAD-127 state. -WARNING: It's necessary to delete the trie clean cache after the pruning. -If you specify another directory for the trie clean cache via "--cache.trie.journal" -during the use of Geth, please also specify it here for correct deletion. Otherwise -the trie clean cache with default directory will be deleted. +WARNING: it's only supported in hash mode(--state.scheme=hash)". `, }, { @@ -97,6 +94,7 @@ the trie clean cache with default directory will be deleted. utils.SepoliaFlag, utils.RinkebyFlag, utils.GoerliFlag, + utils.StateSchemeFlag, }, Description: ` geth snapshot verify-state @@ -119,6 +117,7 @@ In other words, this command does the snapshot to trie conversion. utils.SepoliaFlag, utils.RinkebyFlag, utils.GoerliFlag, + utils.StateSchemeFlag, }, Description: ` geth snapshot traverse-state @@ -143,6 +142,7 @@ It's also usable without snapshot enabled. utils.SepoliaFlag, utils.RinkebyFlag, utils.GoerliFlag, + utils.StateSchemeFlag, }, Description: ` geth snapshot traverse-rawstate @@ -172,6 +172,7 @@ It's also usable without snapshot enabled. utils.ExcludeStorageFlag, utils.StartKeyFlag, utils.DumpLimitFlag, + utils.StateSchemeFlag, }, Description: ` This command is semantically equivalent to 'geth dump', but uses the snapshots @@ -192,6 +193,9 @@ func pruneState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, false) + if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme { + log.Crit("Offline pruning is not required for path scheme") + } pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), ctx.Uint64(utils.BloomFilterSizeFlag.Name)) if err != nil { @@ -227,7 +231,10 @@ func verifyState(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + defer triedb.Close() + + snaptree, err := snapshot.New(chaindb, triedb, 256, headBlock.Root(), false, false, false) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err @@ -260,6 +267,10 @@ func traverseState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, true) + defer chaindb.Close() + + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + defer triedb.Close() headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") @@ -284,7 +295,6 @@ func traverseState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } - triedb := trie.NewDatabase(chaindb) t, err := trie.NewSecure(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -297,7 +307,12 @@ func traverseState(ctx *cli.Context) error { lastReport time.Time start = time.Now() ) - accIter := trie.NewIterator(t.NodeIterator(nil)) + acctIt, err := t.NodeIterator(nil) + if err != nil { + log.Error("Failed to open iterator", "root", root, "err", err) + return err + } + accIter := trie.NewIterator(acctIt) for accIter.Next() { accounts += 1 var acc types.StateAccount @@ -311,7 +326,12 @@ func traverseState(ctx *cli.Context) error { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err } - storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) + storageIt, err := storageTrie.NodeIterator(nil) + if err != nil { + log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) + return err + } + storageIter := trie.NewIterator(storageIt) for storageIter.Next() { slots += 1 } @@ -350,6 +370,10 @@ func traverseRawState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, true) + defer chaindb.Close() + + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + defer triedb.Close() headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") @@ -374,7 +398,6 @@ func traverseRawState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } - triedb := trie.NewDatabase(chaindb) t, err := trie.NewSecure(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -388,7 +411,17 @@ func traverseRawState(ctx *cli.Context) error { lastReport time.Time start = time.Now() ) - accIter := t.NodeIterator(nil) + accIter, err := t.NodeIterator(nil) + if err != nil { + log.Error("Failed to open iterator", "root", root, "err", err) + return err + } + // using reader. + //reader, err := triedb.Reader(root) + if err != nil { + log.Error("State is non-existent", "root", root) + return nil + } for accIter.Next(true) { nodes += 1 node := accIter.Hash() @@ -416,7 +449,11 @@ func traverseRawState(ctx *cli.Context) error { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") } - storageIter := storageTrie.NodeIterator(nil) + storageIter, err := storageTrie.NodeIterator(nil) + if err != nil { + log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) + return err + } for storageIter.Next(true) { nodes += 1 node := storageIter.Hash() @@ -477,7 +514,9 @@ func dumpState(ctx *cli.Context) error { if err != nil { return err } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false) + triedb := utils.MakeTrieDatabase(ctx, db, false, true) + defer triedb.Close() + snaptree, err := snapshot.New(db, triedb, 256, root, false, false, false) if err != nil { return err } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 806384adce..2e070815b9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -31,6 +31,9 @@ import ( "time" "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" @@ -225,18 +228,30 @@ var ( Value: "full", Category: flags.StateCategory, } + StateSchemeFlag = &cli.StringFlag{ + Name: "state.scheme", + Usage: `State scheme to use for trie storage ("hash" or "path")`, + Value: rawdb.HashScheme, // Default to hash scheme + Category: flags.StateCategory, + } + StateHistoryFlag = &cli.Uint64Flag{ + Name: "history.state", + Usage: "Number of recent blocks to retain state history for (default = 90,000 blocks, 0 = entire chain)", + Value: ethconfig.Defaults.StateHistory, + Category: flags.StateCategory, + } + TransactionHistoryFlag = &cli.Uint64Flag{ + Name: "history.transactions", + Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", + Value: ethconfig.Defaults.TransactionHistory, + Category: flags.StateCategory, + } SnapshotFlag = &cli.BoolFlag{ Name: "snapshot", Usage: `Enables snapshot-database mode (default = enable)`, Value: true, Category: flags.EthCategory, } - TxLookupLimitFlag = &cli.Uint64Flag{ - Name: "txlookuplimit", - Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", - Value: ethconfig.Defaults.TxLookupLimit, - Category: flags.EthCategory, - } TriesInMemoryFlag = &cli.IntFlag{ Name: "triesinmemory", Usage: "The number of tries is kept in memory before pruning (default = 128)", @@ -1835,17 +1850,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer - if ctx.String(GCModeFlag.Name) == "archive" && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { - ctx.Set(TxLookupLimitFlag.Name, "0") - log.Warn("Disable transaction unindexing for archive node") - } - if ctx.IsSet(LightServeFlag.Name) && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { - log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") - } + var ks *keystore.KeyStore if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 { ks = keystores[0].(*keystore.KeyStore) } + // Set configurations from CLI flags. setEtherbase(ctx, ks, cfg) setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light") setTxPool(ctx, &cfg.TxPool) @@ -1904,9 +1914,42 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") } - if ctx.IsSet(TxLookupLimitFlag.Name) { - cfg.TxLookupLimit = ctx.Uint64(TxLookupLimitFlag.Name) + if ctx.IsSet(StateHistoryFlag.Name) { + cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) + } + + /* State Scheme Config logic */ + // Parse the state scheme from chaindb firstly. + chaindb := tryMakeReadOnlyDatabase(ctx, stack) + scheme, err := ParseStateScheme(ctx, chaindb) + chaindb.Close() + if err != nil { + Fatalf("%v", err) } + cfg.StateScheme = scheme + + // Parse transaction history flag, if user is still using legacy config + // file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'. + if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit { + log.Warn("The config option 'TxLookupLimit' is deprecated and will be removed, please use 'TransactionHistory'") + cfg.TransactionHistory = cfg.TxLookupLimit + } + + if ctx.IsSet(TransactionHistoryFlag.Name) { + cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name) + } else if ctx.IsSet(TxLookupLimitFlag.Name) { + log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions") + cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name) + } + + if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 { + cfg.TransactionHistory = 0 + log.Warn("Disabled transaction unindexing for archive node") + } + if ctx.IsSet(LightServeFlag.Name) && cfg.TransactionHistory != 0 { + log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") + } + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 } @@ -2356,3 +2399,76 @@ func MakeConsolePreloads(ctx *cli.Context) []string { } return preloads } + +// tryMakeReadOnlyDatabase try to open the chain database in read-only mode, +// or fallback to write mode if the database is not initialized. +func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { + // If datadir doesn't exist we need to open db in write-mode + // so database engine can create files. + readonly := true + if !common.FileExist(stack.ResolvePath("chaindata")) { + readonly = false + } + return MakeChainDatabase(ctx, stack, readonly) +} + +// ParseStateScheme resolves scheme identifier from CLI flag. If the provided +// state scheme is not compatible with the one of persistent scheme, an error +// will be returned +// +// - none: use the scheme consistent with persistent state, or fallback +// to hash-based scheme if state is empty. +// - hash: use hash-based scheme or error out if not compatible with +// persistent state scheme. +// - path: use path-based scheme or error out if not compatible with +// persistent state scheme. +func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) { + // If state scheme is not specified, use the scheme consistent + // with persistent state, or fallback to hash mode if database + // is empty. + stored := rawdb.ReadStateScheme(disk) + + // If the flag is not set. + if !ctx.IsSet(StateSchemeFlag.Name) { + if stored == "" { + // If the database is empty, use hash-based scheme. + log.Info("State schema set to default when database is empty", "scheme", "hash") + return rawdb.HashScheme, nil + } + log.Info("State scheme set to already existing", "scheme", stored) + return stored, nil + } + // If state scheme is specified, ensure it's compatible with + // persistent state. + scheme := ctx.String(StateSchemeFlag.Name) + if stored != "" || scheme == stored { + log.Info("State scheme set by user", "scheme", scheme) + return scheme, nil + } + return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme) +} + +// MakeTrieDatabase constructs a trie database based on the configured scheme. +func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database { + config := &trie.Config{ + Preimages: preimage, + } + scheme, err := ParseStateScheme(ctx, disk) + if err != nil { + Fatalf("%v", err) + } + // Readonly only support in PathScheme + if scheme == rawdb.HashScheme { + // Read-only mode is not implemented in hash mode, + // ignore the parameter silently. TODO(rjl493456442) + // please config it if read mode is implemented. + config.HashDB = hashdb.Defaults + return trie.NewDatabase(disk, config) + } + if readOnly { + config.PathDB = pathdb.ReadOnly + } else { + config.PathDB = pathdb.Defaults + } + return trie.NewDatabase(disk, config) +} diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 63b6d64879..8aa77e1ba0 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -38,6 +38,7 @@ var DeprecatedFlags = []cli.Flag{ NoUSBFlag, CacheTrieJournalFlag, CacheTrieRejournalFlag, + TxLookupLimitFlag, } var ( @@ -63,6 +64,13 @@ var ( Usage: "Time interval to regenerate the trie cache journal", Category: flags.PerfCategory, } + // Deprecated Nov 2024 + TxLookupLimitFlag = &cli.Uint64Flag{ + Name: "txlookuplimit", + Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain) (deprecated, use history.transactions instead)", + Value: ethconfig.Defaults.TransactionHistory, + Category: flags.DeprecatedCategory, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index b82ea54b71..744d84595f 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // This test case is a repro of an annoying bug that took us forever to catch. @@ -53,7 +54,7 @@ func TestReimportMirroredState(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } copy(genspec.ExtraData[extraVanity:], addr[:]) - genesis := genspec.MustCommit(db) + genesis := genspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) // Generate a batch of blocks, each properly signed chain, _ := core.NewBlockChain(db, nil, genspec, nil, engine, vm.Config{}, nil, nil) @@ -88,7 +89,7 @@ func TestReimportMirroredState(t *testing.T) { } // Insert the first two blocks and make sure the chain is valid db = rawdb.NewMemoryDatabase() - genspec.MustCommit(db) + genspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) chain, _ = core.NewBlockChain(db, nil, genspec, nil, engine, vm.Config{}, nil, nil) defer chain.Stop() diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index e10baea556..09d8aaaf5d 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // testerAccountPool is a pool to maintain currently active tester accounts, @@ -403,7 +404,7 @@ func TestClique(t *testing.T) { } // Create a pristine blockchain with the genesis injected db := rawdb.NewMemoryDatabase() - genesisBlock := genesis.MustCommit(db) + genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) // Assemble a chain of headers from the cast votes config := *params.TestChainConfig diff --git a/consensus/consortium/v2/consortium_test.go b/consensus/consortium/v2/consortium_test.go index eca4a4e132..175bf54ef3 100644 --- a/consensus/consortium/v2/consortium_test.go +++ b/consensus/consortium/v2/consortium_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/hashicorp/golang-lru/arc/v2" ) @@ -1417,6 +1418,10 @@ func TestAssembleFinalityVoteTripp(t *testing.T) { } func TestVerifyVote(t *testing.T) { + testVeiryVote(t, rawdb.PathScheme) + testVeiryVote(t, rawdb.HashScheme) +} +func testVeiryVote(t *testing.T, scheme string) { const numValidator = 3 var err error @@ -1441,8 +1446,8 @@ func TestVerifyVote(t *testing.T) { Config: params.TestChainConfig, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis := gspec.MustCommit(db) - chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) + chain, _ := core.NewBlockChain(db, core.DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) bs, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1, nil, true) if _, err := chain.InsertChain(bs[:], nil); err != nil { @@ -1525,6 +1530,11 @@ func TestVerifyVote(t *testing.T) { } func TestKnownBlockReorg(t *testing.T) { + testKnowBlockReorg(t, rawdb.PathScheme) + testKnowBlockReorg(t, rawdb.HashScheme) +} + +func testKnowBlockReorg(t *testing.T, scheme string) { db := rawdb.NewMemoryDatabase() blsKeys := make([]blsCommon.SecretKey, 3) @@ -1572,7 +1582,7 @@ func TestKnownBlockReorg(t *testing.T) { gspec := &core.Genesis{ Config: &chainConfig, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) mock := &mockContract{ validators: make(map[common.Address]blsCommon.PublicKey), @@ -1590,7 +1600,7 @@ func TestKnownBlockReorg(t *testing.T) { db: db, } - chain, _ := core.NewBlockChain(db, nil, gspec, nil, &v2, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, core.DefaultCacheConfigWithScheme(scheme), gspec, nil, &v2, vm.Config{}, nil, nil) extraData := [consortiumCommon.ExtraVanity + consortiumCommon.ExtraSeal]byte{} blocks, _ := core.GenerateConsortiumChain( @@ -1789,6 +1799,11 @@ func TestKnownBlockReorg(t *testing.T) { } func TestUpgradeRoninTrustedOrg(t *testing.T) { + testUpgradeRoninTrustedOrg(t, rawdb.PathScheme) + testUpgradeRoninTrustedOrg(t, rawdb.HashScheme) +} + +func testUpgradeRoninTrustedOrg(t *testing.T, scheme string) { db := rawdb.NewMemoryDatabase() blsSecretKey, err := blst.RandKey() if err != nil { @@ -1824,7 +1839,7 @@ func TestUpgradeRoninTrustedOrg(t *testing.T) { common.Address{0x10}: core.GenesisAccount{Balance: common.Big1}, }, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) mock := &mockContract{ validators: map[common.Address]blsCommon.PublicKey{ @@ -1844,7 +1859,7 @@ func TestUpgradeRoninTrustedOrg(t *testing.T) { }, } - chain, _ := core.NewBlockChain(db, nil, gspec, nil, &v2, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, core.DefaultCacheConfigWithScheme(scheme), gspec, nil, &v2, vm.Config{}, nil, nil) extraData := [consortiumCommon.ExtraVanity + consortiumCommon.ExtraSeal]byte{} parent := genesis @@ -1914,6 +1929,11 @@ func TestUpgradeRoninTrustedOrg(t *testing.T) { } func TestUpgradeAxieProxyCode(t *testing.T) { + testUpgradeAxieProxyCode(t, rawdb.PathScheme) + testUpgradeAxieProxyCode(t, rawdb.HashScheme) +} + +func testUpgradeAxieProxyCode(t *testing.T, scheme string) { secretKey, err := crypto.GenerateKey() if err != nil { t.Fatal(err) @@ -1966,7 +1986,7 @@ func TestUpgradeAxieProxyCode(t *testing.T) { gspec := &core.Genesis{ Config: chainConfig, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) mock := &mockTrippContract{ checkpointValidators: []validatorWithBlsWeight{ validatorWithBlsWeight{ @@ -1993,7 +2013,7 @@ func TestUpgradeAxieProxyCode(t *testing.T) { testTrippEffective: true, } - chain, _ := core.NewBlockChain(db, nil, gspec, nil, v2, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, core.DefaultCacheConfigWithScheme(scheme), gspec, nil, v2, vm.Config{}, nil, nil) extraData := &finality.HeaderExtraData{} parent := genesis @@ -2054,6 +2074,11 @@ func TestUpgradeAxieProxyCode(t *testing.T) { } func TestSystemTransactionOrder(t *testing.T) { + testSystemTransactionOrder(t, rawdb.PathScheme) + testSystemTransactionOrder(t, rawdb.HashScheme) +} + +func testSystemTransactionOrder(t *testing.T, scheme string) { db := rawdb.NewMemoryDatabase() blsSecretKey, err := blst.RandKey() if err != nil { @@ -2093,7 +2118,7 @@ func TestSystemTransactionOrder(t *testing.T) { common.Address{0x10}: core.GenesisAccount{Balance: common.Big1}, }, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) mock := &mockContract{ validators: map[common.Address]blsCommon.PublicKey{ @@ -2113,7 +2138,7 @@ func TestSystemTransactionOrder(t *testing.T) { }, } - chain, _ := core.NewBlockChain(db, nil, gspec, nil, &v2, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, core.DefaultCacheConfigWithScheme(scheme), gspec, nil, &v2, vm.Config{}, nil, nil) extraData := [consortiumCommon.ExtraVanity + consortiumCommon.ExtraSeal]byte{} signer := types.NewEIP155Signer(big.NewInt(2021)) @@ -2194,6 +2219,11 @@ func TestSystemTransactionOrder(t *testing.T) { } func TestIsPeriodBlock(t *testing.T) { + //testIsPeriodBlock(t, rawdb.PathScheme) + testIsPeriodBlock(t, rawdb.HashScheme) +} + +func testIsPeriodBlock(t *testing.T, scheme string) { const NUM_OF_VALIDATORS = 21 dateInSeconds := uint64(86400) now := uint64(time.Now().Unix()) @@ -2215,8 +2245,8 @@ func TestIsPeriodBlock(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), Timestamp: midnight, // genesis at day 1 } - genesis := gspec.MustCommit(db) - chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) + chain, _ := core.NewBlockChain(db, core.DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) // create chain of up to 399 blocks, all of them are not period block bs, _ := core.GenerateChain(&chainConfig, genesis, ethash.NewFaker(), db, 399, nil, true) // create chain of up to 399 blocks if _, err := chain.InsertChain(bs[:], nil); err != nil { @@ -2292,7 +2322,19 @@ func TestIsPeriodBlock(t *testing.T) { } } +/* +Got issues related to parent layer missing in the test +panic: triedb parent [0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421] layer missing [recovered] +panic: triedb parent [0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421] layer missing +Will disable this test firstly for further investigation. +*/ func TestIsTrippEffective(t *testing.T) { + testIsTrippEffective(t, rawdb.HashScheme) + // testIsTrippEffective(t, rawdb.PathScheme) + +} + +func testIsTrippEffective(t *testing.T, scheme string) { now := uint64(time.Now().Unix()) midnight := uint64(now / dayInSeconds * dayInSeconds) db := rawdb.NewMemoryDatabase() @@ -2312,8 +2354,8 @@ func TestIsTrippEffective(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), Timestamp: midnight, // genesis at day 1 } - genesis := gspec.MustCommit(db) - chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) + chain, _ := core.NewBlockChain(db, core.DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) // create chain of up to 399 blocks, all of them are not Tripp effective bs, _ := core.GenerateChain(&chainConfig, genesis, ethash.NewFaker(), db, 399, nil, true) if _, err := chain.InsertChain(bs[:], nil); err != nil { @@ -2744,3 +2786,10 @@ func TestVerifyBlobHeader(t *testing.T) { t.Fatal("Expected sidecars to be kept") } } + +func newDbConfig(scheme string) *trie.Config { + if scheme == rawdb.HashScheme { + return trie.HashDefaults + } + return &trie.Config{PathDB: pathdb.Defaults} +} diff --git a/core/bench_test.go b/core/bench_test.go index 0fd7082e0c..cd0b3fbaba 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) func BenchmarkInsertChain_empty_memdb(b *testing.B) { @@ -196,7 +197,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { Config: params.TestChainConfig, Alloc: GenesisAlloc{benchRootAddr: {Balance: benchRootFunds}}, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, b.N, gen, true) // Time the insertion of the new chain. diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 2ba2cff31e..27138f5df2 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -26,15 +26,20 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) -// Tests that simple header verification works, for both good and bad blocks. func TestHeaderVerification(t *testing.T) { + testHeaderVerification(t, rawdb.HashScheme) + testHeaderVerification(t, rawdb.PathScheme) +} + +func testHeaderVerification(t *testing.T, scheme string) { // Create a simple chain to verify var ( testdb = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig} - genesis = gspec.MustCommit(testdb) + genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, newDbConfig(scheme))) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil, true) ) headers := make([]*types.Header, len(blocks)) @@ -42,7 +47,7 @@ func TestHeaderVerification(t *testing.T) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(testdb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(testdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer chain.Stop() for i := 0; i < len(blocks); i++ { @@ -86,7 +91,7 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) { var ( testdb = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig} - genesis = gspec.MustCommit(testdb) + genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, trie.HashDefaults)) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil, true) ) headers := make([]*types.Header, len(blocks)) @@ -158,7 +163,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) { var ( testdb = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig} - genesis = gspec.MustCommit(testdb) + genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, trie.HashDefaults)) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 1024, nil, true) ) headers := make([]*types.Header, len(blocks)) diff --git a/core/blockchain.go b/core/blockchain.go index 2bb494bc63..44b00a17bc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -46,6 +46,8 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" lru "github.com/hashicorp/golang-lru/v2" ) @@ -139,10 +141,30 @@ type CacheConfig struct { SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk TriesInMemory int // The number of tries is kept in memory before pruning + StateHistory uint64 // Number of blocks from head whose state histories are reserved. + StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } +// triedbConfig derives the configures for trie database. +func (c *CacheConfig) triedbConfig() *trie.Config { + config := &trie.Config{Preimages: c.Preimages} + if c.StateScheme == rawdb.HashScheme { + config.HashDB = &hashdb.Config{ + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + } + } + if c.StateScheme == rawdb.PathScheme { + config.PathDB = &pathdb.Config{ + StateHistory: c.StateHistory, + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024, + } + } + return config +} + // defaultCacheConfig are the default caching values if none are specified by the // user (also used during testing). var defaultCacheConfig = &CacheConfig{ @@ -151,6 +173,15 @@ var defaultCacheConfig = &CacheConfig{ TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 256, SnapshotWait: true, + StateScheme: rawdb.HashScheme, +} + +// DefaultCacheConfigWithScheme returns a deep copied default cache config with +// a provided trie node scheme. +func DefaultCacheConfigWithScheme(scheme string) *CacheConfig { + config := *defaultCacheConfig + config.StateScheme = scheme + return &config } // BlockChain represents the canonical chain given a database with a genesis @@ -260,12 +291,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis blobSidecarsCache, _ := lru.New[common.Hash, types.BlobSidecars](blobSidecarsCacheLimit) // Open trie database with provided config - triedb := trie.NewDatabaseWithConfig( - db, - &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Preimages: cacheConfig.Preimages, - }) + triedb := trie.NewDatabase(db, cacheConfig.triedbConfig()) // Setup the genesis block, commit the provided genesis specification // to database if the genesis block is not present yet, or load the // stored one from database. @@ -276,15 +302,12 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis log.Info("Initialised chain configuration", "config", chainConfig) bc := &BlockChain{ - chainConfig: chainConfig, - cacheConfig: cacheConfig, - db: db, - triedb: triedb, - triegc: prque.New(nil), - stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Preimages: cacheConfig.Preimages, - }), + chainConfig: chainConfig, + cacheConfig: cacheConfig, + db: db, + triedb: triedb, + triegc: prque.New(nil), + stateCache: state.NewDatabaseWithNodeDB(db, triedb), quit: make(chan struct{}), chainmu: syncx.NewClosableMutex(), shouldPreserve: shouldPreserve, @@ -649,6 +672,26 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo pivot := rawdb.ReadLastPivotNumber(bc.db) frozen, _ := bc.db.Ancients() + // resetState resets the persistent state to genesis if it's not available. + resetState := func() { + // Short circuit if the genesis state is already present. + if bc.HasState(bc.genesisBlock.Root()) { + return + } + // Reset the state database to empty for committing genesis state. + // Note, it should only happen in path-based scheme and Reset function + // is also only call-able in this mode. + if bc.triedb.Scheme() == rawdb.PathScheme { + if err := bc.triedb.Reset(types.EmptyRootHash); err != nil { + log.Crit("Failed to clean state", "err", err) // Shouldn't happen + } + } + // Write genesis state into database. + if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil { + log.Crit("Failed to commit genesis state", "err", err) + } + } + updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (uint64, bool) { // Rewind the blockchain, ensuring we don't end up with a stateless head // block. Note, depth equality is permitted to allow using SetHead as a @@ -658,6 +701,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo if newHeadBlock == nil { log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash()) newHeadBlock = bc.genesisBlock + resetState() } else { // Block exists, keep rewinding until we find one with state, // keeping rewinding until we exceed the optional threshold @@ -669,7 +713,9 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root { beyondRoot, rootNumber = true, newHeadBlock.NumberU64() } - if _, err := state.New(newHeadBlock.Root(), bc.stateCache, bc.snaps); err != nil { + + // In pbss, if the state is missing, we can possibly recover it from history db + if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) { log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) if pivot == nil || newHeadBlock.NumberU64() > *pivot { parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) @@ -686,16 +732,12 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo } if beyondRoot || newHeadBlock.NumberU64() == 0 { if newHeadBlock.NumberU64() == 0 { - // Recommit the genesis state into disk in case the rewinding destination - // is genesis block and the relevant state is gone. In the future this - // rewinding destination can be the earliest block stored in the chain - // if the historical chain pruning is enabled. In that case the logic - // needs to be improved here. - if !bc.HasState(bc.genesisBlock.Root()) { - if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil { - log.Crit("Failed to commit genesis state", "err", err) - } - log.Debug("Recommitted genesis state to disk") + resetState() + } else if !bc.HasState(newHeadBlock.Root()) { + // Rewind to a block with recoverable state. If the state is + // missing, run the state recovery here. + if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil { + log.Crit("Failed to rollback state", "err", err) // Shouldn't happen } } log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) @@ -793,7 +835,13 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } + // Reset the trie database with the fresh fast synced state. root := block.Root() + if bc.triedb.Scheme() == rawdb.PathScheme { + if err := bc.triedb.Reset(root); err != nil { + return err + } + } if !bc.HasState(root) { return fmt.Errorf("non existent state [%x..]", root[:4]) } @@ -973,38 +1021,48 @@ func (bc *BlockChain) Stop() { log.Error("Failed to journal state snapshot", "err", err) } } - - // Ensure the state of a recent block is also stored to disk before exiting. - // We're writing three different states to catch different restart scenarios: - // - HEAD: So we don't need to reprocess any blocks in the general case - // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle - // - HEAD-127: So we have a hard limit on the number of blocks reexecuted - if !bc.cacheConfig.TrieDirtyDisabled { - triedb := bc.triedb - - for _, offset := range []uint64{0, 1, uint64(bc.cacheConfig.TriesInMemory) - 1} { - if number := bc.CurrentBlock().NumberU64(); number > offset { - recent := bc.GetBlockByNumber(number - offset) - - log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) - if err := triedb.Commit(recent.Root(), true); err != nil { + if bc.triedb.Scheme() == rawdb.PathScheme { + // Ensure that the in-memory trie nodes are journaled to disk properly. + if err := bc.triedb.Journal(bc.CurrentBlock().Root()); err != nil { + log.Info("Failed to journal in-memory trie nodes", "err", err) + } + } else { + // Ensure the state of a recent block is also stored to disk before exiting. + // We're writing three different states to catch different restart scenarios: + // - HEAD: So we don't need to reprocess any blocks in the general case + // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle + // - HEAD-127: So we have a hard limit on the number of blocks reexecuted + if !bc.cacheConfig.TrieDirtyDisabled { + triedb := bc.triedb + + for _, offset := range []uint64{0, 1, uint64(bc.cacheConfig.TriesInMemory) - 1} { + if number := bc.CurrentBlock().NumberU64(); number > offset { + recent := bc.GetBlockByNumber(number - offset) + + log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) + if err := triedb.Commit(recent.Root(), true); err != nil { + log.Error("Failed to commit recent state trie", "err", err) + } + } + } + if snapBase != (common.Hash{}) { + log.Info("Writing snapshot state to disk", "root", snapBase) + if err := triedb.Commit(snapBase, true); err != nil { log.Error("Failed to commit recent state trie", "err", err) } } - } - if snapBase != (common.Hash{}) { - log.Info("Writing snapshot state to disk", "root", snapBase) - if err := triedb.Commit(snapBase, true); err != nil { - log.Error("Failed to commit recent state trie", "err", err) + for !bc.triegc.Empty() { + triedb.Dereference(bc.triegc.PopItem().(common.Hash)) + } + if size, _ := triedb.Size(); size != 0 { + log.Error("Dangling trie nodes after full cleanup") } - } - for !bc.triegc.Empty() { - triedb.Dereference(bc.triegc.PopItem().(common.Hash)) - } - if size, _ := triedb.Size(); size != 0 { - log.Error("Dangling trie nodes after full cleanup") } } + // Flush the collected preimages to disk + if err := bc.triedb.Close(); err != nil { + log.Error("Failed to close trie db", "err", err) + } log.Info("Blockchain stopped") } @@ -1543,62 +1601,65 @@ func (bc *BlockChain) writeBlockWithState( if err != nil { return NonStatTy, err } - - // If we're running an archive node, always flush - if bc.cacheConfig.TrieDirtyDisabled { - if err := bc.triedb.Commit(root, false); err != nil { - return NonStatTy, err - } - } else { - // Full but not archive node, do proper garbage collection - bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive - bc.triegc.Push(root, -int64(block.NumberU64())) - - triesInMemory := uint64(bc.cacheConfig.TriesInMemory) - if current := block.NumberU64(); current > triesInMemory { - // If we exceeded our memory allowance, flush matured singleton nodes to disk - var ( - nodes, imgs = bc.triedb.Size() - limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024 - ) - if nodes > limit || imgs > 4*1024*1024 { - bc.triedb.Cap(limit - ethdb.IdealBatchSize) - } - // Find the next state trie we need to commit - chosen := current - triesInMemory - - // If we exceeded out time allowance, flush an entire trie to disk - if bc.gcproc > bc.cacheConfig.TrieTimeLimit { - // If the header is missing (canonical chain behind), we're reorging a low - // diff sidechain. Suspend committing until this operation is completed. - header := bc.GetHeaderByNumber(chosen) - if header == nil { - log.Warn("Reorg in progress, trie commit postponed", "number", chosen) - } else { - // If we're exceeding limits but haven't reached a large enough memory gap, - // warn the user that the system is becoming unstable. - if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { - log.Info( - "State in memory for too long, committing", - "time", bc.gcproc, - "allowance", bc.cacheConfig.TrieTimeLimit, - "optimum", float64(chosen-lastWrite)/float64(triesInMemory), - ) + // If node is running in path mode, skip explicit gc operation + // which is unnecessary in this mode. + if bc.triedb.Scheme() != rawdb.PathScheme { + // If we're running an archive node, always flush + if bc.cacheConfig.TrieDirtyDisabled { + if err := bc.triedb.Commit(root, false); err != nil { + return NonStatTy, err + } + } else { + // Full but not archive node, do proper garbage collection + bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive + bc.triegc.Push(root, -int64(block.NumberU64())) + + triesInMemory := uint64(bc.cacheConfig.TriesInMemory) + if current := block.NumberU64(); current > triesInMemory { + // If we exceeded our memory allowance, flush matured singleton nodes to disk + var ( + nodes, imgs = bc.triedb.Size() + limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024 + ) + if nodes > limit || imgs > 4*1024*1024 { + bc.triedb.Cap(limit - ethdb.IdealBatchSize) + } + // Find the next state trie we need to commit + chosen := current - triesInMemory + + // If we exceeded out time allowance, flush an entire trie to disk + if bc.gcproc > bc.cacheConfig.TrieTimeLimit { + // If the header is missing (canonical chain behind), we're reorging a low + // diff sidechain. Suspend committing until this operation is completed. + header := bc.GetHeaderByNumber(chosen) + if header == nil { + log.Warn("Reorg in progress, trie commit postponed", "number", chosen) + } else { + // If we're exceeding limits but haven't reached a large enough memory gap, + // warn the user that the system is becoming unstable. + if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + log.Info( + "State in memory for too long, committing", + "time", bc.gcproc, + "allowance", bc.cacheConfig.TrieTimeLimit, + "optimum", float64(chosen-lastWrite)/float64(triesInMemory), + ) + } + // Flush an entire trie and restart the counters + bc.triedb.Commit(header.Root, true) + lastWrite = chosen + bc.gcproc = 0 } - // Flush an entire trie and restart the counters - bc.triedb.Commit(header.Root, true) - lastWrite = chosen - bc.gcproc = 0 } - } - // Garbage collect anything below our required write retention - for !bc.triegc.Empty() { - root, number := bc.triegc.Pop() - if uint64(-number) > chosen { - bc.triegc.Push(root, number) - break + // Garbage collect anything below our required write retention + for !bc.triegc.Empty() { + root, number := bc.triegc.Pop() + if uint64(-number) > chosen { + bc.triegc.Push(root, number) + break + } + bc.triedb.Dereference(root.(common.Hash)) } - bc.triedb.Dereference(root.(common.Hash)) } } } @@ -2149,6 +2210,13 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, si ) parent := it.previous() for parent != nil && !bc.HasState(parent.Root) { + // If this is in pathdb and the state is in the history, recover it + if bc.stateRecoverable(parent.Root) { + if err := bc.triedb.Recover(parent.Root); err != nil { + return 0, err + } + break + } hashes = append(hashes, parent.Hash()) numbers = append(numbers, parent.Number.Uint64()) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index f8371afa2d..f9e7621d12 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -320,10 +320,16 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { return bc.HasState(block.Root()) } -// TrieNode retrieves a blob of data associated with a trie node -// either from ephemeral in-memory cache, or from persistent storage. -func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { - return bc.stateCache.TrieDB().Node(hash) +// stateRecoverable checks if the specified state is recoverable. +// Note, this function assumes the state is not present, because +// state is not treated as recoverable if it's available, thus +// false will be returned in this case. +func (bc *BlockChain) stateRecoverable(root common.Hash) bool { + if bc.triedb.Scheme() == rawdb.HashScheme { + return false + } + result, _ := bc.triedb.Recoverable(root) + return result } // TrieDB retrieves the low level trie database used for data storage. diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 81503ad132..6d28624c03 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "math/big" "os" + "path" "testing" "time" @@ -33,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // Tests a recovery for a short canonical chain where a recent block was already @@ -1751,12 +1753,18 @@ func testLongReorgedFastSyncingDeepRepair(t *testing.T, snapshots bool) { } func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { + testRepairWithScheme(t, tt, snapshots, rawdb.PathScheme) + testRepairWithScheme(t, tt, snapshots, rawdb.HashScheme) +} + +func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(true)) // Create a temporary persistent database datadir, err := ioutil.TempDir("", "") + ancient := path.Join(datadir, "ancient") if err != nil { t.Fatalf("Failed to create temporary datadir: %v", err) } @@ -1764,7 +1772,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, Ephemeral: true, }) if err != nil { @@ -1775,15 +1783,18 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { // Initialize a fresh chain var ( gspec = &Genesis{Config: params.TestChainConfig, BaseFee: big.NewInt(params.InitialBaseFee)} - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) engine = ethash.NewFullFaker() config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, // Disable snapshot by default + StateScheme: scheme, } ) + triedb.Close() if snapshots { config.SnapshotLimit = 256 config.SnapshotWait = true @@ -1810,7 +1821,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { t.Fatalf("Failed to import canonical chain start: %v", err) } if tt.commitBlock > 0 { - chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true) + chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), true) if snapshots { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { t.Fatalf("Failed to flatten snapshots: %v", err) @@ -1833,11 +1844,12 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } // Pull the plug on the database, simulating a hard crash db.Close() + chain.triedb.Close() // Start a new blockchain back up and see where the repair leads us db, err = rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, Ephemeral: true, }) @@ -1846,7 +1858,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } defer db.Close() - chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err = NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -1889,11 +1901,17 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { // In this case the snapshot layer of B3 is not created because of existent // state. func TestIssue23496(t *testing.T) { + //testIssue23496(t, rawdb.HashScheme) + testIssue23496(t, rawdb.PathScheme) +} + +func testIssue23496(t *testing.T, scheme string) { // It's hard to follow the test case, visualize the input //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // Create a temporary persistent database datadir, err := ioutil.TempDir("", "") + ancient := path.Join(datadir, "ancient") if err != nil { t.Fatalf("Failed to create temporary datadir: %v", err) } @@ -1901,7 +1919,7 @@ func TestIssue23496(t *testing.T) { db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, }) if err != nil { @@ -1912,7 +1930,8 @@ func TestIssue23496(t *testing.T) { // Initialize a fresh chain var ( gspec = &Genesis{Config: params.TestChainConfig, BaseFee: big.NewInt(params.InitialBaseFee)} - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) engine = ethash.NewFullFaker() config = &CacheConfig{ TrieCleanLimit: 256, @@ -1920,8 +1939,10 @@ func TestIssue23496(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 256, SnapshotWait: true, + StateScheme: scheme, } ) + triedb.Close() chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) @@ -1935,7 +1956,7 @@ func TestIssue23496(t *testing.T) { if _, err := chain.InsertChain(blocks[:1], nil); err != nil { t.Fatalf("Failed to import canonical chain start: %v", err) } - chain.stateCache.TrieDB().Commit(blocks[0].Root(), true) + chain.triedb.Commit(blocks[0].Root(), true) // Insert block B2 and commit the snapshot into disk if _, err := chain.InsertChain(blocks[1:2], nil); err != nil { @@ -1949,20 +1970,20 @@ func TestIssue23496(t *testing.T) { if _, err := chain.InsertChain(blocks[2:3], nil); err != nil { t.Fatalf("Failed to import canonical chain start: %v", err) } - chain.stateCache.TrieDB().Commit(blocks[2].Root(), true) + chain.triedb.Commit(blocks[2].Root(), true) // Insert the remaining blocks if _, err := chain.InsertChain(blocks[3:], nil); err != nil { t.Fatalf("Failed to import canonical chain tail: %v", err) } - // Pull the plug on the database, simulating a hard crash db.Close() + chain.triedb.Close() // Start a new blockchain back up and see where the repair leads us db, err = rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, Ephemeral: true, }) if err != nil { @@ -1970,7 +1991,7 @@ func TestIssue23496(t *testing.T) { } defer db.Close() - chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err = NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -1982,8 +2003,12 @@ func TestIssue23496(t *testing.T) { if head := chain.CurrentFastBlock(); head.NumberU64() != uint64(4) { t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), uint64(4)) } - if head := chain.CurrentBlock(); head.NumberU64() != uint64(1) { - t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), uint64(1)) + expHead := uint64(1) + if scheme == rawdb.PathScheme { + expHead = uint64(2) + } + if head := chain.CurrentBlock(); head.NumberU64() != expHead { + t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), expHead) } // Reinsert B2-B4 diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 3a66101c1c..8d48e3134e 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "math/big" "os" + "path" "strings" "testing" "time" @@ -31,9 +32,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // rewindTest is a test case for chain rollback upon user request. @@ -1950,12 +1955,18 @@ func testLongReorgedFastSyncingDeepSetHead(t *testing.T, snapshots bool) { } func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { + testSetHeadWithScheme(t, tt, snapshots, rawdb.PathScheme) + testSetHeadWithScheme(t, tt, snapshots, rawdb.HashScheme) +} + +func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(false)) // Create a temporary persistent database datadir, err := ioutil.TempDir("", "") + ancient := path.Join(datadir, "ancient") if err != nil { t.Fatalf("Failed to create temporary datadir: %v", err) } @@ -1963,7 +1974,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, Ephemeral: true, }) if err != nil { @@ -1974,15 +1985,18 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { // Initialize a fresh chain var ( gspec = &Genesis{Config: params.TestChainConfig, BaseFee: big.NewInt(params.InitialBaseFee)} - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, newDbConfig(scheme)) + genesis = gspec.MustCommit(db, triedb) engine = ethash.NewFullFaker() config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, // Disable snapshot + StateScheme: scheme, } ) + triedb.Close() if snapshots { config.SnapshotLimit = 256 config.SnapshotWait = true @@ -2009,7 +2023,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { t.Fatalf("Failed to import canonical chain start: %v", err) } if tt.commitBlock > 0 { - chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true) + chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), true) if snapshots { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { t.Fatalf("Failed to flatten snapshots: %v", err) @@ -2019,13 +2033,17 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { if _, err := chain.InsertChain(canonblocks[tt.commitBlock:], nil); err != nil { t.Fatalf("Failed to import canonical chain tail: %v", err) } - // Manually dereference anything not committed to not have to work with 128+ tries - for _, block := range sideblocks { - chain.stateCache.TrieDB().Dereference(block.Root()) - } - for _, block := range canonblocks { - chain.stateCache.TrieDB().Dereference(block.Root()) + // Reopen the trie database without persisting in-memory dirty nodes. + chain.triedb.Close() + dbconfig := &trie.Config{} + if scheme == rawdb.PathScheme { + dbconfig.PathDB = pathdb.Defaults + + } else { + dbconfig.HashDB = hashdb.Defaults } + chain.triedb = trie.NewDatabase(chain.db, dbconfig) + chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb) // Force run a freeze cycle type freezer interface { Freeze(threshold uint64) error diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index c4e1229648..d9f54e4a1f 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -22,9 +22,9 @@ package core import ( "bytes" "fmt" - "io/ioutil" "math/big" "os" + "path" "strings" "testing" "time" @@ -40,6 +40,7 @@ import ( // snapshotTestBasic wraps the common testing fields in the snapshot tests. type snapshotTestBasic struct { + scheme string // Disk scheme used for storing trie nodes chainBlocks int // Number of blocks to generate for the canonical chain snapshotBlock uint64 // Block number of the relevant snapshot disk layer commitBlock uint64 // Block number for which to commit the state to disk @@ -52,44 +53,38 @@ type snapshotTestBasic struct { // share fields, set in runtime datadir string + ancient string db ethdb.Database - gendb ethdb.Database + genDb ethdb.Database engine consensus.Engine + gspec *Genesis } func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { // Create a temporary persistent database - datadir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Failed to create temporary datadir: %v", err) - } - os.RemoveAll(datadir) + datadir := t.TempDir() + ancient := path.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, - Ephemeral: true, + AncientsDirectory: ancient, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } // Initialize a fresh chain var ( - gspec = &Genesis{Config: params.AllEthashProtocolChanges, BaseFee: big.NewInt(params.InitialBaseFee)} - genesis = gspec.MustCommit(db) - engine = ethash.NewFullFaker() - gendb = rawdb.NewMemoryDatabase() - - // Snapshot is enabled, the first snapshot is created from the Genesis. - // The snapshot memory allowance is 256MB, it means no snapshot flush - // will happen during the block insertion. - cacheConfig = defaultCacheConfig + gspec = &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: params.AllEthashProtocolChanges, + } + engine = ethash.NewFullFaker() ) - chain, err := NewBlockChain(db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(basic.scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, gendb, basic.chainBlocks, func(i int, b *BlockGen) {}, true) + genDb, blocks, _ := GenerateChainWithGenesis(gspec, engine, basic.chainBlocks, func(i int, b *BlockGen) {}) // Insert the blocks with configured settings. var breakpoints []uint64 @@ -106,7 +101,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo startPoint = point if basic.commitBlock > 0 && basic.commitBlock == point { - chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true) + chain.TrieDB().Commit(blocks[point-1].Root(), false) } if basic.snapshotBlock > 0 && basic.snapshotBlock == point { // Flushing the entire snap tree into the disk, the @@ -125,9 +120,11 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo // Set runtime fields basic.datadir = datadir + basic.ancient = ancient basic.db = db - basic.gendb = gendb + basic.genDb = genDb basic.engine = engine + basic.gspec = gspec return chain, blocks } @@ -139,17 +136,17 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [ if head := chain.CurrentHeader(); head.Number.Uint64() != basic.expHeadHeader { t.Errorf("Head header mismatch: have %d, want %d", head.Number, basic.expHeadHeader) } - if head := chain.CurrentFastBlock(); head.NumberU64() != basic.expHeadFastBlock { - t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadFastBlock) + if head := chain.CurrentFastBlock(); head.Number().Uint64() != basic.expHeadFastBlock { + t.Errorf("Head fast block mismatch: have %d, want %d", head.Number(), basic.expHeadFastBlock) } - if head := chain.CurrentBlock(); head.NumberU64() != basic.expHeadBlock { - t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadBlock) + if head := chain.CurrentBlock(); head.Number().Uint64() != basic.expHeadBlock { + t.Errorf("Head block mismatch: have %d, want %d", head.Number(), basic.expHeadBlock) } // Check the disk layer, ensure they are matched block := chain.GetBlockByNumber(basic.expSnapshotBottom) if block == nil { - t.Errorf("The correspnding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom) + t.Errorf("The corresponding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom) } else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) { t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot()) } @@ -160,6 +157,7 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [ } } +//nolint:unused func (basic *snapshotTestBasic) dump() string { buffer := new(strings.Builder) @@ -210,8 +208,9 @@ func (basic *snapshotTestBasic) dump() string { func (basic *snapshotTestBasic) teardown() { basic.db.Close() - basic.gendb.Close() + basic.genDb.Close() os.RemoveAll(basic.datadir) + os.RemoveAll(basic.ancient) } // snapshotTest is a test case type for normal snapshot recovery. @@ -228,8 +227,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { // Restart the chain normally chain.Stop() - gspec := &Genesis{Config: params.AllEthashProtocolChanges} - newchain, err := NewBlockChain(snaptest.db, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -238,7 +236,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { snaptest.verify(t, newchain, blocks) } -// crashSnapshotTest is a test case type for innormal snapshot recovery. +// crashSnapshotTest is a test case type for irregular snapshot recovery. // It can be used for testing that restart Geth after the crash. type crashSnapshotTest struct { snapshotTestBasic @@ -253,14 +251,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { // Pull the plug on the database, simulating a hard crash db := chain.db db.Close() + chain.triedb.Close() // Start a new blockchain back up and see where the repair leads us newdb, err := rawdb.Open(rawdb.OpenOptions{ Directory: snaptest.datadir, - AncientsDirectory: snaptest.datadir, - Ephemeral: true, + AncientsDirectory: snaptest.ancient, }) - if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } @@ -270,14 +267,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { // the crash, we do restart twice here: one after the crash and one // after the normal stop. It's used to ensure the broken snapshot // can be detected all the time. - gspec := &Genesis{Config: params.AllEthashProtocolChanges} - newchain, err := NewBlockChain(newdb, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } newchain.Stop() - newchain, err = NewBlockChain(newdb, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -304,7 +300,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { // Insert blocks without enabling snapshot if gapping is required. chain.Stop() - gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.gapped, func(i int, b *BlockGen) {}, true) + gappedBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {}, false) // Insert a few more blocks without enabling snapshot var cacheConfig = &CacheConfig{ @@ -312,9 +308,9 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, + StateScheme: snaptest.scheme, } - gspec := &Genesis{Config: params.AllEthashProtocolChanges} - newchain, err := NewBlockChain(snaptest.db, cacheConfig, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -322,7 +318,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { newchain.Stop() // Restart the chain with enabling the snapshot - newchain, err = NewBlockChain(snaptest.db, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -349,56 +345,8 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) { // Rewind the chain if setHead operation is required. chain.SetHead(snaptest.setHead) chain.Stop() - gspec := &Genesis{Config: params.AllEthashProtocolChanges} - newchain, err := NewBlockChain(snaptest.db, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer newchain.Stop() - - snaptest.verify(t, newchain, blocks) -} -// restartCrashSnapshotTest is the test type used to test this scenario: -// - have a complete snapshot -// - restart chain -// - insert more blocks with enabling the snapshot -// - commit the snapshot -// - crash -// - restart again -type restartCrashSnapshotTest struct { - snapshotTestBasic - newBlocks int -} - -func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { - // It's hard to follow the test case, visualize the input - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) - // fmt.Println(tt.dump()) - chain, blocks := snaptest.prepare(t) - - // Firstly, stop the chain properly, with all snapshot journal - // and state committed. - chain.Stop() - gspec := &Genesis{Config: params.AllEthashProtocolChanges} - newchain, err := NewBlockChain(snaptest.db, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}, true) - newchain.InsertChain(newBlocks, nil) - - // Commit the entire snapshot into the disk if requested. Note only - // (a) snapshot root and (b) snapshot generator will be committed, - // the diff journal is not. - newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0) - - // Simulate the blockchain crash - // Don't call chain.Stop here, so that no snapshot - // journal and latest state will be committed - - // Restart the chain after the crash - newchain, err = NewBlockChain(snaptest.db, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -432,35 +380,39 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, + StateScheme: snaptest.scheme, } - gspec := &Genesis{Config: params.AllEthashProtocolChanges} - newchain, err := NewBlockChain(snaptest.db, config, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}, true) + newBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {}, false) newchain.InsertChain(newBlocks, nil) newchain.Stop() - // Restart the chain, the wiper should starts working + // Restart the chain, the wiper should start working config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 256, SnapshotWait: false, // Don't wait rebuild + StateScheme: snaptest.scheme, } - newchain, err = NewBlockChain(snaptest.db, config, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } + // Simulate the blockchain crash. + tmp.triedb.Close() - newchain, err = NewBlockChain(snaptest.db, nil, gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } snaptest.verify(t, newchain, blocks) + newchain.Stop() } // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot @@ -484,20 +436,23 @@ func TestRestartWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C8 // Expected snapshot disk : G - test := &snapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 8, - expSnapshotBottom: 0, // Initial disk layer built from genesis - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &snapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 8, + expSnapshotBottom: 0, // Initial disk layer built from genesis + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -523,20 +478,23 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 0, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -562,20 +520,23 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C2 // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 2, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 2, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 2, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 2, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was crashed and restarts with a broken snapshot. In this case @@ -601,20 +562,27 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 6, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + expHead := uint64(0) + if scheme == rawdb.PathScheme { + expHead = uint64(4) + } + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 6, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: expHead, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was running with snapshot enabled. Then restarts without @@ -638,21 +606,24 @@ func TestGappedNewSnapshot(t *testing.T) { // Expected head fast block: C10 // Expected head block : C10 // Expected snapshot disk : C10 - test := &gappedSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD - }, - gapped: 2, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &gappedSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 10, + expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD + }, + gapped: 2, + } + test.test(t) + test.teardown() + } } // Tests the Geth was running with snapshot enabled and resetHead is applied. @@ -676,21 +647,24 @@ func TestSetHeadWithNewSnapshot(t *testing.T) { // Expected head fast block: C4 // Expected head block : C4 // Expected snapshot disk : G - test := &setHeadSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 4, - expHeadHeader: 4, - expHeadFastBlock: 4, - expHeadBlock: 4, - expSnapshotBottom: 0, // The initial disk layer is built from the genesis - }, - setHead: 4, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &setHeadSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 4, + expHeadHeader: 4, + expHeadFastBlock: 4, + expHeadBlock: 4, + expSnapshotBottom: 0, // The initial disk layer is built from the genesis + }, + setHead: 4, + } + test.test(t) + test.teardown() + } } // Tests the Geth was running with a complete snapshot and then imports a few @@ -714,19 +688,22 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) { // Expected head fast block: C10 // Expected head block : C8 // Expected snapshot disk : C10 - test := &wipeCrashSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, - }, - newBlocks: 2, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &wipeCrashSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 10, + expSnapshotBottom: 10, + }, + newBlocks: 2, + } + test.test(t) + test.teardown() + } } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index d5b72f8c4a..ba8eb240fd 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -57,15 +57,15 @@ var ( // newCanonical creates a chain database, and injects a deterministic canonical // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. -func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) { +func newCanonical(engine consensus.Engine, n int, full bool, scheme string) (ethdb.Database, *BlockChain, error) { var ( db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, BaseFee: big.NewInt(params.InitialBaseFee)} - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) ) - // Initialize a fresh chain with only a genesis block - blockchain, _ := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) // Create and inject the requested chain if n == 0 { return db, blockchain, nil @@ -87,9 +87,9 @@ func newGwei(n int64) *big.Int { } // Test fork of length N starting from block i -func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) { +func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int), scheme string) { // Copy old chain up to #i into a new db - db, blockchain2, err := newCanonical(ethash.NewFaker(), i, full) + db, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, scheme) if err != nil { t.Fatal("could not make new canonical in testFork", err) } @@ -204,7 +204,12 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error } func TestLastBlock(t *testing.T) { - _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + testLastBlock(t, rawdb.HashScheme) + testLastBlock(t, rawdb.PathScheme) +} + +func testLastBlock(t *testing.T, scheme string) { + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, scheme) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -221,14 +226,19 @@ func TestLastBlock(t *testing.T) { // Tests that given a starting canonical chain of a given size, it can be extended // with various length chains. -func TestExtendCanonicalHeaders(t *testing.T) { testExtendCanonical(t, false) } -func TestExtendCanonicalBlocks(t *testing.T) { testExtendCanonical(t, true) } - -func testExtendCanonical(t *testing.T, full bool) { +func TestExtendCanonicalHeaders(t *testing.T) { + testExtendCanonical(t, false, rawdb.HashScheme) + testExtendCanonical(t, false, rawdb.PathScheme) +} +func TestExtendCanonicalBlocks(t *testing.T) { + testExtendCanonical(t, true, rawdb.HashScheme) + testExtendCanonical(t, true, rawdb.PathScheme) +} +func testExtendCanonical(t *testing.T, full bool, scheme string) { length := 5 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -241,22 +251,28 @@ func testExtendCanonical(t *testing.T, full bool) { } } // Start fork from current height - testFork(t, processor, length, 1, full, better) - testFork(t, processor, length, 2, full, better) - testFork(t, processor, length, 5, full, better) - testFork(t, processor, length, 10, full, better) + testFork(t, processor, length, 1, full, better, scheme) + testFork(t, processor, length, 2, full, better, scheme) + testFork(t, processor, length, 5, full, better, scheme) + testFork(t, processor, length, 10, full, better, scheme) } // Tests that given a starting canonical chain of a given size, creating shorter // forks do not take canonical ownership. -func TestShorterForkHeaders(t *testing.T) { testShorterFork(t, false) } -func TestShorterForkBlocks(t *testing.T) { testShorterFork(t, true) } +func TestShorterForkHeaders(t *testing.T) { + testShorterFork(t, false, rawdb.HashScheme) + testShorterFork(t, false, rawdb.PathScheme) +} +func TestShorterForkBlocks(t *testing.T) { + testShorterFork(t, true, rawdb.HashScheme) + testShorterFork(t, true, rawdb.PathScheme) +} -func testShorterFork(t *testing.T, full bool) { +func testShorterFork(t *testing.T, full bool, scheme string) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -269,24 +285,30 @@ func testShorterFork(t *testing.T, full bool) { } } // Sum of numbers must be less than `length` for this to be a shorter fork - testFork(t, processor, 0, 3, full, worse) - testFork(t, processor, 0, 7, full, worse) - testFork(t, processor, 1, 1, full, worse) - testFork(t, processor, 1, 7, full, worse) - testFork(t, processor, 5, 3, full, worse) - testFork(t, processor, 5, 4, full, worse) + testFork(t, processor, 0, 3, full, worse, scheme) + testFork(t, processor, 0, 7, full, worse, scheme) + testFork(t, processor, 1, 1, full, worse, scheme) + testFork(t, processor, 1, 7, full, worse, scheme) + testFork(t, processor, 5, 3, full, worse, scheme) + testFork(t, processor, 5, 4, full, worse, scheme) } // Tests that given a starting canonical chain of a given size, creating longer // forks do take canonical ownership. -func TestLongerForkHeaders(t *testing.T) { testLongerFork(t, false) } -func TestLongerForkBlocks(t *testing.T) { testLongerFork(t, true) } +func TestLongerForkHeaders(t *testing.T) { + testLongerFork(t, false, rawdb.HashScheme) + testLongerFork(t, false, rawdb.PathScheme) +} +func TestLongerForkBlocks(t *testing.T) { + testLongerFork(t, true, rawdb.HashScheme) + testLongerFork(t, true, rawdb.PathScheme) +} -func testLongerFork(t *testing.T, full bool) { +func testLongerFork(t *testing.T, full bool, scheme string) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -299,24 +321,30 @@ func testLongerFork(t *testing.T, full bool) { } } // Sum of numbers must be greater than `length` for this to be a longer fork - testFork(t, processor, 0, 11, full, better) - testFork(t, processor, 0, 15, full, better) - testFork(t, processor, 1, 10, full, better) - testFork(t, processor, 1, 12, full, better) - testFork(t, processor, 5, 6, full, better) - testFork(t, processor, 5, 8, full, better) + testFork(t, processor, 0, 11, full, better, scheme) + testFork(t, processor, 0, 15, full, better, scheme) + testFork(t, processor, 1, 10, full, better, scheme) + testFork(t, processor, 1, 12, full, better, scheme) + testFork(t, processor, 5, 6, full, better, scheme) + testFork(t, processor, 5, 8, full, better, scheme) } // Tests that given a starting canonical chain of a given size, creating equal // forks do take canonical ownership. -func TestEqualForkHeaders(t *testing.T) { testEqualFork(t, false) } -func TestEqualForkBlocks(t *testing.T) { testEqualFork(t, true) } +func TestEqualForkHeaders(t *testing.T) { + testEqualFork(t, false, rawdb.HashScheme) + testEqualFork(t, false, rawdb.PathScheme) +} +func TestEqualForkBlocks(t *testing.T) { + testEqualFork(t, true, rawdb.HashScheme) + testEqualFork(t, true, rawdb.PathScheme) +} -func testEqualFork(t *testing.T, full bool) { +func testEqualFork(t *testing.T, full bool, scheme string) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -329,21 +357,27 @@ func testEqualFork(t *testing.T, full bool) { } } // Sum of numbers must be equal to `length` for this to be an equal fork - testFork(t, processor, 0, 10, full, equal) - testFork(t, processor, 1, 9, full, equal) - testFork(t, processor, 2, 8, full, equal) - testFork(t, processor, 5, 5, full, equal) - testFork(t, processor, 6, 4, full, equal) - testFork(t, processor, 9, 1, full, equal) + testFork(t, processor, 0, 10, full, equal, scheme) + testFork(t, processor, 1, 9, full, equal, scheme) + testFork(t, processor, 2, 8, full, equal, scheme) + testFork(t, processor, 5, 5, full, equal, scheme) + testFork(t, processor, 6, 4, full, equal, scheme) + testFork(t, processor, 9, 1, full, equal, scheme) } // Tests that chains missing links do not get accepted by the processor. -func TestBrokenHeaderChain(t *testing.T) { testBrokenChain(t, false) } -func TestBrokenBlockChain(t *testing.T) { testBrokenChain(t, true) } +func TestBrokenHeaderChain(t *testing.T) { + testBrokenChain(t, false, rawdb.HashScheme) + testBrokenChain(t, false, rawdb.PathScheme) +} +func TestBrokenBlockChain(t *testing.T) { + testBrokenChain(t, true, rawdb.HashScheme) + testBrokenChain(t, true, rawdb.PathScheme) +} -func testBrokenChain(t *testing.T, full bool) { +func testBrokenChain(t *testing.T, full bool, scheme string) { // Make chain starting from genesis - db, blockchain, err := newCanonical(ethash.NewFaker(), 10, full) + db, blockchain, err := newCanonical(ethash.NewFaker(), 10, full, scheme) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -365,19 +399,31 @@ func testBrokenChain(t *testing.T, full bool) { // Tests that reorganising a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. -func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) } -func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true) } +func TestReorgLongHeaders(t *testing.T) { + testReorgLong(t, false, rawdb.HashScheme) + testReorgLong(t, false, rawdb.PathScheme) +} +func TestReorgLongBlocks(t *testing.T) { + testReorgLong(t, true, rawdb.HashScheme) + testReorgLong(t, true, rawdb.PathScheme) +} -func testReorgLong(t *testing.T, full bool) { - testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280+params.GenesisDifficulty.Int64(), full) +func testReorgLong(t *testing.T, full bool, scheme string) { + testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280+params.GenesisDifficulty.Int64(), full, scheme) } // Tests that reorganising a short difficult chain after a long easy one // overwrites the canonical numbers and links in the database. -func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) } -func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) } +func TestReorgShortHeaders(t *testing.T) { + testReorgShort(t, false, rawdb.HashScheme) + testReorgShort(t, false, rawdb.PathScheme) +} +func TestReorgShortBlocks(t *testing.T) { + testReorgShort(t, true, rawdb.HashScheme) + testReorgShort(t, true, rawdb.PathScheme) +} -func testReorgShort(t *testing.T, full bool) { +func testReorgShort(t *testing.T, full bool, scheme string) { // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment // we need a fairly long chain of blocks with different difficulties for a short // one to become heavyer than a long one. The 96 is an empirical value. @@ -389,12 +435,12 @@ func testReorgShort(t *testing.T, full bool) { for i := 0; i < len(diff); i++ { diff[i] = -9 } - testReorg(t, easy, diff, 12615120+params.GenesisDifficulty.Int64(), full) + testReorg(t, easy, diff, 12615120+params.GenesisDifficulty.Int64(), full, scheme) } -func testReorg(t *testing.T, first, second []int64, td int64, full bool) { +func testReorg(t *testing.T, first, second []int64, td int64, full bool, scheme string) { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -462,12 +508,18 @@ func testReorg(t *testing.T, first, second []int64, td int64, full bool) { } // Tests that the insertion functions detect banned hashes. -func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) } -func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true) } +func TestBadHeaderHashes(t *testing.T) { + testBadHashes(t, false, rawdb.HashScheme) + testBadHashes(t, false, rawdb.PathScheme) +} +func TestBadBlockHashes(t *testing.T) { + testBadHashes(t, true, rawdb.HashScheme) + testBadHashes(t, true, rawdb.PathScheme) +} -func testBadHashes(t *testing.T, full bool) { +func testBadHashes(t *testing.T, full bool, scheme string) { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -496,12 +548,18 @@ func testBadHashes(t *testing.T, full bool) { // Tests that bad hashes are detected on boot, and the chain rolled back to a // good state prior to the bad hash. -func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) } -func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true) } +func TestReorgBadHeaderHashes(t *testing.T) { + testReorgBadHashes(t, false, rawdb.HashScheme) + testReorgBadHashes(t, false, rawdb.PathScheme) +} +func TestReorgBadBlockHashes(t *testing.T) { + testReorgBadHashes(t, true, rawdb.HashScheme) + testReorgBadHashes(t, true, rawdb.PathScheme) +} -func testReorgBadHashes(t *testing.T, full bool) { +func testReorgBadHashes(t *testing.T, full bool, scheme string) { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -532,7 +590,7 @@ func testReorgBadHashes(t *testing.T, full bool) { // Create a new BlockChain and check that it rolled back the state. gspec := &Genesis{Config: blockchain.chainConfig} - ncm, err := NewBlockChain(blockchain.db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ncm, err := NewBlockChain(blockchain.db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -552,13 +610,19 @@ func testReorgBadHashes(t *testing.T, full bool) { } // Tests chain insertions in the face of one entity containing an invalid nonce. -func TestHeadersInsertNonceError(t *testing.T) { testInsertNonceError(t, false) } -func TestBlocksInsertNonceError(t *testing.T) { testInsertNonceError(t, true) } +func TestHeadersInsertNonceError(t *testing.T) { + testInsertNonceError(t, false, rawdb.HashScheme) + testInsertNonceError(t, false, rawdb.PathScheme) +} +func TestBlocksInsertNonceError(t *testing.T) { + testInsertNonceError(t, true, rawdb.HashScheme) + testInsertNonceError(t, true, rawdb.PathScheme) +} -func testInsertNonceError(t *testing.T, full bool) { +func testInsertNonceError(t *testing.T, full bool, scheme string) { for i := 1; i < 25 && !t.Failed(); i++ { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -610,6 +674,11 @@ func testInsertNonceError(t *testing.T, full bool) { // Tests that fast importing a block chain produces the same chain data as the // classical full block processing. func TestFastVsFullChains(t *testing.T) { + testFastVsFullChains(t, rawdb.HashScheme) + testFastVsFullChains(t, rawdb.PathScheme) +} + +func testFastVsFullChains(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( gendb = rawdb.NewMemoryDatabase() @@ -621,7 +690,8 @@ func TestFastVsFullChains(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) + triedb = trie.NewDatabase(gendb, nil) + genesis = gspec.MustCommit(gendb, triedb) signer = types.LatestSigner(gspec.Config) ) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, 1024, func(i int, block *BlockGen) { @@ -644,8 +714,8 @@ func TestFastVsFullChains(t *testing.T) { }, true) // Import the chain as an archive node for the comparison baseline archiveDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(archiveDb) - archive, _ := NewBlockChain(archiveDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + gspec.MustCommit(archiveDb, trie.NewDatabase(archiveDb, newDbConfig(scheme))) + archive, _ := NewBlockChain(archiveDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer archive.Stop() if n, err := archive.InsertChain(blocks, nil); err != nil { @@ -653,8 +723,8 @@ func TestFastVsFullChains(t *testing.T) { } // Fast import the chain as a non-archive node to test fastDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(fastDb) - fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + gspec.MustCommit(fastDb, trie.NewDatabase(fastDb, newDbConfig(scheme))) + fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -677,8 +747,9 @@ func TestFastVsFullChains(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + triedb = trie.NewDatabase(ancientDb, nil) + gspec.MustCommit(ancientDb, triedb) + ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -746,6 +817,11 @@ func TestFastVsFullChains(t *testing.T) { // Tests that various import methods move the chain head pointers to the correct // positions. func TestLightVsFastVsFullChainHeads(t *testing.T) { + testLightVsFastVsFullChainHeads(t, rawdb.HashScheme) + testLightVsFastVsFullChainHeads(t, rawdb.PathScheme) +} + +func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( gendb = rawdb.NewMemoryDatabase() @@ -757,7 +833,8 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) + triedb = trie.NewDatabase(gendb, nil) + genesis = gspec.MustCommit(gendb, triedb) ) height := uint64(1024) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil, true) @@ -773,7 +850,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) return db, func() { os.RemoveAll(dir) } } // Configure a subchain to roll back @@ -793,12 +870,14 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { t.Errorf("%s head header mismatch: have #%v, want #%v", kind, num, header) } } + // Import the chain as an archive node and ensure all pointers are updated archiveDb, delfn := makeDb() defer delfn() archiveCaching := *defaultCacheConfig archiveCaching.TrieDirtyDisabled = true + archiveCaching.StateScheme = scheme archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := archive.InsertChain(blocks, nil); err != nil { @@ -813,7 +892,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb, delfn := makeDb() defer delfn() - fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -833,7 +912,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a ancient-first node and ensure all pointers are updated ancientDb, delfn := makeDb() defer delfn() - ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -852,7 +931,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb, delfn := makeDb() defer delfn() - light, _ := NewBlockChain(lightDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + light, _ := NewBlockChain(lightDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) } @@ -865,6 +944,11 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Tests that chain reorganisations handle transaction removals and reinsertions. func TestChainTxReorgs(t *testing.T) { + testChainTxReorgs(t, rawdb.HashScheme) + testChainTxReorgs(t, rawdb.PathScheme) +} + +func testChainTxReorgs(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") @@ -882,10 +966,10 @@ func TestChainTxReorgs(t *testing.T) { addr3: {Balance: big.NewInt(1000000000000000)}, }, } - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) signer = types.LatestSigner(gspec.Config) ) - // Create two transactions shared between the chains: // - postponed: transaction included at a later block in the forked chain // - swapped: transaction included at the same block number in the forked chain @@ -921,7 +1005,7 @@ func TestChainTxReorgs(t *testing.T) { } }, true) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if i, err := blockchain.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -980,6 +1064,11 @@ func TestChainTxReorgs(t *testing.T) { } func TestLogReorgs(t *testing.T) { + testLogReorgs(t, rawdb.HashScheme) + testLogReorgs(t, rawdb.PathScheme) +} + +func testLogReorgs(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -987,11 +1076,11 @@ func TestLogReorgs(t *testing.T) { // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) signer = types.LatestSigner(gspec.Config) ) - - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() rmLogsCh := make(chan RemovedLogsEvent) @@ -1036,15 +1125,20 @@ var logCode = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd // This test checks that log events and RemovedLogsEvent are sent // when the chain reorganizes. func TestLogRebirth(t *testing.T) { + testLogRebirth(t, rawdb.HashScheme) + testLogRebirth(t, rawdb.PathScheme) +} + +func testLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, nil)) signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() - blockchain, _ = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1100,14 +1194,19 @@ func TestLogRebirth(t *testing.T) { // This test is a variation of TestLogRebirth. It verifies that log events are emitted // when a side chain containing log events overtakes the canonical chain. func TestSideLogRebirth(t *testing.T) { + testSideLogRebirth(t, rawdb.HashScheme) + testSideLogRebirth(t, rawdb.PathScheme) +} + +func testSideLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, nil)) signer = types.LatestSigner(gspec.Config) - blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1170,6 +1269,11 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan Re } func TestReorgSideEvent(t *testing.T) { + testReorgSideEvent(t, rawdb.HashScheme) + testReorgSideEvent(t, rawdb.PathScheme) +} + +func testReorgSideEvent(t *testing.T, scheme string) { var ( db = rawdb.NewMemoryDatabase() key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1178,11 +1282,11 @@ func TestReorgSideEvent(t *testing.T) { Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}, } - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) signer = types.LatestSigner(gspec.Config) ) - - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {}, true) @@ -1254,7 +1358,12 @@ done: // Tests if the canonical block can be fetched from the database during chain insertion. func TestCanonicalBlockRetrieval(t *testing.T) { - _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + testCanonicalBlockRetrieval(t, rawdb.HashScheme) + testCanonicalBlockRetrieval(t, rawdb.PathScheme) +} + +func testCanonicalBlockRetrieval(t *testing.T, scheme string) { + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, scheme) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -1300,6 +1409,11 @@ func TestCanonicalBlockRetrieval(t *testing.T) { } func TestEIP155Transition(t *testing.T) { + testEIP155Transition(t, rawdb.HashScheme) + testEIP155Transition(t, rawdb.PathScheme) +} + +func testEIP155Transition(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( db = rawdb.NewMemoryDatabase() @@ -1311,10 +1425,10 @@ func TestEIP155Transition(t *testing.T) { Config: ¶ms.ChainConfig{ChainID: big.NewInt(1), EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)}, Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, } - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) ) - - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) { @@ -1403,6 +1517,11 @@ func TestEIP155Transition(t *testing.T) { } func TestEIP161AccountRemoval(t *testing.T) { + testEIP161AccountRemoval(t, rawdb.HashScheme) + testEIP161AccountRemoval(t, rawdb.PathScheme) +} + +func testEIP161AccountRemoval(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( db = rawdb.NewMemoryDatabase() @@ -1420,9 +1539,10 @@ func TestEIP161AccountRemoval(t *testing.T) { }, Alloc: GenesisAlloc{address: {Balance: funds}}, } - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) ) - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) { @@ -1475,11 +1595,16 @@ func TestEIP161AccountRemoval(t *testing.T) { // // https://github.com/ethereum/go-ethereum/pull/15941 func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { + testBlockchainHeaderchainReorgConsistency(t, rawdb.HashScheme) + testBlockchainHeaderchainReorgConsistency(t, rawdb.PathScheme) +} + +func testBlockchainHeaderchainReorgConsistency(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, nil)) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }, true) // Generate a bunch of fork blocks, each side forking from the canonical chain @@ -1495,10 +1620,10 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { // Import the canonical and fork chain side by side, verifying the current block // and current header consistency diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb, trie.NewDatabase(diskdb, newDbConfig(scheme))) gspec := &Genesis{Config: params.TestChainConfig} - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1525,7 +1650,7 @@ func TestTrieForkGC(t *testing.T) { engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*DefaultTriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }, true) // Generate a bunch of fork blocks, each side forking from the canonical chain @@ -1540,7 +1665,7 @@ func TestTrieForkGC(t *testing.T) { } // Import the canonical and fork chain side by side, forcing the trie cache to cache both diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb, trie.NewDatabase(diskdb, newDbConfig(rawdb.HashScheme))) gspec := &Genesis{Config: params.TestChainConfig} chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) @@ -1557,8 +1682,8 @@ func TestTrieForkGC(t *testing.T) { } // Dereference all the recent tries and ensure no past trie is left in for i := 0; i < DefaultTriesInMemory; i++ { - chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) - chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) + chain.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) + chain.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) } if nodes, _ := chain.TrieDB().Size(); nodes > 0 { t.Fatalf("stale tries still alive after garbase collection") @@ -1568,11 +1693,16 @@ func TestTrieForkGC(t *testing.T) { // Tests that doing large reorgs works even if the state associated with the // forking point is not available any more. func TestLargeReorgTrieGC(t *testing.T) { + testLargeReorgTrieGC(t, rawdb.HashScheme) + testLargeReorgTrieGC(t, rawdb.PathScheme) +} + +func testLargeReorgTrieGC(t *testing.T, scheme string) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, nil)) shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }, true) original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*DefaultTriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }, true) @@ -1580,10 +1710,12 @@ func TestLargeReorgTrieGC(t *testing.T) { // Import the shared chain and the original canonical one diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb, trie.NewDatabase(diskdb, newDbConfig(scheme))) gspec := &Genesis{Config: params.TestChainConfig} + db, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + defer db.Close() - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1594,7 +1726,7 @@ func TestLargeReorgTrieGC(t *testing.T) { t.Fatalf("failed to insert original chain: %v", err) } // Ensure that the state associated with the forking point is pruned away - if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil { + if chain.HasState(shared[len(shared)-1].Root()) { t.Fatalf("common-but-old ancestor still cache") } // Import the competitor chain without exceeding the canonical's TD and ensure @@ -1603,7 +1735,7 @@ func TestLargeReorgTrieGC(t *testing.T) { t.Fatalf("failed to insert competitor chain: %v", err) } for i, block := range competitor[:len(competitor)-2] { - if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { + if chain.HasState(block.Root()) { t.Fatalf("competitor %d: low TD chain became processed", i) } } @@ -1612,14 +1744,30 @@ func TestLargeReorgTrieGC(t *testing.T) { if _, err := chain.InsertChain(competitor[len(competitor)-2:], nil); err != nil { t.Fatalf("failed to finalize competitor chain: %v", err) } - for i, block := range competitor[:len(competitor)-DefaultTriesInMemory] { - if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { + // In path-based trie database implementation, it will keep 128 diff + 1 disk + // layers, totally 129 latest states available. In hash-based it's 128. + states := 128 + if scheme == rawdb.PathScheme { + states = states + 1 + } + for i, block := range competitor[:len(competitor)-states] { + if chain.HasState(block.Root()) { + t.Fatalf("competitor %d: unexpected competing chain state", i) + } + } + for i, block := range competitor[len(competitor)-states:] { + if !chain.HasState(block.Root()) { t.Fatalf("competitor %d: competing chain state missing", i) } } } func TestBlockchainRecovery(t *testing.T) { + testBlockchainRecovery(t, rawdb.HashScheme) + testBlockchainRecovery(t, rawdb.PathScheme) +} + +func testBlockchainRecovery(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( gendb = rawdb.NewMemoryDatabase() @@ -1627,7 +1775,8 @@ func TestBlockchainRecovery(t *testing.T) { address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) + triedb = trie.NewDatabase(gendb, nil) + genesis = gspec.MustCommit(gendb, triedb) ) height := uint64(1024) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil, true) @@ -1643,8 +1792,8 @@ func TestBlockchainRecovery(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + gspec.MustCommit(ancientDb, trie.NewDatabase(ancientDb, nil)) + ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -1664,7 +1813,7 @@ func TestBlockchainRecovery(t *testing.T) { rawdb.WriteHeadFastBlockHash(ancientDb, midBlock.Hash()) // Reopen broken blockchain again - ancient, _ = NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ = NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if num := ancient.CurrentBlock().NumberU64(); num != 0 { t.Errorf("head block mismatch: have #%v, want #%v", num, 0) @@ -1679,8 +1828,13 @@ func TestBlockchainRecovery(t *testing.T) { // This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain. func TestInsertReceiptChainRollback(t *testing.T) { + testInsertReceiptChainRollback(t, rawdb.HashScheme) + testInsertReceiptChainRollback(t, rawdb.PathScheme) +} + +func testInsertReceiptChainRollback(t *testing.T, scheme string) { // Generate forked chain. The returned BlockChain object is used to process the side chain blocks. - tmpChain, sideblocks, canonblocks, err := getLongAndShortChains() + tmpChain, sideblocks, canonblocks, err := getLongAndShortChains(scheme) if err != nil { t.Fatal(err) } @@ -1715,8 +1869,8 @@ func TestInsertReceiptChainRollback(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec := Genesis{Config: params.AllEthashProtocolChanges, BaseFee: big.NewInt(params.InitialBaseFee)} - gspec.MustCommit(ancientDb) - ancientChain, _ := NewBlockChain(ancientDb, nil, &gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + gspec.MustCommit(ancientDb, trie.NewDatabase(ancientDb, nil)) + ancientChain, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), &gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancientChain.Stop() // Import the canonical header chain. @@ -1761,10 +1915,15 @@ func TestInsertReceiptChainRollback(t *testing.T) { // - https://github.com/ethereum/go-ethereum/issues/18977 // - https://github.com/ethereum/go-ethereum/pull/18988 func TestLowDiffLongChain(t *testing.T) { + testLowDiffLongChain(t, rawdb.HashScheme) + testLowDiffLongChain(t, rawdb.PathScheme) +} + +func testLowDiffLongChain(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) // We must use a pretty long chain to ensure that the fork doesn't overtake us // until after at least 128 blocks post tip @@ -1774,11 +1933,12 @@ func TestLowDiffLongChain(t *testing.T) { }, true) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) + diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + defer diskdb.Close() + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb, trie.NewDatabase(diskdb, newDbConfig(rawdb.HashScheme))) gspec := &Genesis{Config: params.TestChainConfig} - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1819,12 +1979,12 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) // Generate and import the canonical chain blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*DefaultTriesInMemory, nil, true) diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb, trie.NewDatabase(diskdb, newDbConfig(rawdb.HashScheme))) gspec := &Genesis{Config: params.TestChainConfig} chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { @@ -1893,15 +2053,24 @@ func TestPrunedImportSide(t *testing.T) { testSideImport(t, 1, -10) } -func TestInsertKnownHeaders(t *testing.T) { testInsertKnownChainData(t, "headers") } -func TestInsertKnownReceiptChain(t *testing.T) { testInsertKnownChainData(t, "receipts") } -func TestInsertKnownBlocks(t *testing.T) { testInsertKnownChainData(t, "blocks") } +func TestInsertKnownHeaders(t *testing.T) { + testInsertKnownChainData(t, "headers", rawdb.HashScheme) + testInsertKnownChainData(t, "headers", rawdb.PathScheme) +} +func TestInsertKnownReceiptChain(t *testing.T) { + testInsertKnownChainData(t, "receipts", rawdb.HashScheme) + testInsertKnownChainData(t, "receipts", rawdb.PathScheme) +} +func TestInsertKnownBlocks(t *testing.T) { + testInsertKnownChainData(t, "blocks", rawdb.HashScheme) + testInsertKnownChainData(t, "blocks", rawdb.PathScheme) +} -func testInsertKnownChainData(t *testing.T, typ string) { +func testInsertKnownChainData(t *testing.T, typ string, scheme string) { engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, nil)) blocks, receipts := GenerateChain(params.TestChainConfig, genesis, engine, db, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }, true) // A longer chain but total difficulty is lower. @@ -1921,11 +2090,11 @@ func testInsertKnownChainData(t *testing.T, typ string) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb) + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb, trie.NewDatabase(chaindb, nil)) defer os.RemoveAll(dir) gspec := &Genesis{Config: params.TestChainConfig} - chain, err := NewBlockChain(chaindb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(chaindb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2020,11 +2189,11 @@ func testInsertKnownChainData(t *testing.T, typ string) { } // getLongAndShortChains returns two chains: A is longer, B is heavier. -func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyChain []*types.Block, err error) { +func getLongAndShortChains(scheme string) (bc *BlockChain, longChain []*types.Block, heavyChain []*types.Block, err error) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, nil)) // Generate and import the canonical chain, // Offset the time, to keep the difficulty low @@ -2032,10 +2201,10 @@ func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyCha b.SetCoinbase(common.Address{1}) }, true) diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb, trie.NewDatabase(diskdb, nil)) gspec := &Genesis{Config: params.TestChainConfig} - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create tester chain: %v", err) } @@ -2081,7 +2250,12 @@ func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyCha // 3. Then there should be no canon mapping for the block at height X // 4. The forked block should still be retrievable by hash func TestReorgToShorterRemovesCanonMapping(t *testing.T) { - chain, canonblocks, sideblocks, err := getLongAndShortChains() + testReorgToShorterRemovesCanonMapping(t, rawdb.HashScheme) + testReorgToShorterRemovesCanonMapping(t, rawdb.PathScheme) +} + +func testReorgToShorterRemovesCanonMapping(t *testing.T, scheme string) { + chain, canonblocks, sideblocks, err := getLongAndShortChains(scheme) if err != nil { t.Fatal(err) } @@ -2117,7 +2291,12 @@ func TestReorgToShorterRemovesCanonMapping(t *testing.T) { // as TestReorgToShorterRemovesCanonMapping, but applied on headerchain // imports -- that is, for fast sync func TestReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T) { - chain, canonblocks, sideblocks, err := getLongAndShortChains() + testReorgToShorterRemovesCanonMappingHeaderChain(t, rawdb.HashScheme) + testReorgToShorterRemovesCanonMappingHeaderChain(t, rawdb.PathScheme) +} + +func testReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T, scheme string) { + chain, canonblocks, sideblocks, err := getLongAndShortChains(scheme) if err != nil { t.Fatal(err) } @@ -2169,7 +2348,8 @@ func TestTransactionIndices(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) + triedb = trie.NewDatabase(gendb, nil) + genesis = gspec.MustCommit(gendb, triedb) signer = types.LatestSigner(gspec.Config) ) height := uint64(128) @@ -2224,7 +2404,7 @@ func TestTransactionIndices(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(ancientDb) + gspec.MustCommit(ancientDb, trie.NewDatabase(ancientDb, nil)) // Import all blocks into ancient db l := uint64(0) @@ -2252,7 +2432,7 @@ func TestTransactionIndices(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(ancientDb) + gspec.MustCommit(ancientDb, trie.NewDatabase(ancientDb, nil)) chain, err = NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) @@ -2272,7 +2452,7 @@ func TestTransactionIndices(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(ancientDb) + gspec.MustCommit(ancientDb, trie.NewDatabase(ancientDb, nil)) limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0} @@ -2287,8 +2467,12 @@ func TestTransactionIndices(t *testing.T) { chain.Stop() } } - func TestSkipStaleTxIndicesInFastSync(t *testing.T) { + testSkipStaleTxIndicesInFastSync(t, rawdb.HashScheme) + testSkipStaleTxIndicesInFastSync(t, rawdb.PathScheme) +} + +func testSkipStaleTxIndicesInFastSync(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( gendb = rawdb.NewMemoryDatabase() @@ -2296,7 +2480,7 @@ func TestSkipStaleTxIndicesInFastSync(t *testing.T) { address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(100000000000000000) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) + genesis = gspec.MustCommit(gendb, trie.NewDatabase(gendb, nil)) signer = types.LatestSigner(gspec.Config) ) height := uint64(128) @@ -2351,11 +2535,14 @@ func TestSkipStaleTxIndicesInFastSync(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(ancientDb) + triedb := trie.NewDatabase(ancientDb, nil) + gspec.MustCommit(ancientDb, triedb) + + defer ancientDb.Close() // Import all blocks into ancient db, only HEAD-32 indices are kept. l := uint64(32) - chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2396,7 +2583,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) blockGenerator := func(i int, block *BlockGen) { block.SetCoinbase(common.Address{1}) @@ -2417,7 +2604,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in for i := 0; i < b.N; i++ { // Import the shared chain and the original canonical one diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) + gspec.MustCommit(diskdb, trie.NewDatabase(diskdb, newDbConfig(rawdb.HashScheme))) chain, err := NewBlockChain(diskdb, nil, &gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { @@ -2492,17 +2679,22 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { // 2. Downloader starts to sync again // 3. The blocks fetched are all known and canonical blocks func TestSideImportPrunedBlocks(t *testing.T) { + testSideImportPrunedBlocks(t, rawdb.HashScheme) + testSideImportPrunedBlocks(t, rawdb.PathScheme) +} + +func testSideImportPrunedBlocks(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, nil)) // Generate and import the canonical chain blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*DefaultTriesInMemory, nil, true) diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) + (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb, trie.NewDatabase(diskdb, nil)) gspec := &Genesis{Config: params.TestChainConfig} - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2510,14 +2702,20 @@ func TestSideImportPrunedBlocks(t *testing.T) { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - lastPrunedIndex := len(blocks) - DefaultTriesInMemory - 1 + // In path-based trie database implementation, it will keep 128 diff + 1 disk + // layers, totally 129 latest states available. In hash-based it's 128. + states := DefaultTriesInMemory + if scheme == rawdb.PathScheme { + states = DefaultTriesInMemory + 1 + } + lastPrunedIndex := len(blocks) - states - 1 lastPrunedBlock := blocks[lastPrunedIndex] // Verify pruning of lastPrunedBlock if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64()) } - firstNonPrunedBlock := blocks[len(blocks)-DefaultTriesInMemory] + firstNonPrunedBlock := blocks[len(blocks)-states] // Verify firstNonPrunedBlock is not pruned if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) { t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64()) @@ -2539,6 +2737,11 @@ func TestSideImportPrunedBlocks(t *testing.T) { // each transaction, so this works ok. The rework accumulated writes in memory // first, but the journal wiped the entire state object on create-revert. func TestDeleteCreateRevert(t *testing.T) { + testDeleteCreateRevert(t, rawdb.HashScheme) + testDeleteCreateRevert(t, rawdb.PathScheme) +} + +func testDeleteCreateRevert(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb") @@ -2578,7 +2781,8 @@ func TestDeleteCreateRevert(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) ) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) { @@ -2594,9 +2798,9 @@ func TestDeleteCreateRevert(t *testing.T) { }, true) // Import the canonical chain diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) + gspec.MustCommit(diskdb, trie.NewDatabase(diskdb, nil)) - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2613,6 +2817,11 @@ func TestDeleteCreateRevert(t *testing.T) { // Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct, // and then the new slots exist func TestDeleteRecreateSlots(t *testing.T) { + testDeleteRecreateSlots(t, rawdb.HashScheme) + testDeleteRecreateSlots(t, rawdb.PathScheme) +} + +func testDeleteRecreateSlots(t *testing.T, scheme string) { var ( // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() @@ -2693,7 +2902,8 @@ func TestDeleteRecreateSlots(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) + triedb := trie.NewDatabase(db, nil) + genesis := gspec.MustCommit(db, triedb) blocks, _ := GenerateChain(&chainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) @@ -2707,9 +2917,9 @@ func TestDeleteRecreateSlots(t *testing.T) { b.AddTx(tx) }, true) // Import the canonical chain + db.Close() diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ Debug: true, Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -2742,6 +2952,11 @@ func TestDeleteRecreateSlots(t *testing.T) { // regular value-transfer // Expected outcome is that _all_ slots are cleared from A func TestDeleteRecreateAccount(t *testing.T) { + testDeleteRecreateAccount(t, rawdb.HashScheme) + testDeleteRecreateAccount(t, rawdb.PathScheme) +} + +func testDeleteRecreateAccount(t *testing.T, scheme string) { var ( // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() @@ -2775,7 +2990,7 @@ func TestDeleteRecreateAccount(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) blocks, _ := GenerateChain(&chainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) @@ -2790,8 +3005,8 @@ func TestDeleteRecreateAccount(t *testing.T) { }, true) // Import the canonical chain diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{ + gspec.MustCommit(diskdb, trie.NewDatabase(diskdb, nil)) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ Debug: true, Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -2820,6 +3035,11 @@ func TestDeleteRecreateAccount(t *testing.T) { // Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct, // and then the new slots exist func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { + testDeleteRecreateSlotsAcrossManyBlocks(t, rawdb.HashScheme) + testDeleteRecreateSlotsAcrossManyBlocks(t, rawdb.PathScheme) +} + +func testDeleteRecreateSlotsAcrossManyBlocks(t *testing.T, scheme string) { var ( // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() @@ -2901,7 +3121,9 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) + triedb := trie.NewDatabase(db, nil) + genesis := gspec.MustCommit(db, triedb) + var nonce uint64 type expectation struct { @@ -2965,8 +3187,10 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { }, true) // Import the canonical chain diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{ + triedb = trie.NewDatabase(diskdb, nil) + gspec.MustCommit(diskdb, triedb) + + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3023,6 +3247,11 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { // We need to either roll back the snapDestructs, or not place it into snapDestructs // in the first place. func TestInitThenFailCreateContract(t *testing.T) { + testInitThenFailCreateContract(t, rawdb.HashScheme) + testInitThenFailCreateContract(t, rawdb.PathScheme) +} + +func testInitThenFailCreateContract(t *testing.T, scheme string) { var ( // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() @@ -3085,7 +3314,9 @@ func TestInitThenFailCreateContract(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) + triedb := trie.NewDatabase(db, nil) + genesis := gspec.MustCommit(db, triedb) + nonce := uint64(0) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 4, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) @@ -3098,8 +3329,9 @@ func TestInitThenFailCreateContract(t *testing.T) { // Import the canonical chain diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{ + triedb = trie.NewDatabase(diskdb, nil) + gspec.MustCommit(diskdb, triedb) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3135,6 +3367,11 @@ func TestInitThenFailCreateContract(t *testing.T) { // checking that the gas usage of a hot SLOAD and a cold SLOAD are calculated // correctly. func TestEIP2718Transition(t *testing.T) { + testEIP2718Transition(t, rawdb.HashScheme) + testEIP2718Transition(t, rawdb.PathScheme) +} + +func testEIP2718Transition(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") @@ -3163,7 +3400,8 @@ func TestEIP2718Transition(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) ) blocks, _ := GenerateChain(gspec.Config, genesis, engine, db, 1, func(i int, b *BlockGen) { @@ -3187,9 +3425,9 @@ func TestEIP2718Transition(t *testing.T) { // Import the canonical chain diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) + gspec.MustCommit(diskdb, trie.NewDatabase(diskdb, nil)) - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3218,6 +3456,11 @@ func TestEIP2718Transition(t *testing.T) { // gasFeeCap - gasTipCap < baseFee. // 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). func TestEIP1559Transition(t *testing.T) { + testEIP1559Transition(t, rawdb.HashScheme) + testEIP1559Transition(t, rawdb.PathScheme) +} + +func testEIP1559Transition(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") @@ -3253,7 +3496,9 @@ func TestEIP1559Transition(t *testing.T) { gspec.Config.BerlinBlock = common.Big0 gspec.Config.LondonBlock = common.Big0 - genesis := gspec.MustCommit(db) + triedb := trie.NewDatabase(db, nil) + genesis := gspec.MustCommit(db, triedb) + signer := types.LatestSigner(gspec.Config) blocks, _ := GenerateChain(gspec.Config, genesis, engine, db, 1, func(i int, b *BlockGen) { @@ -3282,9 +3527,10 @@ func TestEIP1559Transition(t *testing.T) { }, true) diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) + triedb = trie.NewDatabase(diskdb, nil) + gspec.MustCommit(diskdb, triedb) - chain, err := NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3362,6 +3608,11 @@ func TestEIP1559Transition(t *testing.T) { } func TestSponsoredTxTransitionBeforeMiko(t *testing.T) { + testSponsoredTxTransitionBeforeMiko(t, rawdb.HashScheme) + testSponsoredTxTransitionBeforeMiko(t, rawdb.PathScheme) +} + +func testSponsoredTxTransitionBeforeMiko(t *testing.T, scheme string) { var chainConfig params.ChainConfig chainConfig.HomesteadBlock = common.Big0 @@ -3385,8 +3636,8 @@ func TestSponsoredTxTransitionBeforeMiko(t *testing.T) { gspec := &Genesis{ Config: &chainConfig, } - genesis := gspec.MustCommit(db) - chain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -3428,6 +3679,10 @@ func TestSponsoredTxTransitionBeforeMiko(t *testing.T) { } func TestSponsoredTxTransition(t *testing.T) { + testSponsoredTxTransition(t, rawdb.HashScheme) + testSponsoredTxTransition(t, rawdb.PathScheme) +} +func testSponsoredTxTransition(t *testing.T, scheme string) { var chainConfig params.ChainConfig chainConfig.HomesteadBlock = common.Big0 @@ -3464,8 +3719,8 @@ func TestSponsoredTxTransition(t *testing.T) { adminAddr: {Balance: math.BigPow(10, 18)}, }, } - genesis := gspec.MustCommit(db) - chain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -3577,6 +3832,7 @@ func TestSponsoredTxTransition(t *testing.T) { // 5. Sender does not have sufficient fund gasFee := new(big.Int).Mul(innerTx.GasFeeCap, new(big.Int).SetUint64(innerTx.Gas)) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, nil)) blocks, _ := GenerateChain(&chainConfig, genesis, engine, db, 1, func(i int, bg *BlockGen) { tx, err := types.SignTx(types.NewTransaction(0, payerAddr, gasFee, params.TxGas, bg.header.BaseFee, nil), mikoSigner, adminKey) if err != nil { @@ -3683,6 +3939,11 @@ func TestSponsoredTxTransition(t *testing.T) { // TestTransientStorageReset ensures the transient storage is wiped correctly // between transactions. func TestTransientStorageReset(t *testing.T) { + testTransientStorageReset(t, rawdb.HashScheme) + testTransientStorageReset(t, rawdb.PathScheme) +} + +func testTransientStorageReset(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -3754,7 +4015,7 @@ func TestTransientStorageReset(t *testing.T) { }) // Initialize the blockchain with 1153 enabled. - chain, err := NewBlockChain(db, nil, gspec, nil, engine, vmConfig, nil, nil) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vmConfig, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3775,6 +4036,11 @@ func TestTransientStorageReset(t *testing.T) { } func TestEIP3651(t *testing.T) { + testEIP3651(t, rawdb.HashScheme) + testEIP3651(t, rawdb.PathScheme) +} + +func testEIP3651(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb") @@ -3845,7 +4111,7 @@ func TestEIP3651(t *testing.T) { b.AddTx(tx) }) - chain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{Tracer: logger.NewMarkdownLogger(&logger.Config{}, os.Stderr)}, nil, nil) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{Tracer: logger.NewMarkdownLogger(&logger.Config{}, os.Stderr)}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3934,6 +4200,11 @@ func randBlob() (*kzg4844.Blob, *kzg4844.Commitment, *kzg4844.Proof) { } func TestInsertChainWithSidecars(t *testing.T) { + testInsertChainWithSidecars(t, rawdb.HashScheme) + testInsertChainWithSidecars(t, rawdb.PathScheme) +} + +func testInsertChainWithSidecars(t *testing.T, scheme string) { privateKey, _ := crypto.GenerateKey() address := crypto.PubkeyToAddress(privateKey.PublicKey) chainConfig := params.TestChainConfig @@ -3948,8 +4219,10 @@ func TestInsertChainWithSidecars(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) - chain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + triedb := trie.NewDatabase(db, nil) + gspec.MustCommit(db, triedb) + + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -3999,6 +4272,7 @@ func TestInsertChainWithSidecars(t *testing.T) { t.Fatal(err) } + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) blocks, _ := GenerateChain(chainConfig, genesis, engine, db, 1, func(i int, bg *BlockGen) { bg.AddTx(tx1) bg.AddTx(tx2) @@ -4033,8 +4307,11 @@ func TestInsertChainWithSidecars(t *testing.T) { // Reset database db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) + + chain.triedb.Close() + chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -4057,8 +4334,11 @@ func TestInsertChainWithSidecars(t *testing.T) { // Reset database db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + triedb = trie.NewDatabase(db, nil) + + chain.triedb.Close() + genesis = gspec.MustCommit(db, triedb) + chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -4105,8 +4385,11 @@ func TestInsertChainWithSidecars(t *testing.T) { // Reset database db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) + + chain.triedb.Close() + chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -4167,8 +4450,8 @@ func TestInsertChainWithSidecars(t *testing.T) { // Reset database db := rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - chain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -4201,8 +4484,9 @@ func TestInsertChainWithSidecars(t *testing.T) { // Reset database db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + triedb = trie.NewDatabase(db, nil) + genesis = gspec.MustCommit(db, triedb) + chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -4234,6 +4518,11 @@ func TestInsertChainWithSidecars(t *testing.T) { } func TestSidecarsPruning(t *testing.T) { + testSidecarsPruning(t, rawdb.HashScheme) + testSidecarsPruning(t, rawdb.PathScheme) +} + +func testSidecarsPruning(t *testing.T, scheme string) { privateKey, _ := crypto.GenerateKey() address := crypto.PubkeyToAddress(privateKey.PublicKey) chainConfig := params.TestChainConfig @@ -4248,8 +4537,8 @@ func TestSidecarsPruning(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) - chain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create blockchain, err %s", err) } @@ -4322,3 +4611,113 @@ func TestSidecarsPruning(t *testing.T) { } } } + +func TestDeleteThenCreate(t *testing.T) { + var ( + engine = ethash.NewFaker() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + factoryAddr = crypto.CreateAddress(address, 0) + funds = big.NewInt(1000000000000000) + ) + /* + contract Factory { + function deploy(bytes memory code) public { + address addr; + assembly { + addr := create2(0, add(code, 0x20), mload(code), 0) + if iszero(extcodesize(addr)) { + revert(0, 0) + } + } + } + } + */ + factoryBIN := common.Hex2Bytes("608060405234801561001057600080fd5b50610241806100206000396000f3fe608060405234801561001057600080fd5b506004361061002a5760003560e01c80627743601461002f575b600080fd5b610049600480360381019061004491906100d8565b61004b565b005b6000808251602084016000f59050803b61006457600080fd5b5050565b600061007b61007684610146565b610121565b905082815260208101848484011115610097576100966101eb565b5b6100a2848285610177565b509392505050565b600082601f8301126100bf576100be6101e6565b5b81356100cf848260208601610068565b91505092915050565b6000602082840312156100ee576100ed6101f5565b5b600082013567ffffffffffffffff81111561010c5761010b6101f0565b5b610118848285016100aa565b91505092915050565b600061012b61013c565b90506101378282610186565b919050565b6000604051905090565b600067ffffffffffffffff821115610161576101606101b7565b5b61016a826101fa565b9050602081019050919050565b82818337600083830152505050565b61018f826101fa565b810181811067ffffffffffffffff821117156101ae576101ad6101b7565b5b80604052505050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600080fd5b600080fd5b600080fd5b600080fd5b6000601f19601f830116905091905056fea2646970667358221220ea8b35ed310d03b6b3deef166941140b4d9e90ea2c92f6b41eb441daf49a59c364736f6c63430008070033") + + /* + contract C { + uint256 value; + constructor() { + value = 100; + } + function destruct() public payable { + selfdestruct(payable(msg.sender)); + } + receive() payable external {} + } + */ + contractABI := common.Hex2Bytes("6080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c63430008070033") + contractAddr := crypto.CreateAddress2(factoryAddr, [32]byte{}, crypto.Keccak256(contractABI)) + + gspec := &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{ + address: {Balance: funds}, + }, + } + nonce := uint64(0) + signer := types.HomesteadSigner{} + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 2, func(i int, b *BlockGen) { + fee := big.NewInt(1) + if b.header.BaseFee != nil { + fee = b.header.BaseFee + } + b.SetCoinbase(common.Address{1}) + + // Block 1 + if i == 0 { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + Data: factoryBIN, + }) + nonce++ + b.AddTx(tx) + + data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &factoryAddr, + Data: data, + }) + b.AddTx(tx) + nonce++ + } else { + // Block 2 + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &contractAddr, + Data: common.Hex2Bytes("2b68b9c6"), // destruct + }) + nonce++ + b.AddTx(tx) + + data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &factoryAddr, // re-creation + Data: data, + }) + b.AddTx(tx) + nonce++ + } + }) + // Import the canonical chain + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + for _, block := range blocks { + if _, err := chain.InsertChain([]*types.Block{block}, nil); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) + } + } +} diff --git a/core/chain_makers.go b/core/chain_makers.go index d0e20302a8..08f82b157d 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -21,6 +21,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -264,7 +265,7 @@ func generateChain( } blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) chainreader := newFakeChainReader(config, db) - genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { + genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} b.header = makeHeader(chainreader, parent, statedb, b.engine) @@ -323,7 +324,7 @@ func generateChain( panic(fmt.Sprintf("state write error: %v", err)) } if flushDisk { - if err := statedb.Database().TrieDB().Commit(root, false); err != nil { + if err := triedb.Commit(root, false); err != nil { panic(fmt.Sprintf("trie write error: %v", err)) } } @@ -331,14 +332,16 @@ func generateChain( } return nil, nil } - // Create an ephemeral database - database := state.NewDatabase(db) + // Forcibly use hash-based state scheme for retaining all nodes in disk. + triedb := trie.NewDatabase(db, trie.HashDefaults) + defer triedb.Close() + for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), database, nil) + statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { panic(err) } - block, receipt := genblock(i, parent, statedb) + block, receipt := genblock(i, parent, triedb, statedb) // Prepare Blob receipt var blobGasPrice *big.Int @@ -362,7 +365,10 @@ func generateChain( // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { db := rawdb.NewMemoryDatabase() - genesis.MustCommit(db) + triedb := trie.NewDatabase(db, trie.HashDefaults) + + defer triedb.Close() + genesis.MustCommit(db, triedb) blocks, receipts := GenerateChain(genesis.Config, genesis.ToBlock(), engine, db, n, gen, true) return db, blocks, receipts } diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 22b59e4754..adf2476bc3 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) func ExampleGenerateChain() { @@ -44,7 +45,7 @@ func ExampleGenerateChain() { Config: ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) // This call generates a chain of 5 blocks. The function runs for // each block and adds different features to gen based on the @@ -79,7 +80,7 @@ func ExampleGenerateChain() { }, true) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.HashScheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if i, err := blockchain.InsertChain(chain, nil); err != nil { diff --git a/core/dao_test.go b/core/dao_test.go index 2fa5b4e26c..0c862a9c11 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -34,18 +34,14 @@ func TestDAOForkRangeExtradata(t *testing.T) { chainConfig.HomesteadBlock = big.NewInt(0) // Generate a common prefix for both pro-forkers and non-forkers - db := rawdb.NewMemoryDatabase() gspec := &Genesis{ BaseFee: big.NewInt(params.InitialBaseFee), Config: &chainConfig, } - genesis := gspec.MustCommit(db) - prefix, _ := GenerateChain(&chainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}, true) + db, prefix, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) // Create the concurrent, conflicting two nodes proDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(proDb) - proConf := *params.NonActivatedConfig proConf.HomesteadBlock = big.NewInt(0) proConf.DAOForkBlock = forkBlock @@ -54,13 +50,10 @@ func TestDAOForkRangeExtradata(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), Config: &proConf, } - proBc, _ := NewBlockChain(proDb, nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer proBc.Stop() conDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(conDb) - conConf := *params.NonActivatedConfig conConf.HomesteadBlock = big.NewInt(0) conConf.DAOForkBlock = forkBlock @@ -69,7 +62,6 @@ func TestDAOForkRangeExtradata(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), Config: &conConf, } - conBc, _ := NewBlockChain(conDb, nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer conBc.Stop() @@ -82,10 +74,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Try to expand both pro-fork and non-fork chains iteratively with other camp's blocks for i := int64(0); i < params.DAOForkExtraRange.Int64(); i++ { // Create a pro-fork block, and try to feed into the no-fork chain - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) - defer bc.Stop() + bc, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) for j := 0; j < len(blocks)/2; j++ { @@ -94,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks, nil); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, true); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}, true) @@ -107,9 +96,8 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -119,7 +107,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks, nil); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, true); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}, true) @@ -133,9 +121,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { } } // Verify that contra-forkers accept pro-fork extra-datas after forking finishes - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -145,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks, nil); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, true); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}, true) @@ -153,9 +139,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -165,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks, nil); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, true); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}, true) diff --git a/core/error.go b/core/error.go index ca1724b9a6..a36db136e9 100644 --- a/core/error.go +++ b/core/error.go @@ -1,3 +1,4 @@ + // Copyright 2014 The go-ethereum Authors // This file is part of the go-ethereum library. // diff --git a/core/genesis.go b/core/genesis.go index ef3e3014aa..d1c8ff94fe 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -272,10 +272,12 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen } return genesis.Config, block.Hash(), nil } - // We have the genesis block in database(perhaps in ancient database) - // but the corresponding state is missing. + // The genesis block is present(perhaps in ancient database) while the + // state database is not initialized yet. It can happen that the node + // is initialized with an external ancient store. Commit genesis state + // in this case. header := rawdb.ReadHeader(db, stored, 0) - if _, err := state.New(header.Root, state.NewDatabaseWithNodeDB(db, triedb), nil); err != nil { + if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) { if genesis == nil { genesis = DefaultGenesisBlock() } @@ -449,8 +451,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block // MustCommit writes the genesis block and state to db, panicking on error. // The block is committed as the canonical head block. -func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { - block, err := g.Commit(db, trie.NewDatabase(db)) +func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block { + block, err := g.Commit(db, triedb) if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index aced782f3e..9fb0a988e8 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -30,18 +30,23 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) func TestInvalidCliqueConfig(t *testing.T) { block := DefaultGoerliGenesisBlock() block.ExtraData = []byte{} db := rawdb.NewMemoryDatabase() - if _, err := block.Commit(db, trie.NewDatabase(db)); err == nil { + if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil { t.Fatal("Expected error on invalid clique config") } } - func TestSetupGenesis(t *testing.T) { + testSetupGenesis(t, rawdb.HashScheme) + testSetupGenesis(t, rawdb.PathScheme) +} + +func testSetupGenesis(t *testing.T, scheme string) { var ( customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50") customg = Genesis{ @@ -63,7 +68,7 @@ func TestSetupGenesis(t *testing.T) { { name: "genesis without ChainConfig", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db), new(Genesis), false) + return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis), false) }, wantErr: errGenesisNoConfig, wantConfig: params.AllEthashProtocolChanges, @@ -71,7 +76,7 @@ func TestSetupGenesis(t *testing.T) { { name: "no block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db), nil, false) + return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil, false) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -79,8 +84,9 @@ func TestSetupGenesis(t *testing.T) { { name: "mainnet block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - DefaultGenesisBlock().MustCommit(db) - return SetupGenesisBlock(db, trie.NewDatabase(db), nil, false) + tdb := newDbConfig(scheme) + DefaultGenesisBlock().MustCommit(db, trie.NewDatabase(db, tdb)) + return SetupGenesisBlock(db, trie.NewDatabase(db, tdb), nil, false) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -88,8 +94,9 @@ func TestSetupGenesis(t *testing.T) { { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - customg.MustCommit(db) - return SetupGenesisBlock(db, trie.NewDatabase(db), nil, false) + tdb := newDbConfig(scheme) + customg.MustCommit(db, trie.NewDatabase(db, tdb)) + return SetupGenesisBlock(db, trie.NewDatabase(db, tdb), nil, false) }, wantHash: customghash, wantConfig: customg.Config, @@ -97,8 +104,9 @@ func TestSetupGenesis(t *testing.T) { { name: "custom block in DB, genesis == ropsten", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - customg.MustCommit(db) - return SetupGenesisBlock(db, trie.NewDatabase(db), DefaultRopstenGenesisBlock(), false) + tdb := newDbConfig(scheme) + customg.MustCommit(db, trie.NewDatabase(db, tdb)) + return SetupGenesisBlock(db, trie.NewDatabase(db, tdb), DefaultRopstenGenesisBlock(), false) }, wantErr: &GenesisMismatchError{Stored: customghash, New: params.RopstenGenesisHash}, wantHash: params.RopstenGenesisHash, @@ -107,8 +115,9 @@ func TestSetupGenesis(t *testing.T) { { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - oldcustomg.MustCommit(db) - return SetupGenesisBlock(db, trie.NewDatabase(db), &customg, false) + tdb := newDbConfig(scheme) + oldcustomg.MustCommit(db, trie.NewDatabase(db, tdb)) + return SetupGenesisBlock(db, trie.NewDatabase(db, tdb), &customg, false) }, wantHash: customghash, wantConfig: customg.Config, @@ -118,16 +127,17 @@ func TestSetupGenesis(t *testing.T) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. - genesis := oldcustomg.MustCommit(db) + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + oldcustomg.MustCommit(db, tdb) - bc, _ := NewBlockChain(db, nil, &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) defer bc.Stop() - blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil, true) + _, blocks, _ := GenerateChainWithGenesis(&oldcustomg, ethash.NewFaker(), 4, nil) bc.InsertChain(blocks, nil) bc.CurrentBlock() // This should return a compatibility error. - return SetupGenesisBlock(db, trie.NewDatabase(db), &customg, false) + return SetupGenesisBlock(db, tdb, &customg, false) }, wantHash: customghash, wantConfig: customg.Config, @@ -177,7 +187,8 @@ func TestGenesisHashes(t *testing.T) { {DefaultSepoliaGenesisBlock(), params.SepoliaGenesisHash}, } { // Test via MustCommit - if have := c.genesis.MustCommit(rawdb.NewMemoryDatabase()).Hash(); have != c.want { + db := rawdb.NewMemoryDatabase() + if have := c.genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)).Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock @@ -195,7 +206,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock := genesis.MustCommit(db) + genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") } @@ -243,3 +254,10 @@ func TestReadWriteGenesisAlloc(t *testing.T) { } } } + +func newDbConfig(scheme string) *trie.Config { + if scheme == rawdb.HashScheme { + return trie.HashDefaults + } + return &trie.Config{PathDB: pathdb.Defaults} +} diff --git a/core/headerchain_test.go b/core/headerchain_test.go index 3dd7d8c624..2163926fb5 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) func verifyUnbrokenCanonchain(hc *HeaderChain) error { @@ -72,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, BaseFee: big.NewInt(params.InitialBaseFee)} - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, nil)) ) hc, err := NewHeaderChain(db, params.AllEthashProtocolChanges, ethash.NewFaker(), func() bool { return false }) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 978220f15c..4706156602 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -36,7 +36,7 @@ import ( // // Now this scheme is still kept for backward compatibility, and it will be used // for archive node and some other tries(e.g. light trie). -const HashScheme = "hashScheme" +const HashScheme = "hash" // PathScheme is the new path-based state scheme with which trie nodes are stored // in the disk with node path as the database key. This scheme will only store one @@ -44,7 +44,7 @@ const HashScheme = "hashScheme" // is native. At the same time, this scheme will put adjacent trie nodes in the same // area of the disk with good data locality property. But this scheme needs to rely // on extra state diffs to survive deep reorg. -const PathScheme = "pathScheme" +const PathScheme = "path" // hasher is used to compute the sha256 hash of the provided data. type hasher struct{ sha crypto.KeccakState } @@ -208,9 +208,12 @@ func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash } else { blob, nHash = ReadStorageTrieNode(db, owner, path) } - if nHash != hash { + if len(blob) == 0 { return nil } + if nHash != hash { + return nil // Exists but not match + } return blob default: panic(fmt.Sprintf("Unknown scheme %v", scheme)) @@ -262,3 +265,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has panic(fmt.Sprintf("Unknown scheme %v", scheme)) } } + +// ReadStateScheme reads the state scheme of persistent state, or none +// if the state is not present in database. +func ReadStateScheme(db ethdb.Reader) string { + // Check if state in path-based scheme is present + blob, _ := ReadAccountTrieNode(db, nil) + if len(blob) != 0 { + return PathScheme + } + // In a hash-based scheme, the genesis state is consistently stored + // on the disk. To assess the scheme of the persistent state, it + // suffices to inspect the scheme of the genesis state. + header := ReadHeader(db, ReadCanonicalHash(db, 0), 0) + if header == nil { + return "" // empty datadir + } + blob = ReadLegacyTrieNode(db, header.Root) + if len(blob) == 0 { + return "" // no state in disk + } + return HashScheme +} diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go index b0f507cdd5..2773d3611a 100644 --- a/core/rawdb/ancient_scheme.go +++ b/core/rawdb/ancient_scheme.go @@ -51,9 +51,9 @@ const ( namespace = "eth/db/state" ) -// stateHistoryFreezerNoSnappy configures whether compression is disabled for the stateHistory. +// stateFreezerNoSnappy configures whether compression is disabled for the stateHistory. // https://github.com/golang/snappy, Reason for splititng files for looking up in archive mode easily. -var stateHistoryFreezerNoSnappy = map[string]bool{ +var stateFreezerNoSnappy = map[string]bool{ stateHistoryMeta: true, stateHistoryAccountIndex: false, stateHistoryStorageIndex: false, @@ -80,9 +80,9 @@ var ( // freezers the collections of all builtin freezers. var freezers = []string{chainFreezerName} -// NewStateHistoryFreezer initializes the freezer for state history. -func NewStateHistoryFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) { +// NewStateFreezer initializes the freezer for state history. +func NewStateFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) { return NewResettableFreezer( filepath.Join(ancientDir, stateFreezerName), namespace, readOnly, - stateHistoryTableSize, stateHistoryFreezerNoSnappy) + stateHistoryTableSize, stateFreezerNoSnappy) } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 5fe2423694..2a88ae5c18 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -50,39 +50,60 @@ func (info *freezerInfo) size() common.StorageSize { return total } +func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) { + info := freezerInfo{name: name} + for t := range order { + size, err := reader.AncientSize(t) + if err != nil { + return freezerInfo{}, err + } + info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)}) + } + // Retrieve the number of last stored item + ancients, err := reader.Ancients() + if err != nil { + return freezerInfo{}, err + } + info.head = ancients - 1 + + // Retrieve the number of first stored item + tail, err := reader.Tail() + if err != nil { + return freezerInfo{}, err + } + info.tail = tail + return info, nil +} + // inspectFreezers inspects all freezers registered in the system. func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { - var ( - infos []freezerInfo - ) + var infos []freezerInfo for _, freezer := range freezers { switch freezer { - case chainFreezerName: // We only support chain freezer for now. - // Chain ancient store is a bit special. It's always opened along - // with a key-value store, inspect the chain store directly. - info := freezerInfo{name: freezer} - // Retrieve storage size of every contained table. - for table := range chainFreezerNoSnappy { - size, err := db.AncientSize(table) - if err != nil { - return nil, err - } - info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)}) + case chainFreezerName: + info, err := inspect(chainFreezerName, chainFreezerNoSnappy, db) + if err != nil { + return nil, err } - // Retrieve the number of last stored item - ancients, err := db.Ancients() + infos = append(infos, info) + + case stateFreezerName: + datadir, err := db.AncientDatadir() if err != nil { return nil, err } - info.head = ancients - 1 + f, err := NewStateFreezer(datadir, true) + if err != nil { + return nil, err + } + defer f.Close() - // Retrieve the number of first stored item - tail, err := db.Tail() + info, err := inspect(stateFreezerName, stateFreezerNoSnappy, f) if err != nil { return nil, err } - info.tail = tail infos = append(infos, info) + default: return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index cf0cc15096..522fa33ca7 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -446,7 +446,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { tds stat numHashPairings stat hashNumPairings stat - tries stat + legacyTries stat + stateLookups stat + accountTries stat + storageTries stat codes stat txLookups stat accountSnaps stat @@ -487,8 +490,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { numHashPairings.Add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.Add(size) - case len(key) == common.HashLength: - tries.Add(size) + case IsLegacyTrieNode(key, it.Value()): + legacyTries.Add(size) + case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: + stateLookups.Add(size) + case IsAccountTrieNode(key): + accountTries.Add(size) + case IsStorageTrieNode(key): + storageTries.Add(size) case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: codes.Add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): @@ -526,7 +535,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, badBlockKey, highestFinalityVoteKey, storeInternalTxsEnabledKey, - snapshotSyncStatusKey, + snapshotSyncStatusKey, persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, } { if bytes.Equal(key, meta) { metadata.Add(size) @@ -556,7 +565,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, - {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, + {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, + {"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, + {"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()}, + {"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()}, {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 6c2afc1d1c..84b2bc8e5f 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -270,9 +270,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { return bytes.Equal(key, crypto.Keccak256(val)) } -// IsAccountTrieNode reports whether a provided database entry is an account -// trie node in path-based state scheme. -func IsAccountTrieNode(key []byte) (bool, []byte) { +// ResolveAccountTrieNodeKey reports whether a provided database entry is an +// account trie node in path-based state scheme, and returns the resolved +// node path if so. +func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { if !bytes.HasPrefix(key, trieNodeAccountPrefix) { return false, nil } @@ -285,9 +286,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) { return true, key[len(trieNodeAccountPrefix):] } -// IsStorageTrieNode reports whether a provided database entry is a storage +// IsAccountTrieNode reports whether a provided database entry is an account // trie node in path-based state scheme. -func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) { +func IsAccountTrieNode(key []byte) bool { + ok, _ := ResolveAccountTrieNodeKey(key) + return ok +} + +// ResolveStorageTrieNode reports whether a provided database entry is a storage +// trie node in path-based state scheme, and returns the resolved account hash +// and node path if so. +func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { if !bytes.HasPrefix(key, trieNodeStoragePrefix) { return false, common.Hash{}, nil } @@ -304,6 +313,13 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) { return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] } +// IsStorageTrieNode reports whether a provided database entry is a storage +// trie node in path-based state scheme. +func IsStorageTrieNode(key []byte) bool { + ok, _, _ := ResolveStorageTrieNode(key) + return ok +} + // stateIDKey = stateIDPrefix + root (32 bytes) func stateIDKey(root common.Hash) []byte { return append(stateIDPrefix, root.Bytes()...) diff --git a/core/rawdb/schema_test.go b/core/rawdb/schema_test.go index a1009b3bd2..06b017d41c 100644 --- a/core/rawdb/schema_test.go +++ b/core/rawdb/schema_test.go @@ -78,7 +78,7 @@ func TestIsLegacyTrieNode(t *testing.T) { } } -func TestIsAccountTrieNode(t *testing.T) { +func TestResolveAccountTrieNodeKey(t *testing.T) { tests := []struct { name string inputKey []byte @@ -137,14 +137,14 @@ func TestIsAccountTrieNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if check, key := IsAccountTrieNode(test.inputKey); check != test.expectedCheck || !bytes.Equal(key, test.expectedKey) { + if check, key := ResolveAccountTrieNodeKey(test.inputKey); check != test.expectedCheck || !bytes.Equal(key, test.expectedKey) { t.Errorf("expected %v, %v, got %v, %v", test.expectedCheck, test.expectedKey, check, key) } }) } } -func TestIsStorageTrieNode(t *testing.T) { +func TestResolveStorageTrieNode(t *testing.T) { tests := []struct { name string inputKey []byte @@ -219,7 +219,7 @@ func TestIsStorageTrieNode(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if check, hash, key := IsStorageTrieNode(test.inputKey); check != test.expectedCheck || !bytes.Equal(key, test.expectedKey) || hash != test.expectedHash { + if check, hash, key := ResolveStorageTrieNode(test.inputKey); check != test.expectedCheck || !bytes.Equal(key, test.expectedKey) || hash != test.expectedHash { t.Errorf("expected %v, %v, %v, got %v, %v, %v", test.expectedCheck, test.expectedHash, test.expectedKey, check, hash, key) } }) diff --git a/core/rlp_test.go b/core/rlp_test.go index bf30eff24f..ba4a9266fd 100644 --- a/core/rlp_test.go +++ b/core/rlp_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" ) @@ -45,7 +46,7 @@ func getBlock(transactions int, uncles int, dataSize int) *types.Block { Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}, } - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) ) // We need to generate as many blocks +1 as uncles diff --git a/core/state/database.go b/core/state/database.go index 6991e85843..01c7c51f26 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -97,7 +97,7 @@ type Trie interface { // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. - NodeIterator(startKey []byte) trie.NodeIterator + NodeIterator(startKey []byte) (trie.NodeIterator, error) // Prove constructs a Merkle proof for key. The result contains all encoded nodes // on the path to the value at key. The value itself is also included in the last @@ -122,7 +122,7 @@ func NewDatabase(db ethdb.Database) Database { func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New[common.Hash, int](codeSizeCacheSize) return &cachingDB{ - triedb: trie.NewDatabaseWithConfig(db, config), + triedb: trie.NewDatabase(db, config), codeSizeCache: csc, codeCache: fastcache.New(codeCacheSize), } diff --git a/core/state/dump.go b/core/state/dump.go index caa0061ce7..e9891960f9 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -138,8 +138,11 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] ) log.Info("Trie dumping started", "root", s.trie.Hash()) c.OnRoot(s.trie.Hash()) - - it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) + trieIt, err := s.trie.NodeIterator(conf.Start) + if err != nil { + return nil + } + it := trie.NewIterator(trieIt) for it.Next() { var data types.StateAccount if err := rlp.DecodeBytes(it.Value, &data); err != nil { @@ -166,9 +169,15 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] if !conf.SkipCode { account.Code = obj.Code() } + if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) - storageIt := trie.NewIterator(obj.getTrie().NodeIterator(nil)) + trieIt, err := obj.getTrie().NodeIterator(nil) + if err != nil { + log.Error("Failed to create trie iterator", "err", err) + continue + } + storageIt := trie.NewIterator(trieIt) for storageIt.Next() { _, content, _, err := rlp.Split(storageIt.Value) if err != nil { diff --git a/core/state/iterator.go b/core/state/iterator.go index ba7efd4653..653d566f85 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -74,8 +74,12 @@ func (it *NodeIterator) step() error { return nil } // Initialize the iterator if we've just started + var err error if it.stateIt == nil { - it.stateIt = it.state.trie.NodeIterator(nil) + it.stateIt, err = it.state.trie.NodeIterator(nil) + if err != nil { + return err + } } // If we had data nodes previously, we surely have at least state nodes if it.dataIt != nil { @@ -113,7 +117,10 @@ func (it *NodeIterator) step() error { if err != nil { return err } - it.dataIt = dataTrie.NodeIterator(nil) + it.dataIt, err = dataTrie.NodeIterator(nil) + if err != nil { + return err + } if !it.dataIt.Next(true) { it.dataIt = nil } diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 24b192c26c..2419c9175c 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -26,9 +26,14 @@ import ( // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { + testNodeIteratorCoverage(t, rawdb.HashScheme) + testNodeIteratorCoverage(t, rawdb.PathScheme) +} + +func testNodeIteratorCoverage(t *testing.T, scheme string) { // Create some arbitrary test state to iterate - db, sdb, root, _ := makeTestState() - sdb.TrieDB().Commit(root, false) + db, sdb, ndb, root, _ := makeTestState(scheme) + ndb.Commit(root, false) state, err := New(root, sdb, nil) if err != nil { @@ -48,7 +53,7 @@ func TestNodeIteratorCoverage(t *testing.T) { ) it := db.NewIterator(nil, nil) for it.Next() { - ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) + ok, hash := isTrieNode(scheme, it.Key(), it.Value()) if !ok { continue } @@ -90,11 +95,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) { return true, common.BytesToHash(key) } } else { - ok, _ := rawdb.IsAccountTrieNode(key) + ok := rawdb.IsAccountTrieNode(key) if ok { return true, crypto.Keccak256Hash(val) } - ok, _, _ = rawdb.IsStorageTrieNode(key) + ok = rawdb.IsStorageTrieNode(key) if ok { return true, crypto.Keccak256Hash(val) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index f46eb66990..05825cc3ad 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -88,7 +88,9 @@ func NewPruner(db ethdb.Database, datadir string, bloomSize uint64) (*Pruner, er if headBlock == nil { return nil, errors.New("Failed to load head block") } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false) + // Offline pruning is only supported in legacy hash based scheme. + triedb := trie.NewDatabase(db, trie.HashDefaults) + snaptree, err := snapshot.New(db, triedb, 256, headBlock.Root(), false, false, false) if err != nil { return nil, err // The relevant snapshot(s) might not exist } @@ -355,7 +357,9 @@ func RecoverPruning(datadir string, db ethdb.Database) error { // - The state HEAD is rewound already because of multiple incomplete `prune-state` // In this case, even the state HEAD is not exactly matched with snapshot, it // still feasible to recover the pruning correctly. - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, true) + // Offline pruning is only supported in legacy hash based scheme. + triedb := trie.NewDatabase(db, trie.HashDefaults) + snaptree, err := snapshot.New(db, triedb, 256, headBlock.Root(), false, false, true) if err != nil { return err // The relevant snapshot(s) might not exist } @@ -397,11 +401,14 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewSecure(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db)) + t, err := trie.NewSecure(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults)) + if err != nil { + return err + } + accIter, err := t.NodeIterator(nil) if err != nil { return err } - accIter := t.NodeIterator(nil) for accIter.Next(true) { hash := accIter.Hash() @@ -417,11 +424,14 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root), trie.NewDatabase(db)) + storageTrie, err := trie.NewSecure(trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root), trie.NewDatabase(db, trie.HashDefaults)) + if err != nil { + return err + } + storageIter, err := storageTrie.NodeIterator(nil) if err != nil { return err } - storageIter := storageTrie.NodeIterator(nil) for storageIter.Next(true) { hash := storageIter.Hash() if hash != (common.Hash{}) { diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 80e3d61bba..d8ecba8746 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -432,8 +432,9 @@ func (dl *diskLayer) generateRange(trieID *trie.ID, prefix []byte, kind string, var resolver trie.NodeResolver if len(result.keys) > 0 { mdb := rawdb.NewMemoryDatabase() - tdb := trie.NewDatabase(mdb) + tdb := trie.NewDatabase(mdb, trie.HashDefaults) snapTrie := trie.NewEmpty(tdb) + defer tdb.Close() for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } @@ -459,7 +460,7 @@ func (dl *diskLayer) generateRange(trieID *trie.ID, prefix []byte, kind string, var ( trieMore bool - nodeIt = tr.NodeIterator(origin) + nodeIt = tr.MustNodeIterator(origin) iter = trie.NewIterator(nodeIt) kvkeys, kvvals = result.keys, result.vals diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 08d83c46f6..caf13e42bb 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" ) @@ -45,10 +47,15 @@ func hashData(input []byte) common.Hash { // Tests that snapshot generation from an empty database. func TestGeneration(t *testing.T) { + testGeneration(t, rawdb.HashScheme) + testGeneration(t, rawdb.PathScheme) +} + +func testGeneration(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var helper = newHelper() + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) @@ -79,10 +86,15 @@ func TestGeneration(t *testing.T) { // Tests that snapshot generation with existent flat state. func TestGenerateExistentState(t *testing.T) { + testGenerateExistentState(t, rawdb.HashScheme) + testGenerateExistentState(t, rawdb.PathScheme) +} + +func testGenerateExistentState(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var helper = newHelper() + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) @@ -143,9 +155,15 @@ type testHelper struct { nodes *trienode.MergedNodeSet } -func newHelper() *testHelper { +func newHelper(scheme string) *testHelper { diskdb := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(diskdb) + config := &trie.Config{} + if scheme == rawdb.PathScheme { + config.PathDB = &pathdb.Config{} // disable caching + } else { + config.HashDB = &hashdb.Config{} // disable caching + } + triedb := trie.NewDatabase(diskdb, config) accTrie, _ := trie.NewSecure(trie.StateTrieID(common.Hash{}), triedb) return &testHelper{ diskdb: diskdb, @@ -228,7 +246,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { // - extra slots in the middle // - extra slots in the end func TestGenerateExistentStateWithWrongStorage(t *testing.T) { - helper := newHelper() + testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme) + testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme) +} + +func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { + helper := newHelper(scheme) // Account one, empty root but non-empty database helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: emptyRoot, CodeHash: emptyCode.Bytes()}) @@ -320,7 +343,13 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // - wrong accounts // - extra accounts func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { - helper := newHelper() + + testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme) + testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme) +} + +func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) { + helper := newHelper(scheme) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -375,10 +404,16 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { // Tests that snapshot generation errors out correctly in case of a missing trie // node in the account trie. func TestGenerateCorruptAccountTrie(t *testing.T) { + + testGenerateCorruptAccountTrie(t, rawdb.HashScheme) + testGenerateCorruptAccountTrie(t, rawdb.PathScheme) +} + +func testGenerateCorruptAccountTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // without any storage slots to keep the test smaller. - helper := newHelper() + helper := newHelper(scheme) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: emptyRoot, CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: emptyRoot, CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 @@ -386,9 +421,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 - // Delete an account trie leaf and ensure the generator chokes - helper.triedb.Commit(root, false) - helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) + // Delete an account trie node and ensure the generator chokes + targetPath := []byte{0xc} + targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7") + + rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { @@ -409,10 +446,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // trie node for a storage trie. It's similar to internal corruption but it is // handled differently inside the generator. func TestGenerateMissingStorageTrie(t *testing.T) { + testGenerateMissingStorageTrie(t, rawdb.HashScheme) + testGenerateMissingStorageTrie(t, rawdb.PathScheme) +} + +func testGenerateMissingStorageTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - helper := newHelper() + var ( + acc1 = hashData([]byte("acc-1")) + acc3 = hashData([]byte("acc-3")) + helper = newHelper(scheme) + ) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -421,8 +467,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) { helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() - // Delete a storage trie root and ensure the generator chokes - helper.diskdb.Delete(stRoot.Bytes()) + // Delete storage trie root of account one and three. + rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme) + rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { @@ -442,10 +489,16 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // Tests that snapshot generation errors out correctly in case of a missing trie // node in a storage trie. func TestGenerateCorruptStorageTrie(t *testing.T) { + + testGenerateCorruptStorageTrie(t, rawdb.HashScheme) + testGenerateCorruptStorageTrie(t, rawdb.PathScheme) +} + +func testGenerateCorruptStorageTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - helper := newHelper() + helper := newHelper(scheme) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -454,8 +507,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { helper.addTrieAccount("acc-3", &types.StateAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() - // Delete a storage trie leaf and ensure the generator chokes - helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) + // Delete a node in the storage trie. + targetPath := []byte{0x4} + targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371") + rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme) + rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { @@ -474,7 +530,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithExtraAccounts(t *testing.T) { - helper := newHelper() + testGenerateWithExtraAccounts(t, rawdb.HashScheme) + testGenerateWithExtraAccounts(t, rawdb.PathScheme) +} + +func testGenerateWithExtraAccounts(t *testing.T, scheme string) { + helper := newHelper(scheme) { // Account one in the trie stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), @@ -542,10 +603,15 @@ func enableLogging() { // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithManyExtraAccounts(t *testing.T) { + testGenerateWithManyExtraAccounts(t, rawdb.HashScheme) + testGenerateWithManyExtraAccounts(t, rawdb.PathScheme) +} + +func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { // Account one in the trie stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), @@ -599,11 +665,16 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { // So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted. // But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished. func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { + testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme) + testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme) +} + +func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) { accountCheckRange = 3 if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { acc := &types.StateAccount{Balance: big.NewInt(1), Root: emptyRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) @@ -636,11 +707,16 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { // TestGenerateWithMalformedSnapdata tests what happes if we have some junk // in the snapshot database, which cannot be parsed back to an account func TestGenerateWithMalformedSnapdata(t *testing.T) { + testGenerateWithMalformedSnapdata(t, rawdb.HashScheme) + testGenerateWithMalformedSnapdata(t, rawdb.PathScheme) +} + +func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) { accountCheckRange = 3 if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { acc := &types.StateAccount{Balance: big.NewInt(1), Root: emptyRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) @@ -673,10 +749,15 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { } func TestGenerateFromEmptySnap(t *testing.T) { + testGenerateFromEmptySnap(t, rawdb.HashScheme) + testGenerateFromEmptySnap(t, rawdb.PathScheme) +} + +func testGenerateFromEmptySnap(t *testing.T, scheme string) { //enableLogging() accountCheckRange = 10 storageCheckRange = 20 - helper := newHelper() + helper := newHelper(scheme) // Add 1K accounts to the trie for i := 0; i < 400; i++ { stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -708,8 +789,13 @@ func TestGenerateFromEmptySnap(t *testing.T) { // This hits a case where the snap verification passes, but there are more elements in the trie // which we must also add. func TestGenerateWithIncompleteStorage(t *testing.T) { + testGenerateWithIncompleteStorage(t, rawdb.HashScheme) + testGenerateWithIncompleteStorage(t, rawdb.PathScheme) +} + +func testGenerateWithIncompleteStorage(t *testing.T, scheme string) { storageCheckRange = 4 - helper := newHelper() + helper := newHelper(scheme) stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} // We add 8 accounts, each one is missing exactly one of the storage slots. This means diff --git a/core/state/state_test.go b/core/state/state_test.go index 881509f474..a66e87de00 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" @@ -41,7 +42,8 @@ func newStateEnv() *stateEnv { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) + tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + sdb, _ := New(types.EmptyRootHash, tdb, nil) s := &stateEnv{db: db, state: sdb} // generate a few entries @@ -56,8 +58,10 @@ func TestDump(t *testing.T) { s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) s.state.Commit(0, false) + root, _ := s.state.Commit(0, false) // check that DumpToCollector contains the state objects that are in trie + s.state, _ = New(root, tdb, nil) got := string(s.state.Dump(nil)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", diff --git a/core/state/statedb.go b/core/state/statedb.go index b4d66f0687..1d5ff3b369 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -64,6 +64,11 @@ func (n *proofList) Delete(key []byte) error { // nested states. It's the general query interface to retrieve: // * Contracts // * Accounts +// +// Once the state is committed, tries cached in stateDB (including account +// trie, storage tries) will no longer be functional. A new state instance +// must be created with new root and updated database for accessing post- +// commit states. type StateDB struct { db Database prefetcher *triePrefetcher @@ -707,15 +712,19 @@ func (s *StateDB) CreateAccount(addr common.Address) { } } -func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { - so := db.getStateObject(addr) +func (s *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { + so := s.getStateObject(addr) if so == nil { return nil } - it := trie.NewIterator(so.getTrie().NodeIterator(nil)) + trieIt, err := so.getTrie().NodeIterator(nil) + if err != nil { + return err + } + it := trie.NewIterator(trieIt) for it.Next() { - key := common.BytesToHash(db.trie.GetKey(it.Key)) + key := common.BytesToHash(s.trie.GetKey(it.Key)) if value, dirty := so.dirtyStorage[key]; dirty { if !cb(key, value) { return nil @@ -997,7 +1006,10 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root if err != nil { return false, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } - it := tr.NodeIterator(nil) + it, err := tr.NodeIterator(nil) + if err != nil { + return false, nil, nil, fmt.Errorf("failed to create iterator, err: %w", err) + } var ( set = trienode.NewNodeSet(addrHash) slots = make(map[common.Hash][]byte) @@ -1137,6 +1149,10 @@ func (s *StateDB) clearJournalAndRefund() { // // The associated block number of the state transition is also provided // for more chain context. +// Once the state is committed, tries cached in stateDB (including account +// trie, storage tries) will no longer be functional. A new state instance +// must be created with new root and updated database for accessing post- +// commit states. func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, error) { if s.dbErr != nil { return common.Hash{}, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) @@ -1151,8 +1167,9 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er storageTrieNodesUpdated int storageTrieNodesDeleted int nodes = trienode.NewMergedNodeSet() + codeWriter = s.db.TrieDB().DiskDB().NewBatch() ) - codeWriter := s.db.TrieDB().DiskDB().NewBatch() + // Handle all state deletions first incomplete, err := s.handleDestruction(nodes) if err != nil { diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index bea297d0f9..1bef564ed9 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -179,7 +179,7 @@ func (test *stateTest) run() bool { storageList = append(storageList, copy2DSet(states.Storages)) } disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit}) + tdb = trie.NewDatabase(disk, &trie.Config{OnCommit: onCommit}) sdb = NewDatabaseWithNodeDB(disk, tdb) byzantium = rand.Intn(2) == 0 ) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index aa08ebb926..841702f98c 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" ) // Tests that updating a state trie does not leak any database writes prior to @@ -518,7 +519,8 @@ func TestCopyOfCopy(t *testing.T) { // // See https://github.com/ethereum/go-ethereum/issues/20106. func TestCopyCommitCopy(t *testing.T) { - state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) + tdb := NewDatabase(rawdb.NewMemoryDatabase()) + state, _ := New(types.EmptyRootHash, tdb, nil) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") @@ -556,19 +558,6 @@ func TestCopyCommitCopy(t *testing.T) { t.Fatalf("first copy pre-commit committed storage slot mismatch: have %x, want %x", val, common.Hash{}) } - copyOne.Commit(0, false) - if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { - t.Fatalf("first copy post-commit balance mismatch: have %v, want %v", balance, 42) - } - if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { - t.Fatalf("first copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) - } - if val := copyOne.GetState(addr, skey); val != sval { - t.Fatalf("first copy post-commit non-committed storage slot mismatch: have %x, want %x", val, sval) - } - if val := copyOne.GetCommittedState(addr, skey); val != sval { - t.Fatalf("first copy post-commit committed storage slot mismatch: have %x, want %x", val, sval) - } // Copy the copy and check the balance once more copyTwo := copyOne.Copy() if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { @@ -580,8 +569,23 @@ func TestCopyCommitCopy(t *testing.T) { if val := copyTwo.GetState(addr, skey); val != sval { t.Fatalf("second copy non-committed storage slot mismatch: have %x, want %x", val, sval) } - if val := copyTwo.GetCommittedState(addr, skey); val != sval { - t.Fatalf("second copy post-commit committed storage slot mismatch: have %x, want %x", val, sval) + if val := copyTwo.GetCommittedState(addr, skey); val != (common.Hash{}) { + t.Fatalf("second copy committed storage slot mismatch: have %x, want %x", val, sval) + } + // Commit state, ensure states can be loaded from disk + root, _ := state.Commit(0, false) + state, _ = New(root, tdb, nil) + if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + t.Fatalf("state post-commit balance mismatch: have %v, want %v", balance, 42) + } + if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + t.Fatalf("state post-commit code mismatch: have %x, want %x", code, []byte("hello")) + } + if val := state.GetState(addr, skey); val != sval { + t.Fatalf("state post-commit non-committed storage slot mismatch: have %x, want %x", val, sval) + } + if val := state.GetCommittedState(addr, skey); val != sval { + t.Fatalf("state post-commit committed storage slot mismatch: have %x, want %x", val, sval) } } @@ -641,19 +645,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if val := copyTwo.GetCommittedState(addr, skey); val != (common.Hash{}) { t.Fatalf("second copy pre-commit committed storage slot mismatch: have %x, want %x", val, common.Hash{}) } - copyTwo.Commit(0, false) - if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { - t.Fatalf("second copy post-commit balance mismatch: have %v, want %v", balance, 42) - } - if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { - t.Fatalf("second copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) - } - if val := copyTwo.GetState(addr, skey); val != sval { - t.Fatalf("second copy post-commit non-committed storage slot mismatch: have %x, want %x", val, sval) - } - if val := copyTwo.GetCommittedState(addr, skey); val != sval { - t.Fatalf("second copy post-commit committed storage slot mismatch: have %x, want %x", val, sval) - } + // Copy the copy-copy and check the balance once more copyThree := copyTwo.Copy() if balance := copyThree.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { @@ -665,11 +657,60 @@ func TestCopyCopyCommitCopy(t *testing.T) { if val := copyThree.GetState(addr, skey); val != sval { t.Fatalf("third copy non-committed storage slot mismatch: have %x, want %x", val, sval) } - if val := copyThree.GetCommittedState(addr, skey); val != sval { + if val := copyThree.GetCommittedState(addr, skey); val != (common.Hash{}) { t.Fatalf("third copy committed storage slot mismatch: have %x, want %x", val, sval) } } +// TestCommitCopy tests the copy from a committed state is not functional. +func TestCommitCopy(t *testing.T) { + state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + + // Create an account and check if the retrieved balance is correct + addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") + skey := common.HexToHash("aaa") + sval := common.HexToHash("bbb") + + state.SetBalance(addr, big.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie + + if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) + } + if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + t.Fatalf("initial code mismatch: have %x, want %x", code, []byte("hello")) + } + if val := state.GetState(addr, skey); val != sval { + t.Fatalf("initial non-committed storage slot mismatch: have %x, want %x", val, sval) + } + if val := state.GetCommittedState(addr, skey); val != (common.Hash{}) { + t.Fatalf("initial committed storage slot mismatch: have %x, want %x", val, common.Hash{}) + } + // Copy the committed state database, the copied one is not functional. + state.Commit(0, true) + copied := state.Copy() + if balance := copied.GetBalance(addr); balance.Cmp(big.NewInt(0)) != 0 { + t.Fatalf("unexpected balance: have %v", balance) + } + if code := copied.GetCode(addr); code != nil { + t.Fatalf("unexpected code: have %x", code) + } + if val := copied.GetState(addr, skey); val != (common.Hash{}) { + t.Fatalf("unexpected storage slot: have %x", val) + } + if val := copied.GetCommittedState(addr, skey); val != (common.Hash{}) { + t.Fatalf("unexpected storage slot: have %x", val) + } + // Should compare based on the error message. + if !strings.Contains(copied.Error().Error(), trie.ErrCommitted.Error()) { + t.Fatalf("unexpected state error, %v", copied.Error()) + } + // if !errors.Is(copied.Error(), trie.ErrCommitted) { + // t.Fatalf("unexpected state error, %v", copied.Error()) + // } +} + // TestDeleteCreateRevert tests a weird state transition corner case that we hit // while changing the internals of StateDB. The workflow is that a contract is // self-destructed, then in a follow-up transaction (but same block) it's created diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 30742ed3b2..559f37044c 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -29,6 +29,8 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // testAccount is the data associated with an account used by the state tests. @@ -40,24 +42,28 @@ type testAccount struct { } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { +func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) { // Create an empty state + config := &trie.Config{Preimages: true} + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } else { + config.HashDB = hashdb.Defaults + } db := rawdb.NewMemoryDatabase() - sdb := NewDatabase(db) - state, _ := New(common.Hash{}, sdb, nil) + nodeDb := trie.NewDatabase(db, config) + sdb := NewDatabaseWithNodeDB(db, nodeDb) + state, _ := New(types.EmptyRootHash, sdb, nil) // Fill it with some arbitrary data var accounts []*testAccount for i := byte(0); i < 96; i++ { obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i})) acc := &testAccount{address: common.BytesToAddress([]byte{i})} - obj.AddBalance(big.NewInt(int64(11 * i))) acc.balance = big.NewInt(int64(11 * i)) - obj.SetNonce(uint64(42 * i)) acc.nonce = uint64(42 * i) - if i%3 == 0 { obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i}) acc.code = []byte{i, i, i, i, i} @@ -68,24 +74,27 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { obj.SetState(hash, hash) } } - state.updateStateObject(obj) accounts = append(accounts, acc) } root, _ := state.Commit(0, false) // Return the generated state - return db, sdb, root, accounts + return db, sdb, nodeDb, root, accounts } // checkStateAccounts cross references a reconstructed state with an expected // account array. -func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { +func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root common.Hash, accounts []*testAccount) { + var config trie.Config + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } // Check root availability and state contents - state, err := New(root, NewDatabase(db), nil) + state, err := New(root, NewDatabaseWithConfig(db, &config), nil) if err != nil { t.Fatalf("failed to create state trie at %x: %v", root, err) } - if err := checkStateConsistency(db, root); err != nil { + if err := checkStateConsistency(db, scheme, root); err != nil { t.Fatalf("inconsistent state trie at %x: %v", root, err) } for i, acc := range accounts { @@ -102,27 +111,35 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou } // checkTrieConsistency checks that all nodes in a (sub-)trie are indeed present. -func checkTrieConsistency(db ethdb.Database, root common.Hash) error { +func checkTrieConsistency(db ethdb.Database, scheme string, root common.Hash) error { + config := &trie.Config{Preimages: true} + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } if v, _ := db.Get(root[:]); v == nil { return nil // Consider a non existent state consistent. } - trie, err := trie.New(trie.StateTrieID(root), trie.NewDatabase(db)) + trie, err := trie.New(trie.StateTrieID(root), trie.NewDatabase(db, config)) if err != nil { return err } - it := trie.NodeIterator(nil) + it := trie.MustNodeIterator(nil) for it.Next(true) { } return it.Error() } // checkStateConsistency checks that all data of a state root is present. -func checkStateConsistency(db ethdb.Database, root common.Hash) error { - // Create and iterate a state trie rooted in a sub-node - if _, err := db.Get(root.Bytes()); err != nil { - return nil // Consider a non existent state consistent. +func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) error { + config := &trie.Config{Preimages: true} + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults } - state, err := New(root, NewDatabase(db), nil) + // // Create and iterate a state trie rooted in a sub-node + // if _, err := db.Get(root.Bytes()); err != nil { + // return err // Consider a non existent state consistent. + // } + state, err := New(root, NewDatabaseWithConfig(db, config), nil) if err != nil { return err } @@ -134,9 +151,14 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { - db := trie.NewDatabase(rawdb.NewMemoryDatabase()) - empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), trie.NewSyncBloom(1, memorydb.New()), nil, db.Scheme()) + dbA := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbB := trie.NewDatabase(rawdb.NewMemoryDatabase(), &trie.Config{PathDB: pathdb.Defaults}) + + sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), trie.NewSyncBloom(1, memorydb.New()), nil, dbA.Scheme()) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes) + } + sync = NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), trie.NewSyncBloom(1, memorydb.New()), nil, dbB.Scheme()) if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { t.Errorf(" content requested for empty state: %v, %v, %v", nodes, paths, codes) } @@ -145,22 +167,29 @@ func TestEmptyStateSync(t *testing.T) { // Tests that given a root hash, a state can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go. func TestIterativeStateSyncIndividual(t *testing.T) { - testIterativeStateSync(t, 1, false, false) + + testIterativeStateSync(t, 1, false, false, rawdb.HashScheme) + testIterativeStateSync(t, 1, false, false, rawdb.PathScheme) } func TestIterativeStateSyncBatched(t *testing.T) { - testIterativeStateSync(t, 100, false, false) + testIterativeStateSync(t, 100, false, false, rawdb.HashScheme) + testIterativeStateSync(t, 100, false, false, rawdb.PathScheme) } func TestIterativeStateSyncIndividualFromDisk(t *testing.T) { - testIterativeStateSync(t, 1, true, false) + testIterativeStateSync(t, 1, true, false, rawdb.HashScheme) + testIterativeStateSync(t, 1, true, false, rawdb.PathScheme) } func TestIterativeStateSyncBatchedFromDisk(t *testing.T) { - testIterativeStateSync(t, 100, true, false) + testIterativeStateSync(t, 100, true, false, rawdb.HashScheme) + testIterativeStateSync(t, 100, true, false, rawdb.PathScheme) } func TestIterativeStateSyncIndividualByPath(t *testing.T) { - testIterativeStateSync(t, 1, false, true) + testIterativeStateSync(t, 1, false, true, rawdb.HashScheme) + testIterativeStateSync(t, 1, false, true, rawdb.PathScheme) } func TestIterativeStateSyncBatchedByPath(t *testing.T) { - testIterativeStateSync(t, 100, false, true) + testIterativeStateSync(t, 100, false, true, rawdb.HashScheme) + testIterativeStateSync(t, 100, false, true, rawdb.PathScheme) } // stateElement represents the element in the state trie(bytecode or trie node). @@ -171,17 +200,17 @@ type stateElement struct { syncPath trie.SyncPath } -func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { +func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, scheme string) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + _, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) if commit { - srcDb.TrieDB().Commit(srcRoot, false) + ndb.Commit(srcRoot, false) } - srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB()) + srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), ndb) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, ndb.Scheme()) var ( nodeElements []stateElement @@ -196,9 +225,11 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { }) } for i := 0; i < len(codes); i++ { - codeElements = append(codeElements, stateElement{ - code: codes[i], - }) + codeElements = append(codeElements, stateElement{code: codes[i]}) + } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("failed to create reader for root %x: %v", srcRoot, err) } for len(nodeElements)+len(codeElements) > 0 { var ( @@ -225,7 +256,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil { t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) } - stTrie, err := trie.New(trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root), srcDb.TrieDB()) + stTrie, err := trie.New(trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root), ndb) if err != nil { t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) } @@ -236,7 +267,9 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } } else { - data, err := srcDb.TrieDB().Node(node.hash) + owner, inner := trie.ResolvePath([]byte(node.path)) + + data, err := reader.Node(owner, inner, node.hash) if err != nil { t.Fatalf("failed to retrieve node data for key %v", []byte(node.path)) } @@ -276,18 +309,23 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { } } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned, and the others sent only later. func TestIterativeDelayedStateSync(t *testing.T) { + testIterativeDelayedStateSync(t, rawdb.HashScheme) + testIterativeDelayedStateSync(t, rawdb.PathScheme) +} + +func testIterativeDelayedStateSync(t *testing.T, scheme string) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + _, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, ndb.Scheme()) var ( nodeElements []stateElement @@ -302,10 +340,13 @@ func TestIterativeDelayedStateSync(t *testing.T) { }) } for i := 0; i < len(codes); i++ { - codeElements = append(codeElements, stateElement{ - code: codes[i], - }) + codeElements = append(codeElements, stateElement{code: codes[i]}) + } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("failed to create reader for root %x: %v", srcRoot, err) } + for len(nodeElements)+len(codeElements) > 0 { // Sync only half of the scheduled nodes var nodeProcessd int @@ -330,7 +371,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1) for i, element := range nodeElements[:len(nodeResults)] { owner, inner := trie.ResolvePath([]byte(element.path)) - data, err := srcDb.TrieDB().Reader(srcRoot).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve contract bytecode for %x", element.code) } @@ -366,22 +407,28 @@ func TestIterativeDelayedStateSync(t *testing.T) { } } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that given a root hash, a trie can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go, however in a // random order. -func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) } -func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) } +func TestIterativeRandomStateSyncIndividual(t *testing.T) { + testIterativeRandomStateSync(t, 1, rawdb.HashScheme) + testIterativeRandomStateSync(t, 1, rawdb.PathScheme) +} +func TestIterativeRandomStateSyncBatched(t *testing.T) { + testIterativeRandomStateSync(t, 100, rawdb.HashScheme) + testIterativeRandomStateSync(t, 100, rawdb.PathScheme) +} -func testIterativeRandomStateSync(t *testing.T, count int) { +func testIterativeRandomStateSync(t *testing.T, count int, scheme string) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + _, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, ndb.Scheme()) nodeQueue := make(map[string]stateElement) codeQueue := make(map[common.Hash]struct{}) @@ -396,6 +443,10 @@ func testIterativeRandomStateSync(t *testing.T, count int) { for _, hash := range codes { codeQueue[hash] = struct{}{} } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("failed to create reader for root %x: %v", srcRoot, err) + } for len(nodeQueue)+len(codeQueue) > 0 { // Fetch all the queued nodes in a random order if len(codeQueue) > 0 { @@ -417,7 +468,7 @@ func testIterativeRandomStateSync(t *testing.T, count int) { results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) for path, element := range nodeQueue { owner, inner := trie.ResolvePath([]byte(element.path)) - data, err := srcDb.TrieDB().Reader(srcRoot).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path) } @@ -451,18 +502,23 @@ func testIterativeRandomStateSync(t *testing.T, count int) { } } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned (Even those randomly), others sent only later. func TestIterativeRandomDelayedStateSync(t *testing.T) { + testIterativeRandomDelayedStateSync(t, rawdb.HashScheme) + testIterativeRandomDelayedStateSync(t, rawdb.PathScheme) +} + +func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + _, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil, ndb.Scheme()) nodeQueue := make(map[string]stateElement) codeQueue := make(map[common.Hash]struct{}) @@ -477,6 +533,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { for _, hash := range codes { codeQueue[hash] = struct{}{} } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("failed to create reader for root %x: %v", srcRoot, err) + } for len(nodeQueue)+len(codeQueue) > 0 { // Sync only half of the scheduled nodes, even those in random order if len(codeQueue) > 0 { @@ -506,7 +566,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { delete(nodeQueue, path) owner, inner := trie.ResolvePath([]byte(element.path)) - data, err := srcDb.TrieDB().Reader(srcRoot).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x", element.hash) } @@ -542,14 +602,19 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { } } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that at any point in time during a sync, only complete sub-tries are in // the database. func TestIncompleteStateSync(t *testing.T) { + testIncompleteStateSync(t, rawdb.HashScheme) + testIncompleteStateSync(t, rawdb.PathScheme) +} + +func testIncompleteStateSync(t *testing.T, scheme string) { // Create a random state to copy - db, srcDb, srcRoot, srcAccounts := makeTestState() + db, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // isCodeLookup to save some hashing var isCode = make(map[common.Hash]struct{}) @@ -559,7 +624,7 @@ func TestIncompleteStateSync(t *testing.T) { } } isCode[common.BytesToHash(emptyCodeHash)] = struct{}{} - checkTrieConsistency(db, srcRoot) + checkTrieConsistency(db, ndb.Scheme(), srcRoot) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -570,6 +635,10 @@ func TestIncompleteStateSync(t *testing.T) { addedPaths []string addedHashes []common.Hash ) + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not available %x", srcRoot) + } nodeQueue := make(map[string]stateElement) codeQueue := make(map[common.Hash]struct{}) paths, nodes, codes := sched.Missing(1) @@ -583,6 +652,9 @@ func TestIncompleteStateSync(t *testing.T) { for _, hash := range codes { codeQueue[hash] = struct{}{} } + if err != nil { + t.Fatalf("failed to create reader for root %x: %v", srcRoot, err) + } for len(nodeQueue)+len(codeQueue) > 0 { // Fetch a batch of state nodes if len(codeQueue) > 0 { @@ -607,7 +679,7 @@ func TestIncompleteStateSync(t *testing.T) { results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) for path, element := range nodeQueue { owner, inner := trie.ResolvePath([]byte(element.path)) - data, err := srcDb.TrieDB().Reader(srcRoot).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x", element.hash) } @@ -635,7 +707,7 @@ func TestIncompleteStateSync(t *testing.T) { for _, root := range nodehashes { // Can't use checkStateConsistency here because subtrie keys may have odd // length and crash in LeafKey. - if err := checkTrieConsistency(dstDb, root); err != nil { + if err := checkTrieConsistency(dstDb, scheme, root); err != nil { t.Fatalf("state inconsistent: %v", err) } } @@ -658,12 +730,12 @@ func TestIncompleteStateSync(t *testing.T) { for _, node := range addedCodes { val := rawdb.ReadCode(dstDb, node) rawdb.DeleteCode(dstDb, node) - if err := checkStateConsistency(dstDb, srcRoot); err == nil { + if err := checkStateConsistency(dstDb, ndb.Scheme(), srcRoot); err == nil { t.Errorf("trie inconsistency not caught, missing: %x", node) } rawdb.WriteCode(dstDb, node, val) } - scheme := srcDb.TrieDB().Scheme() + for i, path := range addedPaths { owner, inner := trie.ResolvePath([]byte(path)) hash := addedHashes[i] @@ -672,8 +744,10 @@ func TestIncompleteStateSync(t *testing.T) { t.Error("missing trie node") } rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme) - if err := checkStateConsistency(dstDb, srcRoot); err == nil { + + if err := checkStateConsistency(dstDb, ndb.Scheme(), srcRoot); err == nil { t.Errorf("trie inconsistency not caught, missing: %v", path) + } rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme) } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 30fdf55606..c4556bd622 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -106,7 +106,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -241,7 +241,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -281,7 +281,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -335,7 +335,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) tooBigInitCode = [params.MaxInitCodeSize + 1]byte{} smallInitCode = [320]byte{} @@ -398,7 +398,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, newDbConfig(rawdb.HashScheme))) blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -496,7 +496,8 @@ func TestBlobTxStateTransition(t *testing.T) { Alloc: GenesisAlloc{addr: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) + triedb = trie.NewDatabase(gendb, nil) + genesis = gspec.MustCommit(gendb, triedb) signer = types.LatestSigner(gspec.Config) ) gspec.Config.ConsortiumV2Block = common.Big0 @@ -574,7 +575,8 @@ func TestBaseFee(t *testing.T) { Alloc: GenesisAlloc{addr: {Balance: big.NewInt(int64(initialFund))}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) + triedb = trie.NewDatabase(gendb, nil) + genesis = gspec.MustCommit(gendb, triedb) signer = types.LatestSigner(gspec.Config) ) gspec.Config.ConsortiumV2Block = common.Big0 diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index 44726c9cbb..b9df0c1e84 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -39,7 +39,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -86,7 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) } }) @@ -107,7 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -135,7 +135,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) diff --git a/core/vote/vote_pool_test.go b/core/vote/vote_pool_test.go index eba91b5f61..33768cfa76 100644 --- a/core/vote/vote_pool_test.go +++ b/core/vote/vote_pool_test.go @@ -15,6 +15,7 @@ import ( wallet "github.com/ethereum/go-ethereum/accounts/bls" "github.com/ethereum/go-ethereum/crypto/bls" + "github.com/ethereum/go-ethereum/trie" "github.com/google/uuid" keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" @@ -99,7 +100,7 @@ func testVotePool(t *testing.T, isValidRules bool) { Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) mux := new(event.TypeMux) @@ -394,7 +395,7 @@ func TestVotePoolDosProtection(t *testing.T) { Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) bs, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 25, nil, true) @@ -510,7 +511,7 @@ func TestVotePoolWrongTargetNumber(t *testing.T) { Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) bs, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1, nil, true) diff --git a/eth/api.go b/eth/api.go index d6333770f0..2d00595b6a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -468,7 +468,12 @@ func (api *PrivateDebugAPI) StorageRangeAt(ctx context.Context, blockHash common } func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeResult, error) { - it := trie.NewIterator(st.NodeIterator(start)) + + trieIt, err := st.NodeIterator(start) + if err != nil { + return StorageRangeResult{}, err + } + it := trie.NewIterator(trieIt) result := StorageRangeResult{Storage: storageMap{}} for i := 0; i < maxResult && it.Next(); i++ { _, content, _, err := rlp.Split(it.Value) @@ -549,7 +554,7 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } - triedb := api.eth.BlockChain().StateCache().TrieDB() + triedb := api.eth.BlockChain().TrieDB() oldTrie, err := trie.NewSecure(trie.StateTrieID(startBlock.Root()), triedb) if err != nil { @@ -559,7 +564,15 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc if err != nil { return nil, err } - diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{})) + oldIt, err := oldTrie.NodeIterator([]byte{}) + if err != nil { + return nil, err + } + newIt, err := newTrie.NodeIterator([]byte{}) + if err != nil { + return nil, err + } + diff, _ := trie.NewDifferenceIterator(oldIt, newIt) iter := trie.NewIterator(diff) var dirty []common.Address diff --git a/eth/api_backend.go b/eth/api_backend.go index 93637e7c17..b7e4b1c2ae 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -417,7 +417,7 @@ func (b *EthAPIBackend) StartMining(threads int) error { } func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { - return b.eth.StateAtBlock(ctx, block, reexec, base, checkLive, preferDisk) + return b.eth.stateAtBlock(ctx, block, reexec, base, checkLive, preferDisk) } func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { diff --git a/eth/api_test.go b/eth/api_test.go index 86812fce5b..01b5d1b2b3 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/trie" ) @@ -67,34 +68,34 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) - state, _ = state.New(common.Hash{}, statedb, nil) - addrs = [AccountRangeMaxResults * 2]common.Address{} - m = map[common.Address]bool{} + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) + sdb, _ = state.New(types.EmptyRootHash, statedb, nil) + addrs = [AccountRangeMaxResults * 2]common.Address{} + m = map[common.Address]bool{} ) for i := range addrs { hash := common.HexToHash(fmt.Sprintf("%x", i)) addr := common.BytesToAddress(crypto.Keccak256Hash(hash.Bytes()).Bytes()) addrs[i] = addr - state.SetBalance(addrs[i], big.NewInt(1)) + sdb.SetBalance(addrs[i], big.NewInt(1)) if _, ok := m[addr]; ok { t.Fatalf("bad") } else { m[addr] = true } } - state.Commit(0, true) - root := state.IntermediateRoot(true) + root, _ := sdb.Commit(0, true) + sdb, _ = state.New(root, statedb, nil) trie, err := statedb.OpenTrie(root) if err != nil { t.Fatal(err) } - accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2) + accountRangeTest(t, &trie, sdb, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2) // test pagination - firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults) - secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) + firstResult := accountRangeTest(t, &trie, sdb, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults) + secondResult := accountRangeTest(t, &trie, sdb, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) hList := make(resultHash, 0) for addr1 := range firstResult.Accounts { @@ -112,7 +113,7 @@ func TestAccountRange(t *testing.T) { // set and get an even split between the first and second sets. sort.Sort(hList) middleH := hList[AccountRangeMaxResults/2] - middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults) + middleResult := accountRangeTest(t, &trie, sdb, middleH, AccountRangeMaxResults, AccountRangeMaxResults) missing, infirst, insecond := 0, 0, 0 for h := range middleResult.Accounts { if _, ok := firstResult.Accounts[h]; ok { @@ -141,8 +142,9 @@ func TestEmptyAccountRange(t *testing.T) { statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) st, _ = state.New(common.Hash{}, statedb, nil) ) + // Commit(although nothing to flush) and re-init the statedb st.Commit(0, true) - st.IntermediateRoot(true) + st, _ = state.New(types.EmptyRootHash, statedb, nil) results := st.IteratorDump(&state.DumpConfig{ SkipCode: true, SkipStorage: true, diff --git a/eth/backend.go b/eth/backend.go index 3850b94458..12cdafeb3d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -141,8 +141,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil { - log.Error("Failed to recover state", "error", err) + // Recover the pruning data only in hash scheme + if config.StateScheme == rawdb.HashScheme { + if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil { + log.Error("Failed to recover state", "error", err) + } } eth := &Ethereum{ config: config, @@ -199,9 +202,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, TriesInMemory: config.TriesInMemory, + StateHistory: config.StateHistory, + StateScheme: config.StateScheme, } ) - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, config.OverrideArrowGlacier, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, config.OverrideArrowGlacier, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory) chainConfig := eth.blockchain.Config() genesisHash := eth.blockchain.Genesis().Hash() if err != nil { @@ -599,7 +604,7 @@ func (s *Ethereum) StartMining(threads int) error { } // If mining is started, we can disable the transaction rejection mechanism // introduced to speed sync times. - atomic.StoreUint32(&s.handler.acceptTxs, 1) + s.handler.enableSyncedFeatures() go s.miner.Start(eb) } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index bc66329217..990a1d30ba 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -57,7 +58,7 @@ func generateTestChain() (*core.Genesis, []*types.Block) { } gblock := genesis.ToBlock() engine := ethash.NewFaker() - genesis.MustCommit(db) + genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) blocks, _ := core.GenerateChain(config, gblock, engine, db, 10, generate, true) blocks = append([]*types.Block{gblock}, blocks...) return genesis, blocks diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 49804de0fa..8c660d0e86 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -88,7 +88,7 @@ func newTester() *downloadTester { } tester.stateDb = rawdb.NewMemoryDatabase() tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) - tester.triedb = trie.NewDatabase(tester.stateDb) + tester.triedb = trie.NewDatabase(tester.stateDb, nil) tester.downloader = New(0, tester.stateDb, trie.NewSyncBloom(1, tester.stateDb), new(event.TypeMux), tester, nil, tester.dropPeer, tester.verifyBlobHeader) return tester @@ -234,7 +234,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewSecure(trie.StateTrieID(block.Root()), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewSecure(trie.StateTrieID(block.Root()), trie.NewDatabase(dl.stateDb, nil)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 4e7f818135..c50ed367b0 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -298,7 +298,7 @@ type codeTask struct { func newStateSync(d *Downloader, root common.Hash) *stateSync { // Hack the node scheme here. It's a dead code is not used // by light client at all. Just aim for passing tests. - scheme := trie.NewDatabase(rawdb.NewMemoryDatabase()).Scheme() + scheme := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil).Scheme() return &stateSync{ d: d, root: root, diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 01ebca97a4..2f081a080f 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // Test chain parameters. @@ -40,7 +41,7 @@ var ( Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - testGenesis = testGspec.MustCommit(testDB) + testGenesis = testGspec.MustCommit(testDB, trie.NewDatabase(testDB, nil)) ) // The common prefix of all test chains: diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index ac257d8a68..d0e8d22093 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/downloader" @@ -78,6 +79,9 @@ var Defaults = Config{ }, NetworkId: 1, TxLookupLimit: 2350000, + TransactionHistory: 2350000, + StateHistory: params.FullImmutabilityThreshold, + StateScheme: rawdb.HashScheme, LightPeers: 100, UltraLightFraction: 75, DatabaseCache: 512, @@ -141,8 +145,13 @@ type Config struct { NoPruning bool // Whether to disable pruning and flush everything to disk NoPrefetch bool // Whether to disable prefetching and only load state on demand + // Deprecated, use 'TransactionHistory' instead. TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. + StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top + // Whitelist of required block number -> hash values to accept Whitelist map[uint64]common.Hash `toml:"-"` diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 0af7eeced0..3f86095dae 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -27,6 +27,9 @@ func (c Config) MarshalTOML() (interface{}, error) { NoPruning bool NoPrefetch bool TxLookupLimit uint64 `toml:",omitempty"` + TransactionHistory uint64 `toml:",omitempty"` + StateHistory uint64 `toml:",omitempty"` + StateScheme string `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` LightIngress int `toml:",omitempty"` @@ -69,6 +72,9 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit + enc.TransactionHistory = c.TransactionHistory + enc.StateHistory = c.StateHistory + enc.StateScheme = c.StateScheme enc.Whitelist = c.Whitelist enc.LightServ = c.LightServ enc.LightIngress = c.LightIngress @@ -115,6 +121,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NoPruning *bool NoPrefetch *bool TxLookupLimit *uint64 `toml:",omitempty"` + TransactionHistory *uint64 `toml:",omitempty"` + StateHistory *uint64 `toml:",omitempty"` + StateScheme *string `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` LightIngress *int `toml:",omitempty"` @@ -176,6 +185,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } + if dec.TransactionHistory != nil { + c.TransactionHistory = *dec.TransactionHistory + } + if dec.StateHistory != nil { + c.StateHistory = *dec.StateHistory + } + if dec.StateScheme != nil { + c.StateScheme = *dec.StateScheme + } if dec.Whitelist != nil { c.Whitelist = dec.Whitelist } diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index ebb025d9a5..bfe5269b11 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -42,7 +42,7 @@ var ( Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(testdb) + genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, nil)) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index d30c9f9592..9aa34d9a98 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -164,7 +165,7 @@ func TestFinalizedBlockSubscription(t *testing.T) { db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} api = NewPublicFilterAPI(backend, false, deadline) - genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, nil)) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}, true) chainEvents = []core.ChainEvent{} ) @@ -225,7 +226,7 @@ func TestBlockSubscription(t *testing.T) { db = rawdb.NewMemoryDatabase() backend = &testBackend{db: db} api = NewPublicFilterAPI(backend, false, deadline) - genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db, trie.NewDatabase(db, nil)) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}, true) chainEvents = []core.ChainEvent{} ) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index e79b3dc9a0..4cd9961006 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) func makeReceipt(addr common.Address) *types.Receipt { @@ -65,7 +66,7 @@ func BenchmarkFilters(b *testing.B) { ) defer db.Close() - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 100010, func(i int, gen *core.BlockGen) { switch i { @@ -132,7 +133,7 @@ func TestFilters(t *testing.T) { ) defer db.Close() - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { switch i { diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index e35f2eab20..cc3ca5a04e 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) const testHead = 32 @@ -110,7 +111,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke config.ArrowGlacierBlock = londonBlock engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) // Generate testing blocks blocks, _ := core.GenerateChain(gspec.Config, genesis, engine, db, testHead+1, func(i int, b *core.BlockGen) { @@ -141,7 +142,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke }, true) // Construct testing chain diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) + gspec.MustCommit(diskdb, trie.NewDatabase(diskdb, nil)) chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create local chain, %v", err) diff --git a/eth/handler.go b/eth/handler.go index 29efa79309..c6353e650e 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vote" @@ -44,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "golang.org/x/crypto/sha3" ) @@ -241,7 +243,7 @@ func newHandler(config *handlerConfig) (*handler, error) { } n, err := h.chain.InsertChain(blocks, sidecars) if err == nil { - atomic.StoreUint32(&h.acceptTxs, 1) // Mark initial sync done on any fetcher import + h.enableSyncedFeatures() // Mark initial sync done on any fetcher import } return n, err } @@ -700,3 +702,12 @@ func (h *handler) voteBroadcastLoop() { } } } + +// enableSyncedFeatures enables the post-sync functionalities when the initial +// sync is finished. +func (h *handler) enableSyncedFeatures() { + atomic.StoreUint32(&h.acceptTxs, 1) + if h.chain.TrieDB().Scheme() == rawdb.PathScheme { + h.chain.TrieDB().SetBufferSize(pathdb.DefaultBufferSize) + } +} diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index da85e829dc..73f16a4771 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -117,8 +117,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { gspecNoFork = &core.Genesis{Config: configNoFork} gspecProFork = &core.Genesis{Config: configProFork} - genesisNoFork = gspecNoFork.MustCommit(dbNoFork) - genesisProFork = gspecProFork.MustCommit(doFork) + genesisNoFork = gspecNoFork.MustCommit(dbNoFork, trie.NewDatabase(dbNoFork, nil)) + genesisProFork = gspecProFork.MustCommit(doFork, trie.NewDatabase(doFork, nil)) chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, gspecNoFork, nil, engine, vm.Config{}, nil, nil) chainProFork, _ = core.NewBlockChain(doFork, nil, gspecProFork, nil, engine, vm.Config{}, nil, nil) diff --git a/eth/handler_test.go b/eth/handler_test.go index 137cd52e97..58dc896a37 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" ) @@ -147,7 +148,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, } - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) @@ -222,7 +223,7 @@ func newTestHandlerWithBlocks100(blocks int) (*testHandler, []*types.BlobTxSidec }, }, } - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) chain, err := core.NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { panic(err) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index a038316347..56f465774e 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -70,7 +70,7 @@ func newTestBackendWithGenerator(blocks int, generator func(int, *core.BlockGen) Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, } - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 8444f56be3..95743577fb 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -202,6 +203,10 @@ func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { } func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer) [][]byte { + // Request nodes by hash is not supported in path-based scheme. + if backend.Chain().TrieDB().Scheme() == rawdb.PathScheme { + return nil + } // Gather state data until the fetch or network limits is reached var ( bytes int @@ -217,7 +222,7 @@ func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer // Only lookup the trie node if there's chance that we actually have it continue } - entry, err := backend.Chain().TrieNode(hash) + entry, err := backend.Chain().TrieDB().Node(hash) if len(entry) == 0 || err != nil { // Read the contract code with prefix only to save unnecessary lookups. entry, err = backend.Chain().ContractCodeWithPrefix(hash) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 235340fa96..0cc6f0d1de 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -165,7 +165,7 @@ func handleMessage(backend Backend, peer *Peer) error { req.Bytes = softResponseLimit } // Retrieve the requested state and bail out if non existent - tr, err := trie.New(trie.StateTrieID(req.Root), backend.Chain().StateCache().TrieDB()) + tr, err := trie.New(trie.StateTrieID(req.Root), backend.Chain().TrieDB()) if err != nil { return p2p.Send(peer.rw, AccountRangeMsg, &AccountRangePacket{ID: req.ID}) } @@ -315,7 +315,7 @@ func handleMessage(backend Backend, peer *Peer) error { if origin != (common.Hash{}) || abort { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.New(trie.StateTrieID(req.Root), backend.Chain().StateCache().TrieDB()) + accTrie, err := trie.New(trie.StateTrieID(req.Root), backend.Chain().TrieDB()) if err != nil { return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) } @@ -323,7 +323,7 @@ func handleMessage(backend Backend, peer *Peer) error { if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) } - stTrie, err := trie.New(trie.StorageTrieID(req.Root, account, acc.Root), backend.Chain().StateCache().TrieDB()) + stTrie, err := trie.New(trie.StorageTrieID(req.Root, account, acc.Root), backend.Chain().TrieDB()) if err != nil { return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) } @@ -428,7 +428,7 @@ func handleMessage(backend Backend, peer *Peer) error { req.Bytes = softResponseLimit } // Make sure we have the state associated with the request - triedb := backend.Chain().StateCache().TrieDB() + triedb := backend.Chain().TrieDB() accTrie, err := trie.NewSecure(trie.StateTrieID(req.Root), triedb) if err != nil { diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index ecfc76fe9e..fcf3373740 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" ) @@ -560,6 +561,11 @@ func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Has // the remote side does not do any follow-up requests func TestSyncBloatedProof(t *testing.T) { t.Parallel() + testSyncBloatedProof(t, rawdb.HashScheme) + testSyncBloatedProof(t, rawdb.PathScheme) +} + +func testSyncBloatedProof(t *testing.T, scheme string) { var ( once sync.Once @@ -571,7 +577,7 @@ func TestSyncBloatedProof(t *testing.T) { } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 100) source := newTestPeer("source", t, term) source.accountTrie = sourceAccountTrie source.accountValues = elems @@ -639,6 +645,11 @@ func setupSyncer(scheme string, peers ...*testPeer) *Syncer { func TestSync(t *testing.T) { t.Parallel() + testSync(t, rawdb.HashScheme) + testSync(t, rawdb.PathScheme) +} + +func testSync(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -648,7 +659,7 @@ func TestSync(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 100) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -660,14 +671,18 @@ func TestSync(t *testing.T) { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncTinyTriePanic tests a basic sync with one peer, and a tiny trie. This caused a // panic within the prover func TestSyncTinyTriePanic(t *testing.T) { t.Parallel() + testSyncTinyTriePanic(t, rawdb.HashScheme) + testSyncTinyTriePanic(t, rawdb.PathScheme) +} +func testSyncTinyTriePanic(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -677,7 +692,7 @@ func TestSyncTinyTriePanic(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(1) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 1) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -691,13 +706,17 @@ func TestSyncTinyTriePanic(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSync tests a basic sync with multiple peers func TestMultiSync(t *testing.T) { t.Parallel() + testMultiSync(t, rawdb.HashScheme) + testMultiSync(t, rawdb.PathScheme) +} +func testMultiSync(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -707,7 +726,7 @@ func TestMultiSync(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 100) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -721,13 +740,19 @@ func TestMultiSync(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorage tests basic sync using accounts + storage + code func TestSyncWithStorage(t *testing.T) { t.Parallel() + testSyncWithStorage(t, rawdb.HashScheme) + testSyncWithStorage(t, rawdb.PathScheme) +} + +func testSyncWithStorage(t *testing.T, scheme string) { + var ( once sync.Once cancel = make(chan struct{}) @@ -737,7 +762,8 @@ func TestSyncWithStorage(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(3, 3000, true, false) + // Create 3 accounts with 3000 storage slots each, code is true, and boundary is false. + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 3, 3000, true, false) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -753,13 +779,17 @@ func TestSyncWithStorage(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all func TestMultiSyncManyUseless(t *testing.T) { t.Parallel() + testMultiSyncManyUseless(t, rawdb.HashScheme) + testMultiSyncManyUseless(t, rawdb.PathScheme) +} +func testMultiSyncManyUseless(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -769,7 +799,7 @@ func TestMultiSyncManyUseless(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -801,11 +831,17 @@ func TestMultiSyncManyUseless(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { + t.Parallel() + testMultiSyncManyUselessWithLowTimeout(t, rawdb.HashScheme) + testMultiSyncManyUselessWithLowTimeout(t, rawdb.PathScheme) +} + +func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -815,7 +851,7 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -853,11 +889,17 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all func TestMultiSyncManyUnresponsive(t *testing.T) { + t.Parallel() + testMultiSyncManyUnresponsive(t, rawdb.HashScheme) + testMultiSyncManyUnresponsive(t, rawdb.PathScheme) +} + +func testMultiSyncManyUnresponsive(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -867,7 +909,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -888,7 +930,8 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { return source } - syncer := setupSyncer(nodeScheme, + syncer := setupSyncer( + nodeScheme, mkSource("full", true, true, true), mkSource("noAccounts", false, true, true), mkSource("noStorage", true, false, true), @@ -902,7 +945,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } func checkStall(t *testing.T, term func()) chan struct{} { @@ -923,7 +966,11 @@ func checkStall(t *testing.T, term func()) chan struct{} { // account trie has a few boundary elements. func TestSyncBoundaryAccountTrie(t *testing.T) { t.Parallel() + testSyncBoundaryAccountTrie(t, rawdb.HashScheme) + testSyncBoundaryAccountTrie(t, rawdb.PathScheme) +} +func testSyncBoundaryAccountTrie(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -933,7 +980,7 @@ func TestSyncBoundaryAccountTrie(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(3000) + nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(scheme, 3000) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -950,14 +997,18 @@ func TestSyncBoundaryAccountTrie(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCappedPeer tests sync using accounts and no storage, where one peer is // consistently returning very small results func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneCappedPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneCappedPeer(t, rawdb.PathScheme) +} +func testSyncNoStorageAndOneCappedPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -967,7 +1018,7 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 3000) mkSource := func(name string, slow bool) *testPeer { source := newTestPeer(name, t, term) @@ -991,14 +1042,18 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCodeCorruptPeer has one peer which doesn't deliver // code requests properly. func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.PathScheme) +} +func testSyncNoStorageAndOneCodeCorruptPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1008,7 +1063,7 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 3000) mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1030,12 +1085,16 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.PathScheme) +} +func testSyncNoStorageAndOneAccountCorruptPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1045,7 +1104,7 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 3000) mkSource := func(name string, accFn accountHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1067,14 +1126,18 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCodeCappedPeer has one peer which delivers code hashes // one by one func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.PathScheme) +} +func testSyncNoStorageAndOneCodeCappedPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1084,7 +1147,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 3000) mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1116,14 +1179,18 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { if threshold := 100; counter > threshold { t.Fatalf("Error, expected < %d invocations, got %d", threshold, counter) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncBoundaryStorageTrie tests sync against a few normal peers, but the // storage trie has a few boundary elements. func TestSyncBoundaryStorageTrie(t *testing.T) { t.Parallel() + testSyncBoundaryStorageTrie(t, rawdb.HashScheme) + testSyncBoundaryStorageTrie(t, rawdb.PathScheme) +} +func testSyncBoundaryStorageTrie(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1133,7 +1200,7 @@ func TestSyncBoundaryStorageTrie(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(10, 1000, false, true) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 10, 1000, false, true) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -1152,14 +1219,18 @@ func TestSyncBoundaryStorageTrie(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorageAndOneCappedPeer tests sync using accounts + storage, where one peer is // consistently returning very small results func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { t.Parallel() + testSyncWithStorageAndOneCappedPeer(t, rawdb.HashScheme) + testSyncWithStorageAndOneCappedPeer(t, rawdb.PathScheme) +} +func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1169,7 +1240,7 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(300, 1000, false, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 300, 1000, false, false) mkSource := func(name string, slow bool) *testPeer { source := newTestPeer(name, t, term) @@ -1193,14 +1264,19 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorageAndCorruptPeer tests sync using accounts + storage, where one peer is // sometimes sending bad proofs func TestSyncWithStorageAndCorruptPeer(t *testing.T) { t.Parallel() + testSyncWithStorageAndCorruptPeer(t, rawdb.HashScheme) + testSyncWithStorageAndCorruptPeer(t, rawdb.PathScheme) +} + +func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1210,7 +1286,7 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1233,12 +1309,16 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { t.Parallel() + testSyncWithStorageAndNonProvingPeer(t, rawdb.HashScheme) + testSyncWithStorageAndNonProvingPeer(t, rawdb.PathScheme) +} +func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1248,7 +1328,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1270,7 +1350,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorage tests basic sync using accounts + storage + code, against @@ -1279,6 +1359,11 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { // did not mark the account for healing. func TestSyncWithStorageMisbehavingProve(t *testing.T) { t.Parallel() + testSyncWithStorageMisbehavingProve(t, rawdb.HashScheme) + testSyncWithStorageMisbehavingProve(t, rawdb.PathScheme) +} + +func testSyncWithStorageMisbehavingProve(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1288,7 +1373,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(10, 30, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(scheme, 10, 30, false) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -1303,7 +1388,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } type kv struct { @@ -1356,10 +1441,10 @@ func getCodeByHash(hash common.Hash) []byte { } // makeAccountTrieNoStorage spits out a trie, along with the leafs -func makeAccountTrieNoStorage(n int) (string, *trie.Trie, entrySlice) { +func makeAccountTrieNoStorage(scheme string, n int) (string, *trie.Trie, entrySlice) { // Create emptry Trie var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries entrySlice ) @@ -1391,12 +1476,12 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, entrySlice) { // makeBoundaryAccountTrie constructs an account trie. Instead of filling // accounts normally, this function will fill a few accounts which have // boundary hash. -func makeBoundaryAccountTrie(n int) (string, *trie.Trie, entrySlice) { +func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, entrySlice) { var ( entries entrySlice boundaries []common.Hash - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) ) // Initialize boundaries @@ -1451,9 +1536,9 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, entrySlice) { // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts // has a unique storage set. Code is true when u pass a random code hash to the account -func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { +func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots int, code bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries entrySlice storageRoots = make(map[common.Hash]common.Hash) @@ -1504,9 +1589,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) } // makeAccountTrieWithStorage spits out a trie, along with the leafs -func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { +func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries entrySlice storageRoots = make(map[common.Hash]common.Hash) @@ -1562,7 +1647,8 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin } for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) - trie, err := trie.New(trie.StorageTrieID(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)]), db) + id := trie.StorageTrieID(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)]) + trie, err := trie.New(id, db) if err != nil { panic(err) } @@ -1575,7 +1661,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trienode.NodeSet, entrySlice) { - trie, _ := trie.New(trie.StorageTrieID(common.Hash{}, owner, common.Hash{}), db) + trie, _ := trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) var entries entrySlice for i := uint64(1); i <= n; i++ { // store 'x' at slot 'x' @@ -1601,7 +1687,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo var ( entries entrySlice boundaries []common.Hash - trie, _ = trie.New(trie.StorageTrieID(common.Hash{}, owner, common.Hash{}), db) + trie, _ = trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) ) // Initialize boundaries var next common.Hash @@ -1645,15 +1731,15 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo return root, nodes, entries } -func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { +func verifyTrie(scheme string, db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() - triedb := trie.NewDatabase(rawdb.NewDatabase(db)) + triedb := trie.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme)) accTrie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { t.Fatal(err) } accounts, slots := 0, 0 - accIt := trie.NewIterator(accTrie.NodeIterator(nil)) + accIt := trie.NewIterator(accTrie.MustNodeIterator(nil)) for accIt.Next() { var acc struct { Nonce uint64 @@ -1665,12 +1751,14 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { log.Crit("Invalid account encountered during snapshot creation", "err", err) } accounts++ - if acc.Root != emptyRoot { - storeTrie, err := trie.NewSecure(trie.StorageTrieID(root, common.BytesToHash(accIt.Key), acc.Root), triedb) + if acc.Root != types.EmptyRootHash { + id := trie.StorageTrieID(root, common.BytesToHash(accIt.Key), acc.Root) + storeTrie, err := trie.NewSecure(id, triedb) if err != nil { t.Fatal(err) } - storeIt := trie.NewIterator(storeTrie.NodeIterator(nil)) + + storeIt := trie.NewIterator(storeTrie.MustNodeIterator(nil)) for storeIt.Next() { slots++ } @@ -1688,6 +1776,12 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { // TestSyncAccountPerformance tests how efficient the snap algo is at minimizing // state healing func TestSyncAccountPerformance(t *testing.T) { + t.Parallel() + testSyncAccountPerformance(t, rawdb.HashScheme) + testSyncAccountPerformance(t, rawdb.PathScheme) +} + +func testSyncAccountPerformance(t *testing.T, scheme string) { // Set the account concurrency to 1. This _should_ result in the // range root to become correct, and there should be no healing needed defer func(old int) { accountConcurrency = old }(accountConcurrency) @@ -1702,7 +1796,7 @@ func TestSyncAccountPerformance(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(scheme, 100) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -1715,7 +1809,7 @@ func TestSyncAccountPerformance(t *testing.T) { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) // The trie root will always be requested, since it is added when the snap // sync cycle starts. When popping the queue, we do not look it up again. // Doing so would bring this number down to zero in this artificial testcase, @@ -1775,3 +1869,10 @@ func TestSlotEstimation(t *testing.T) { } } } + +func newDbConfig(scheme string) *trie.Config { + if scheme == rawdb.HashScheme { + return &trie.Config{} + } + return &trie.Config{PathDB: pathdb.Defaults} +} diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 06bbfdbd46..c4cac850bf 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/consortium" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -38,38 +39,24 @@ import ( // for releasing state. var noopReleaser = tracers.StateReleaseFunc(func() {}) -// StateAtBlock retrieves the state database associated with a certain block. -// If no state is locally available for the given block, a number of blocks -// are attempted to be reexecuted to generate the desired state. The optional -// base layer statedb can be passed then it's regarded as the statedb of the -// parent block. -// Parameters: -// - block: The block for which we want the state (== state at the stateRoot of the parent) -// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state -// - base: If the caller is tracing multiple blocks, the caller can provide the parent state -// continuously from the callsite. -// - checklive: if true, then the live 'blockchain' state database is used. If the caller want to -// perform Commit or other 'save-to-disk' changes, this should be set to false to avoid -// storing trash persistently -// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is provided, -// it would be preferrable to start from a fresh state, if we have it on disk. -func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { +func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { var ( current *types.Block database state.Database + triedb *trie.Database report = true origin = block.NumberU64() ) // The state is only for reading purposes, check the state presence in // live database. - if checkLive { + if readOnly { // The state is available in live database, create a reference // on top to prevent garbage collection and return a release // function to deref it. - statedb, err = eth.blockchain.StateAt(block.Root()) - if err == nil { + if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil { + eth.blockchain.TrieDB().Reference(block.Root(), common.Hash{}) return statedb, func() { - statedb.Database().TrieDB().Dereference(block.Root()) + eth.blockchain.TrieDB().Dereference(block.Root()) }, nil } } @@ -80,27 +67,32 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe if preferDisk { // Create an ephemeral trie.Database for isolating the live one. Otherwise // the internal junks created by tracing will be persisted into the disk. - database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) + // TODO(rjl493456442), clean cache is disabled to prevent memory leak, + // please re-enable it for better performance. + database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults) if statedb, err = state.New(block.Root(), database, nil); err == nil { log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) return statedb, noopReleaser, nil } } // The optional base statedb is given, mark the start point as parent block - statedb, database, report = base, base.Database(), false + statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) } else { - // Otherwise try to reexec blocks until we find a state or reach our limit + // Otherwise, try to reexec blocks until we find a state or reach our limit current = block // Create an ephemeral trie.Database for isolating the live one. Otherwise // the internal junks created by tracing will be persisted into the disk. - database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) + // TODO(rjl493456442), clean cache is disabled to prevent memory leak, + // please re-enable it for better performance. + triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults) + database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb) // If we didn't check the live database, do check state over ephemeral database, // otherwise we would rewind past a persisted block (specific corner case is // chain tracing from the genesis). - if !checkLive { + if !readOnly { statedb, err = state.New(current.Root(), database, nil) if err == nil { return statedb, noopReleaser, nil @@ -108,6 +100,9 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe } // Database does not have the state for the given block, try to regenerate for i := uint64(0); i < reexec; i++ { + if err := ctx.Err(); err != nil { + return nil, nil, err + } if current.NumberU64() == 0 { return nil, nil, errors.New("genesis state is missing") } @@ -168,17 +163,58 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe } // Hold the state reference and also drop the parent state // to prevent accumulating too many nodes in memory. - database.TrieDB().Reference(root, common.Hash{}) + triedb.Reference(root, common.Hash{}) if parent != (common.Hash{}) { - database.TrieDB().Dereference(parent) + triedb.Dereference(parent) } parent = root } if report { - nodes, imgs := database.TrieDB().Size() + nodes, imgs := triedb.Size() log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) } - return statedb, func() { database.TrieDB().Dereference(block.Root()) }, nil + return statedb, func() { triedb.Dereference(block.Root()) }, nil +} + +func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) { + // Check if the requested state is available in the live chain. + statedb, err := eth.blockchain.StateAt(block.Root()) + if err == nil { + return statedb, noopReleaser, nil + } + // TODO historic state is not supported in path-based scheme. + // Fully archive node in pbss will be implemented by relying + // on state history, but needs more work on top. + return nil, nil, errors.New("historical state not available in path scheme yet") +} + +// stateAtBlock retrieves the state database associated with a certain block. +// If no state is locally available for the given block, a number of blocks +// are attempted to be reexecuted to generate the desired state. The optional +// base layer statedb can be provided which is regarded as the statedb of the +// parent block. +// +// An additional release function will be returned if the requested state is +// available. Release is expected to be invoked when the returned state is no +// longer needed. Its purpose is to prevent resource leaking. Though it can be +// noop in some cases. +// +// Parameters: +// - block: The block for which we want the state(state = block.Root) +// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state +// - base: If the caller is tracing multiple blocks, the caller can provide the parent +// state continuously from the callsite. +// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should +// be made from caller, e.g. perform Commit or other 'save-to-disk' changes. +// Otherwise, the trash generated by caller may be persisted permanently. +// - preferDisk: This arg can be used by the caller to signal that even though the 'base' is +// provided, it would be preferable to start from a fresh state, if we have it +// on disk. +func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { + if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme { + return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk) + } + return eth.pathState(block) } // stateAtTransaction returns the execution environment of a certain transaction. @@ -194,7 +230,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, } // Lookup the statedb of parent block from the live database, // otherwise regenerate it on the flight. - statedb, release, err := eth.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := eth.stateAtBlock(ctx, parent, reexec, nil, true, false) if err != nil { return nil, vm.BlockContext{}, nil, nil, err } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index c6d98e0704..0cc8a85cbe 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -85,7 +85,7 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i i TrieDirtyDisabled: true, // Archive mode } - _, _, genesisErr := core.SetupGenesisBlockWithOverride(backend.chaindb, trie.NewDatabase(backend.chaindb), gspec, nil, true) + _, _, genesisErr := core.SetupGenesisBlockWithOverride(backend.chaindb, trie.NewDatabase(backend.chaindb, nil), gspec, nil, true) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { t.Fatal(genesisErr.Error()) } diff --git a/eth/tracers/internal/tracetest/calltrace2_test.go b/eth/tracers/internal/tracetest/calltrace2_test.go index d0559e2804..c854828885 100644 --- a/eth/tracers/internal/tracetest/calltrace2_test.go +++ b/eth/tracers/internal/tracetest/calltrace2_test.go @@ -162,8 +162,9 @@ func testCallTracer2(tracerName string, dirPath string, t *testing.T) { Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) + defer triedb.Close() tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) @@ -265,7 +266,8 @@ func benchTracer2(tracerName string, test *callTracer2Test, b *testing.B) { Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer triedb.Close() b.ReportAllocs() b.ResetTimer() @@ -335,7 +337,8 @@ func TestZeroValueToNotExitCall2(t *testing.T) { Balance: big.NewInt(500000000000000), }, } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) + defer triedb.Close() // Create the tracer, the EVM environment and run it tracer, err := tracers.DefaultDirectory.New("callTracer2", new(tracers.Context), nil) if err != nil { diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 1b3c121a89..233c1eb8e7 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -137,8 +137,9 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) + triedb.Close() tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) @@ -237,7 +238,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer triedb.Close() b.ReportAllocs() b.ResetTimer() @@ -343,7 +345,7 @@ func TestInternals(t *testing.T) { want: `{"0x0000000000000000000000000000000000000000":{"balance":"0x0"},"0x000000000000000000000000000000000000feed":{"balance":"0x1c6bf52640350"},"0x00000000000000000000000000000000deadbeef":{"balance":"0x0","code":"0x6001600052600164ffffffffff60016000f560ff6000a0"}}`, }, } { - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), core.GenesisAlloc{ to: core.GenesisAccount{ Code: tc.code, @@ -351,7 +353,8 @@ func TestInternals(t *testing.T) { origin: core.GenesisAccount{ Balance: big.NewInt(500000000000000), }, - }, false) + }, false, rawdb.HashScheme) + defer triedb.Close() evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer}) msg := types.NewMessage(origin, &to, 0, big.NewInt(0), 50000, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, false, nil, nil) st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(msg.Gas())) diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 10cf1ecdcd..f2e9d741ff 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -100,7 +100,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer triedb.Close() // Create the tracer, the EVM environment and run it tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index db97c6e63d..579eb9fe71 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -108,8 +108,9 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) + defer triedb.Close() tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 277cb471c6..f08f156612 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -79,7 +79,9 @@ func BenchmarkTransactionTrace(b *testing.B) { Code: []byte{}, Balance: big.NewInt(500000000000000), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) + defer triedb.Close() + // Create the tracer, the EVM environment and run it tracer := logger.NewStructLogger(&logger.Config{ Debug: false, diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 0a33a593e3..3c63cab01d 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" ) @@ -287,7 +288,7 @@ func generateTestChain() ([]*types.Block, [][]*types.BlobTxSidecar, [][]common.H blobTxHashes = append(blobTxHashes, []common.Hash{}) } } - gblock := genesis.MustCommit(db) + gblock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(genesis.Config, gblock, engine, db, 2, generate, true) // add genesis blob/sidecars/txhash to the begining of the list @@ -724,7 +725,7 @@ func testGetBlobSidecars(t *testing.T, chain []*types.Block, blobSidecars [][]*t wantErr error }{ "first_block_blob_notfound_by_number": { - blkNum: chain[1].Number(), + blkNum: chain[1].Number(), }, "first_block_blob_notfound_by_hash": { blkHash: chain[1].Hash(), diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index a1f0cf385b..2a4954cfb1 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -82,7 +83,7 @@ func generateTestChain() (*core.Genesis, []*types.Block) { g.OffsetTime(5) g.SetExtra([]byte("test")) } - gblock := genesis.MustCommit(db) + gblock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, 1, generate, true) blocks = append([]*types.Block{gblock}, blocks...) diff --git a/les/client.go b/les/client.go index 6e8fcbc308..3ea07911f5 100644 --- a/les/client.go +++ b/les/client.go @@ -89,7 +89,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, trie.NewDatabase(chainDb), config.Genesis, config.OverrideArrowGlacier, false) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, trie.NewDatabase(chainDb, nil), config.Genesis, config.OverrideArrowGlacier, false) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index 963d4d9035..fe686ab407 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewSecure(trie.StateTrieID(block.Root()), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewSecure(trie.StateTrieID(block.Root()), trie.NewDatabase(dl.stateDb, nil)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) diff --git a/les/downloader/statesync.go b/les/downloader/statesync.go index 4e7f818135..c50ed367b0 100644 --- a/les/downloader/statesync.go +++ b/les/downloader/statesync.go @@ -298,7 +298,7 @@ type codeTask struct { func newStateSync(d *Downloader, root common.Hash) *stateSync { // Hack the node scheme here. It's a dead code is not used // by light client at all. Just aim for passing tests. - scheme := trie.NewDatabase(rawdb.NewMemoryDatabase()).Scheme() + scheme := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil).Scheme() return &stateSync{ d: d, root: root, diff --git a/les/downloader/testchain_test.go b/les/downloader/testchain_test.go index 359dc4bad4..41cb789817 100644 --- a/les/downloader/testchain_test.go +++ b/les/downloader/testchain_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // Test chain parameters. @@ -40,7 +41,7 @@ var ( Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - testGenesis = gspec.MustCommit(testDB) + testGenesis = gspec.MustCommit(testDB, trie.NewDatabase(testDB, nil)) ) // The common prefix of all test chains: diff --git a/les/fetcher/block_fetcher_test.go b/les/fetcher/block_fetcher_test.go index a3d320c126..6a64d3933b 100644 --- a/les/fetcher/block_fetcher_test.go +++ b/les/fetcher/block_fetcher_test.go @@ -43,7 +43,7 @@ var ( Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(testdb) + genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, nil)) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/les/handler_test.go b/les/handler_test.go index 4fa19e9915..61500d99f9 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -406,7 +406,7 @@ func testGetProofs(t *testing.T, protocol int) { accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) - trie, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) + trie, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB()) for _, acc := range accounts { req := ProofReq{ @@ -457,7 +457,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { proofsV2 := light.NewNodeSet() - t, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) + t, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB()) t.Prove(account, 0, proofsV2) expected = proofsV2.NodeList() } @@ -513,7 +513,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxData: [][]byte{rlp}, } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) - trie, _ := trie.New(trie.StateTrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(trie.StateTrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix), nil)) trie.Prove(key, 0, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ @@ -578,7 +578,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(trie.StateTrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) + trie, _ := trie.New(trie.StateTrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix), nil)) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response diff --git a/les/server_handler.go b/les/server_handler.go index 9029034a5c..dc2b3d7527 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -392,7 +392,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { if root == (common.Hash{}) { return nil } - trie, _ := trie.New(trie.StateTrieID(root), trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + trie, _ := trie.New(trie.StateTrieID(root), trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix), nil)) return trie } diff --git a/les/test_helper.go b/les/test_helper.go index bd967b112d..01dcf89132 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -52,6 +52,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -203,7 +204,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index } oracle *checkpointoracle.CheckpointOracle ) - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) chain, _ := light.NewLightChain(odr, gspec.Config, engine, nil) if indexers != nil { checkpointConfig := ¶ms.CheckpointOracleConfig{ @@ -263,7 +264,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da } oracle *checkpointoracle.CheckpointOracle ) - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) // create a simulation backend and pre-commit several customized block to the database. simulation := backends.NewSimulatedBackendWithDatabase(db, gspec.Alloc, 100000000) diff --git a/light/lightchain_test.go b/light/lightchain_test.go index e9d43d6da0..d9fb159119 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // So we can deterministically seed different blockchains @@ -55,7 +56,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [ func newCanonical(n int) (ethdb.Database, *LightChain, error) { db := rawdb.NewMemoryDatabase() gspec := core.Genesis{Config: params.TestChainConfig} - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker(), nil) // Create and inject the requested chain @@ -75,7 +76,7 @@ func newTestLightChain() *LightChain { Difficulty: big.NewInt(1), Config: params.TestChainConfig, } - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, nil)) lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker(), nil) if err != nil { panic(err) diff --git a/light/odr_test.go b/light/odr_test.go index 1cadad5ec2..6929696cc3 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -82,7 +82,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number) } case *TrieRequest: - t, _ := trie.New(trie.StorageTrieID(req.Id.StateRoot, common.BytesToHash(req.Id.AccKey), req.Id.Root), trie.NewDatabase(odr.sdb)) + t, _ := trie.New(trie.StorageTrieID(req.Id.StateRoot, common.BytesToHash(req.Id.AccKey), req.Id.Root), trie.NewDatabase(odr.sdb, nil)) nodes := NewNodeSet() t.Prove(req.Key, 0, nodes) req.Proof = nodes @@ -258,9 +258,9 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(sdb) + genesis = gspec.MustCommit(sdb, trie.NewDatabase(sdb, nil)) ) - gspec.MustCommit(ldb) + gspec.MustCommit(ldb, trie.NewDatabase(ldb, nil)) // Assemble the test environment blockchain, _ := core.NewBlockChain(sdb, nil, &gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, 4, testChainGen, true) diff --git a/light/postprocess.go b/light/postprocess.go index f43fdb3cb5..4dcc358129 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -149,7 +149,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, dis diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down + triedb: trie.NewDatabase(trieTable, nil), // Use a tiny cache only to keep memory down trieset: mapset.NewSet(), sectionSize: size, disablePruning: disablePruning, @@ -354,7 +354,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down + triedb: trie.NewDatabase(trieTable, nil), // Use a tiny cache only to keep memory down trieset: mapset.NewSet(), parentSize: parentSize, size: size, diff --git a/light/trie.go b/light/trie.go index a09488a4ba..b7c4f311bb 100644 --- a/light/trie.go +++ b/light/trie.go @@ -151,8 +151,8 @@ func (t *odrTrie) Hash() common.Hash { return t.trie.Hash() } -func (t *odrTrie) NodeIterator(startkey []byte) trie.NodeIterator { - return newNodeIterator(t, startkey) +func (t *odrTrie) NodeIterator(startkey []byte) (trie.NodeIterator, error) { + return newNodeIterator(t, startkey), nil } func (t *odrTrie) GetKey(sha []byte) []byte { @@ -175,7 +175,7 @@ func (t *odrTrie) do(key []byte, fn func() error) error { } else { id = trie.StateTrieID(t.id.StateRoot) } - t.trie, err = trie.New(id, trie.NewDatabase(t.db.backend.Database())) + t.trie, err = trie.New(id, trie.NewDatabase(t.db.backend.Database(), nil)) } if err == nil { err = fn() @@ -207,7 +207,7 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { } else { id = trie.StateTrieID(t.id.StateRoot) } - t, err := trie.New(id, trie.NewDatabase(t.db.backend.Database())) + t, err := trie.New(id, trie.NewDatabase(t.db.backend.Database(), nil)) if err == nil { it.t.trie = t } @@ -215,7 +215,11 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { }) } it.do(func() error { - it.NodeIterator = it.t.trie.NodeIterator(startkey) + var err error + it.NodeIterator, err = it.t.trie.NodeIterator(startkey) + if err != nil { + return err + } return it.NodeIterator.Error() }) return it diff --git a/light/trie_test.go b/light/trie_test.go index 3e858c8ab7..be05504f02 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -42,9 +42,9 @@ func TestNodeIterator(t *testing.T) { Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(fulldb) + genesis = gspec.MustCommit(fulldb, trie.NewDatabase(fulldb, nil)) ) - gspec.MustCommit(lightdb) + gspec.MustCommit(lightdb, trie.NewDatabase(lightdb, nil)) blockchain, _ := core.NewBlockChain(fulldb, nil, &gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), fulldb, 4, testChainGen, true) if _, err := blockchain.InsertChain(gchain, nil); err != nil { @@ -62,8 +62,16 @@ func TestNodeIterator(t *testing.T) { } func diffTries(t1, t2 state.Trie) error { - i1 := trie.NewIterator(t1.NodeIterator(nil)) - i2 := trie.NewIterator(t2.NodeIterator(nil)) + trieIt1, err := t1.NodeIterator(nil) + if err != nil { + return err + } + trieIt2, err := t2.NodeIterator(nil) + if err != nil { + return err + } + i1 := trie.NewIterator(trieIt1) + i2 := trie.NewIterator(trieIt2) for i1.Next() && i2.Next() { if !bytes.Equal(i1.Key, i2.Key) { spew.Dump(i2) diff --git a/light/txpool_test.go b/light/txpool_test.go index 3c377f7b00..8398742689 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) type testTxRelay struct { @@ -88,9 +89,9 @@ func TestTxPool(t *testing.T) { Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(sdb) + genesis = gspec.MustCommit(sdb, trie.NewDatabase(sdb, nil)) ) - gspec.MustCommit(ldb) + gspec.MustCommit(ldb, trie.NewDatabase(ldb, nil)) // Assemble the test environment blockchain, _ := core.NewBlockChain(sdb, nil, &gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, poolTestBlocks, txPoolTestChainGen, true) diff --git a/miner/miner_test.go b/miner/miner_test.go index 1b6eeecd66..fa652b27e4 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -251,7 +251,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) { if err != nil { t.Fatalf("can't create new chain %v", err) } - statedb, _ := state.New(common.Hash{}, state.NewDatabase(chainDB), nil) + statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache(), nil) blockchain := &testBlockChain{statedb, 10000000, new(event.Feed)} legacyPool := legacypool.New(testTxPoolConfig, bc.Config(), blockchain) diff --git a/miner/worker_test.go b/miner/worker_test.go index 4b150e1a85..11d2f51c4f 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -42,6 +42,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" ) @@ -142,7 +143,7 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine default: t.Fatalf("unexpected consensus engine type: %T", engine) } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, nil)) chain, _ := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, &gspec, nil, engine, vm.Config{}, nil, nil) legacyPool := legacypool.New(testTxPoolConfig, chainConfig, chain) @@ -244,7 +245,7 @@ func testGenerateBlockAndImport(t *testing.T, isClique bool) { // This test chain imports the mined blocks. db2 := rawdb.NewMemoryDatabase() - b.genesis.MustCommit(db2) + b.genesis.MustCommit(db2, trie.NewDatabase(db2, nil)) chain, _ := core.NewBlockChain(db2, nil, b.genesis, nil, engine, vm.Config{}, nil, nil) defer chain.Stop() diff --git a/tests/block_test.go b/tests/block_test.go index 74c7ed8197..baea3615d9 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -18,6 +18,8 @@ package tests import ( "testing" + + "github.com/ethereum/go-ethereum/core/rawdb" ) func TestBlockchain(t *testing.T) { @@ -47,12 +49,19 @@ func TestBlockchain(t *testing.T) { // using 4.6 TGas bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if err := bt.checkFailure(t, test.Run(false)); err != nil { - t.Errorf("test without snapshotter failed: %v", err) + if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme)); err != nil { + t.Errorf("test in hash mode without snapshotter failed: %v", err) } - if err := bt.checkFailure(t, test.Run(true)); err != nil { - t.Errorf("test with snapshotter failed: %v", err) + if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme)); err != nil { + t.Errorf("test in hash mode with snapshotter failed: %v", err) } + // if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme)); err != nil { + + // t.Errorf("test in path mode without snapshotter failed: %v", err) + // } + // if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme)); err != nil { + // t.Errorf("test in path mode with snapshotter failed: %v", err) + // } }) // There is also a LegacyTests folder, containing blockchain tests generated // prior to Istanbul. However, they are all derived from GeneralStateTests, diff --git a/tests/block_test_util.go b/tests/block_test_util.go index df955e3d69..227b9bf605 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -37,6 +37,9 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // A BlockTest checks handling of entire blocks. @@ -98,15 +101,26 @@ type btHeaderMarshaling struct { BaseFeePerGas *math.HexOrDecimal256 } -func (t *BlockTest) Run(snapshotter bool) error { +func (t *BlockTest) Run(snapshotter bool, scheme string) error { config, ok := Forks[t.json.Network] if !ok { return UnsupportedForkError{t.json.Network} } // import pre accounts & construct test genesis block & state root - db := rawdb.NewMemoryDatabase() - gblock := t.genesis(config).MustCommit(db) + var ( + db = rawdb.NewMemoryDatabase() + tconf = &trie.Config{} + ) + if scheme == rawdb.PathScheme { + tconf.PathDB = pathdb.Defaults + } else { + tconf.HashDB = hashdb.Defaults + } + triedb := trie.NewDatabase(db, tconf) + // Commit genesis state + gblock := t.genesis(config).MustCommit(db, triedb) + triedb.Close() if gblock.Hash() != t.json.Genesis.Hash { return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6]) } @@ -119,7 +133,7 @@ func (t *BlockTest) Run(snapshotter bool) error { } else { engine = ethash.NewShared() } - cache := &core.CacheConfig{TrieCleanLimit: 0} + cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme} if snapshotter { cache.SnapshotLimit = 1 cache.SnapshotWait = true diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index 494979109a..3f48578291 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -62,7 +62,7 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { Alloc: core.GenesisAlloc{bankAddr: {Balance: bankFunds}}, GasLimit: 100000000, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, nil) signer := types.HomesteadSigner{} blocks, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, testChainLen, func(i int, gen *core.BlockGen) { @@ -90,8 +90,8 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { } func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { - chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) - bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)) + bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < testChainLen; i++ { // The element in CHT is -> key := make([]byte, 8) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 5a65152aa8..aea4a47985 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -62,7 +62,7 @@ func (f *fuzzer) readInt() uint64 { } func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { - trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) size := f.readInt() // Fill it with some fluff diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 6e728ac2c2..9ade6f2b2f 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -149,10 +149,10 @@ func (f *fuzzer) fuzz() int { // This spongeDb is used to check the sequence of disk-db-writes var ( spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA)) + dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA), nil) trieA = trie.NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB)) + dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB), nil) trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme()) }) @@ -237,7 +237,7 @@ func (f *fuzzer) fuzz() int { panic(fmt.Sprintf("roots differ: (trie) %x != %x (stacktrie)", rootA, rootC)) } trieA, _ = trie.New(trie.TrieID(rootA), dbA) - iterA := trieA.NodeIterator(nil) + iterA := trieA.MustNodeIterator(nil) for iterA.Next(true) { if iterA.Hash() == (common.Hash{}) { if _, present := nodeset[string(iterA.Path())]; present { diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index dc2e689a56..617d4f7c30 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -145,7 +145,7 @@ func Fuzz(input []byte) int { func runRandTest(rt randTest) error { var ( - triedb = trie.NewDatabase(rawdb.NewMemoryDatabase()) + triedb = trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) tr = trie.NewEmpty(triedb) origin = types.EmptyRootHash values = make(map[string]string) // tracks content of the trie @@ -185,7 +185,7 @@ func runRandTest(rt randTest) error { origin = hash case opItercheckhash: checktr := trie.NewEmpty(triedb) - it := trie.NewIterator(tr.NodeIterator(nil)) + it := trie.NewIterator(tr.MustNodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) } diff --git a/tests/state_test.go b/tests/state_test.go index 16975cef31..6eb65350a4 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -25,6 +25,9 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/vm" ) @@ -68,29 +71,52 @@ func TestState(t *testing.T) { subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) - t.Run(key+"/trie", func(t *testing.T) { + t.Run(key+"/hash/trie", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - _, _, err := test.Run(subtest, vmconfig, false) - if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { - // Ignore expected errors (TODO MariusVanDerWijden check error string) - return nil - } - return st.checkFailure(t, err) + var result error + test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + result = st.checkFailure(t, err) + }) + return result }) }) - t.Run(key+"/snap", func(t *testing.T) { + t.Run(key+"/hash/snap", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - snaps, statedb, err := test.Run(subtest, vmconfig, true) - if snaps != nil && statedb != nil { - if _, err := snaps.Journal(statedb.IntermediateRoot(false)); err != nil { - return err + var result error + test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + if snaps != nil && state != nil { + if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { + result = err + return + } } - } - if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { - // Ignore expected errors (TODO MariusVanDerWijden check error string) - return nil - } - return st.checkFailure(t, err) + result = st.checkFailure(t, err) + }) + return result + }) + }) + t.Run(key+"/path/trie", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + result = st.checkFailure(t, err) + }) + return result + }) + }) + t.Run(key+"/path/snap", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + if snaps != nil && state != nil { + if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { + result = err + return + } + } + result = st.checkFailure(t, err) + }) + return result }) }) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index a923535181..e3f576718b 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -37,6 +37,9 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "golang.org/x/crypto/sha3" ) @@ -158,32 +161,41 @@ func (t *StateTest) Subtests() []StateSubtest { } // Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) { - snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter) +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) { + triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) if err != nil { - return snaps, statedb, err + return err } + // Invoke the callback at the end of function for further analysis. + defer func() { + postCheck(result, snaps, statedb) + + if triedb != nil { + triedb.Close() + } + }() post := t.json.Post[subtest.Fork][subtest.Index] // N.B: We need to do this in a two-step process, because the first Commit takes care // of suicides, and we need to touch the coinbase _after_ it has potentially suicided. if root != common.Hash(post.Root) { - return snaps, statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) + return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { - return snaps, statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) + return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - return snaps, statedb, nil + return nil } // RunNoVerify runs a specific subtest and returns the statedb and post-state root -func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, common.Hash, error) { +func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { - return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} + return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips block := t.genesis(config).ToBlock() - snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) + triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme) + defer triedb.Close() var baseFee *big.Int if config.IsLondon(new(big.Int)) { @@ -197,7 +209,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post, baseFee) if err != nil { - return nil, nil, common.Hash{}, err + return nil, nil, nil, common.Hash{}, err } // Try to recover tx with current signer @@ -205,11 +217,11 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh var ttx types.Transaction err := ttx.UnmarshalBinary(post.TxBytes) if err != nil { - return nil, nil, common.Hash{}, err + return nil, nil, nil, common.Hash{}, err } if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { - return nil, nil, common.Hash{}, err + return nil, nil, nil, common.Hash{}, err } } @@ -240,15 +252,22 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number())) // And _now_ get the state root root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) - return snaps, statedb, root, err + return triedb, snaps, statedb, root, err } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas] } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool) (*snapshot.Tree, *state.StateDB) { - sdb := state.NewDatabase(db) +func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) { + tconf := &trie.Config{Preimages: true} + if scheme == rawdb.HashScheme { + tconf.HashDB = hashdb.Defaults + } else { + tconf.PathDB = pathdb.Defaults + } + triedb := trie.NewDatabase(db, tconf) + sdb := state.NewDatabaseWithNodeDB(db, triedb) statedb, _ := state.New(common.Hash{}, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) @@ -263,10 +282,10 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo var snaps *snapshot.Tree if snapshotter { - snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, root, false, true, false) + snaps, _ = snapshot.New(db, triedb, 1, root, false, true, false) } statedb, _ = state.New(root, sdb, snaps) - return snaps, statedb + return triedb, snaps, statedb } func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { diff --git a/trie/database.go b/trie/database.go index 92791a92d8..79ce83588e 100644 --- a/trie/database.go +++ b/trie/database.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" @@ -29,9 +30,10 @@ import ( // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Journal string // Journal of clean cache to survive node restarts - Preimages bool // Flag whether the preimage of trie key is recorded + Preimages bool // Flag whether the preimage of trie key is recorded + + HashDB *hashdb.Config // Configs for hash-based scheme + PathDB *pathdb.Config // Configs for experimental path-based scheme // Testing hooks OnCommit func(states *triestate.Set) // Hook invoked when commit is performed @@ -93,38 +95,50 @@ func prepare(diskdb ethdb.Database, config *Config) *Database { } } -// NewDatabase initializes the trie database with default settings, namely -// the legacy hash-based scheme is used by default. -func NewDatabase(diskdb ethdb.Database) *Database { - return NewDatabaseWithConfig(diskdb, nil) +// HashDefaults represents a config for using hash-based scheme with +// default settings. +var HashDefaults = &Config{ + Preimages: false, + HashDB: hashdb.Defaults, } -// NewDatabaseWithConfig initializes the trie database with provided configs. -// The path-based scheme is not activated yet, always initialized with legacy -// hash-based scheme by default. -func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database { - var cleans int - - if config != nil && config.Cache != 0 { - cleans = config.Cache * 1024 * 1024 +// NewDatabase initializes the trie database with default settings, namely +// the legacy hash-based scheme is used by default. +func NewDatabase(diskdb ethdb.Database, config *Config) *Database { + if config == nil { + config = HashDefaults + } + var preimages *preimageStore + if config.Preimages { + preimages = newPreimageStore(diskdb) + } + db := &Database{ + config: config, + diskdb: diskdb, + preimages: preimages, + } + if config.HashDB != nil && config.PathDB != nil { + log.Crit("Both 'hash' and 'path' mode are configured") + } + if config.PathDB != nil { + db.backend = pathdb.New(diskdb, config.PathDB) + } else { + // Use hashdb by default + db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) } - db := prepare(diskdb, config) - db.backend = hashdb.New(diskdb, cleans, mptResolver{}) return db } // Reader returns a reader for accessing all trie nodes with provided state root. // Nil is returned in case the state is not available. -func (db *Database) Reader(blockRoot common.Hash) Reader { +func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { switch b := db.backend.(type) { case *hashdb.Database: return b.Reader(blockRoot) case *pathdb.Database: - reader, _ := b.Reader(blockRoot) - return reader + return b.Reader(blockRoot) } - return nil - + return nil, errors.New("unsupported") } // Update performs a state transition by committing dirty nodes contained in the @@ -242,3 +256,60 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { } return hdb.Node(hash) } + +// Recover rollbacks the database to a specified historical point. The state is +// supported as the rollback destination only if it's canonical state and the +// corresponding trie histories are existent. It's only supported by path-based +// database and will return an error for others. +func (db *Database) Recover(target common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Recover(target, &trieLoader{db: db}) +} + +// Recoverable returns the indicator if the specified state is enabled to be +// recovered. It's only supported by path-based database and will return an +// error for others. +func (db *Database) Recoverable(root common.Hash) (bool, error) { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return false, errors.New("not supported") + } + return pdb.Recoverable(root), nil +} + +// Reset wipes all available journal from the persistent database and discard +// all caches and diff layers. Using the given root to create a new disk layer. +// It's only supported by path-based database and will return an error for others. +func (db *Database) Reset(root common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Reset(root) +} + +// Journal commits an entire diff hierarchy to disk into a single journal entry. +// This is meant to be used during shutdown to persist the snapshot without +// flattening everything down (bad for reorgs). It's only supported by path-based +// database and will return an error for others. +func (db *Database) Journal(root common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Journal(root) +} + +// SetBufferSize sets the node buffer size to the provided value(in bytes). +// It's only supported by path-based database and will return an error for +// others. +func (db *Database) SetBufferSize(size int) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.SetBufferSize(size) +} diff --git a/trie/database_test.go b/trie/database_test.go index 0d4f63e467..e456bf7177 100644 --- a/trie/database_test.go +++ b/trie/database_test.go @@ -20,6 +20,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // newTestDatabase initializes the trie database with specified scheme. @@ -27,10 +28,9 @@ import ( func newTestDatabase(diskdb ethdb.Database, scheme string) *Database { db := prepare(diskdb, nil) if scheme == rawdb.HashScheme { - db.backend = hashdb.New(diskdb, 0, mptResolver{}) + db.backend = hashdb.New(diskdb, &hashdb.Config{}, mptResolver{}) + } else { + db.backend = pathdb.New(diskdb, &pathdb.Config{}) // disable clean/dirty cache } - // //} else { - // // db.backend = snap.New(diskdb, db.cleans, nil) - // //} return db } diff --git a/trie/errors.go b/trie/errors.go index afe344bed2..f614dd30ad 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -17,11 +17,17 @@ package trie import ( + "errors" "fmt" "github.com/ethereum/go-ethereum/common" ) +// ErrCommitted is returned when a already committed trie is requested for usage. +// The potential usages can be `Get`, `Update`, `Delete`, `NodeIterator`, `Prove` +// and so on. +var ErrCommitted = errors.New("trie is already committed") + // MissingNodeError is returned by the trie functions (TryGet, TryUpdate, TryDelete) // in the case where a trie node is not present in the local database. It contains // information necessary for retrieving the missing node. diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 3527cc0266..240aa25284 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -33,8 +33,8 @@ import ( ) func TestEmptyIterator(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - iter := trie.NodeIterator(nil) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + iter := trie.MustNodeIterator(nil) seen := make(map[string]struct{}) for iter.Next(true) { @@ -46,7 +46,7 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -69,7 +69,7 @@ func TestIterator(t *testing.T) { db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) found := make(map[string]string) - it := NewIterator(trie.NodeIterator(nil)) + it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { found[string(it.Key)] = string(it.Value) } @@ -87,7 +87,7 @@ type kv struct { } func TestIteratorLargeData(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -99,7 +99,7 @@ func TestIteratorLargeData(t *testing.T) { vals[string(value2.k)] = value2 } - it := NewIterator(trie.NodeIterator(nil)) + it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { vals[string(it.Key)].t = true } @@ -128,7 +128,7 @@ type iterationElement struct { func TestNodeIteratorCoverage(t *testing.T) { testNodeIteratorCoverage(t, rawdb.HashScheme) - //testNodeIteratorCoverage(t, rawdb.PathScheme) + testNodeIteratorCoverage(t, rawdb.PathScheme) } func testNodeIteratorCoverage(t *testing.T, scheme string) { @@ -137,7 +137,7 @@ func testNodeIteratorCoverage(t *testing.T, scheme string) { // Gather all the node hashes found by the iterator var elements = make(map[common.Hash]iterationElement) - for it := trie.NodeIterator(nil); it.Next(true); { + for it := trie.MustNodeIterator(nil); it.Next(true); { if it.Hash() != (common.Hash{}) { elements[it.Hash()] = iterationElement{ hash: it.Hash(), @@ -148,7 +148,12 @@ func testNodeIteratorCoverage(t *testing.T, scheme string) { } // Cross check the hashes and the database itself for _, element := range elements { - if blob, err := nodeDb.Reader(trie.Hash()).Node(common.Hash{}, element.path, element.hash); err != nil { + reader, err := nodeDb.Reader(trie.Hash()) + if err != nil { + t.Errorf("failed to retrieve reader %v", err) + } + + if blob, err := reader.Node(common.Hash{}, element.path, element.hash); err != nil { t.Errorf("failed to retrieve reported node %x: %v", element.hash, err) } else if !bytes.Equal(blob, element.blob) { t.Errorf("node blob is different, want %v got %v", element.blob, blob) @@ -202,25 +207,25 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range testdata1 { trie.Update([]byte(val.k), []byte(val.v)) } // Seek to the middle. - it := NewIterator(trie.NodeIterator([]byte("fab"))) + it := NewIterator(trie.MustNodeIterator([]byte("fab"))) if err := checkIteratorOrder(testdata1[4:], it); err != nil { t.Fatal(err) } // Seek to a non-existent key. - it = NewIterator(trie.NodeIterator([]byte("barc"))) + it = NewIterator(trie.MustNodeIterator([]byte("barc"))) if err := checkIteratorOrder(testdata1[1:], it); err != nil { t.Fatal(err) } // Seek beyond the end. - it = NewIterator(trie.NodeIterator([]byte("z"))) + it = NewIterator(trie.MustNodeIterator([]byte("z"))) if err := checkIteratorOrder(nil, it); err != nil { t.Fatal(err) } @@ -243,7 +248,7 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase()) + dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) @@ -252,7 +257,7 @@ func TestDifferenceIterator(t *testing.T) { dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase()) + dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) @@ -262,7 +267,7 @@ func TestDifferenceIterator(t *testing.T) { trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) - di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) + di, _ := NewDifferenceIterator(triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)) it := NewIterator(di) for it.Next() { found[string(it.Key)] = string(it.Value) @@ -285,7 +290,7 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase()) + dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) @@ -294,7 +299,7 @@ func TestUnionIterator(t *testing.T) { dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase()) + dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) @@ -303,7 +308,7 @@ func TestUnionIterator(t *testing.T) { dbb.Update(rootB, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesB), nil) trieb, _ = New(TrieID(rootB), dbb) - di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) + di, _ := NewUnionIterator([]NodeIterator{triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)}) it := NewIterator(di) all := []struct{ k, v string }{ @@ -338,19 +343,19 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } - checkIteratorNoDups(t, tr.NodeIterator(nil), nil) + checkIteratorNoDups(t, tr.MustNodeIterator(nil), nil) } // This test checks that nodeIterator.Next can be retried after inserting missing trie nodes. func TestIteratorContinueAfterError(t *testing.T) { testIteratorContinueAfterError(t, false, rawdb.HashScheme) testIteratorContinueAfterError(t, true, rawdb.HashScheme) - // testIteratorContinueAfterError(t, false, rawdb.PathScheme) - // testIteratorContinueAfterError(t, true, rawdb.PathScheme) + testIteratorContinueAfterError(t, false, rawdb.PathScheme) + testIteratorContinueAfterError(t, true, rawdb.PathScheme) } func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { @@ -367,7 +372,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { tdb.Commit(root, false) } tr, _ = New(TrieID(root), tdb) - wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil) + wantNodeCount := checkIteratorNoDups(t, tr.MustNodeIterator(nil), nil) var ( paths [][]byte @@ -426,7 +431,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { } // Iterate until the error is hit. seen := make(map[string]bool) - it := tr.NodeIterator(nil) + it := tr.MustNodeIterator(nil) checkIteratorNoDups(t, it, seen) missing, ok := it.Error().(*MissingNodeError) if !ok || missing.NodeHash != rhash { @@ -455,8 +460,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { func TestIteratorContinueAfterSeekError(t *testing.T) { testIteratorContinueAfterSeekError(t, false, rawdb.HashScheme) testIteratorContinueAfterSeekError(t, true, rawdb.HashScheme) - // testIteratorContinueAfterSeekError(t, false, rawdb.PathScheme) - // testIteratorContinueAfterSeekError(t, true, rawdb.PathScheme) + testIteratorContinueAfterSeekError(t, false, rawdb.PathScheme) + testIteratorContinueAfterSeekError(t, true, rawdb.PathScheme) } func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme string) { @@ -494,7 +499,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin } // Create a new iterator that seeks to "bars". Seeking can't proceed because // the node is missing. - it := tr.NodeIterator([]byte("bars")) + it := tr.MustNodeIterator([]byte("bars")) missing, ok := it.Error().(*MissingNodeError) if !ok { t.Fatal("want MissingNodeError, got", it.Error()) @@ -528,7 +533,7 @@ func checkIteratorNoDups(t *testing.T, it NodeIterator, seen map[string]bool) in func TestIteratorNodeBlob(t *testing.T) { testIteratorNodeBlob(t, rawdb.HashScheme) - //testIteratorNodeBlob(t, rawdb.PathScheme) + testIteratorNodeBlob(t, rawdb.PathScheme) } type loggingDb struct { @@ -585,8 +590,8 @@ func (l *loggingDb) Close() error { func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} - triedb := NewDatabase(rawdb.NewDatabase(logDb)) - trie, _ := NewSecure(TrieID(common.Hash{}), triedb) + triedb := NewDatabase(rawdb.NewDatabase(logDb), nil) + trie, _ := NewSecure(TrieID(types.EmptyRootHash), triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { @@ -600,7 +605,10 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { } root, nodes, _ := trie.Commit(false) triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) + triedb.Commit(root, false) // Return the generated trie + trie, _ = NewSecure(TrieID(root), triedb) + return triedb, trie, logDb } @@ -612,8 +620,8 @@ func TestNodeIteratorLargeTrie(t *testing.T) { // Do a seek operation trie.NodeIterator(common.FromHex("0x77667766776677766778855885885885")) // master: 24 get operations - // this pr: 5 get operations - if have, want := logDb.getCount, uint64(5); have != want { + // this pr: 6 get operations + if have, want := logDb.getCount, uint64(6); have != want { t.Fatalf("Too many lookups during seek, have %d want %d", have, want) } } @@ -644,7 +652,7 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { var found = make(map[common.Hash][]byte) trie, _ = New(TrieID(root), triedb) - it := trie.NodeIterator(nil) + it := trie.MustNodeIterator(nil) for it.Next(true) { if it.Hash() == (common.Hash{}) { continue @@ -692,7 +700,7 @@ func isTrieNode(scheme string, key, val []byte) (bool, []byte, common.Hash) { } hash = common.BytesToHash(key) } else { - ok, remain := rawdb.IsAccountTrieNode(key) + ok, remain := rawdb.ResolveAccountTrieNodeKey(key) if !ok { return false, nil, common.Hash{} } diff --git a/trie/proof.go b/trie/proof.go index c8179eeeb4..c1b4ae305f 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -35,6 +35,9 @@ import ( // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { + if t.committed { + return ErrCommitted + } // Collect all nodes on the path to key. var ( prefix []byte diff --git a/trie/proof_test.go b/trie/proof_test.go index f772a5e838..9f31b88d17 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -49,7 +49,7 @@ func makeProvers(trie *Trie) []func(key []byte) *memorydb.Database { // Create a leaf iterator based Merkle prover provers = append(provers, func(key []byte) *memorydb.Database { proof := memorydb.New() - if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { + if it := NewIterator(trie.MustNodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { for _, p := range it.Prove() { proof.Put(crypto.Keccak256(p), p) } @@ -80,7 +80,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -131,7 +131,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -387,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.Update(entry.k, entry.v) @@ -459,7 +459,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -494,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) { // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. func TestReverseSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -601,7 +601,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -675,7 +675,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -1028,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -1053,7 +1053,7 @@ func randBytes(n int) []byte { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 973596a58f..8ff4630a7f 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -201,10 +201,16 @@ func (t *SecureTrie) Copy() *SecureTrie { // NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration // starts at the key after the given start key. -func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { +func (t *SecureTrie) NodeIterator(start []byte) (NodeIterator, error) { return t.trie.NodeIterator(start) } +// MustNodeIterator is a wrapper of NodeIterator and will omit any encountered +// error but just print out an error message. +func (t *SecureTrie) MustNodeIterator(start []byte) NodeIterator { + return t.trie.MustNodeIterator(start) +} + // hashKey returns the hash of key as an ephemeral buffer. // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 835608a0e3..797d23d4b0 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -28,14 +28,14 @@ import ( ) func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(TrieID(common.Hash{}), NewDatabase(rawdb.NewMemoryDatabase())) + trie, _ := NewSecure(TrieID(common.Hash{}), NewDatabase(rawdb.NewMemoryDatabase(), nil)) return trie } // makeTestSecureTrie creates a large enough secure trie for testing. func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie, _ := NewSecure(TrieID(common.Hash{}), triedb) // Fill it with some arbitrary data diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index dd4c75f5f8..10baeaf441 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -189,7 +189,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -204,7 +204,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -230,7 +230,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -255,7 +255,7 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) kvs := []struct { K string @@ -284,7 +284,7 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) kvs := []struct { K string @@ -355,7 +355,7 @@ func TestStacktrieNotModifyValues(t *testing.T) { func TestStacktrieSerialization(t *testing.T) { var ( st = NewStackTrie(nil) - nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) keyB = big.NewInt(1) keyDelta = big.NewInt(1) vals [][]byte diff --git a/trie/sync.go b/trie/sync.go index d68ea85673..a6338b8a89 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -282,17 +282,16 @@ func (s *Sync) Missing(max int) ([]string, []common.Hash, []common.Hash) { s.queue.Pop() s.fetches[depth]++ - switch item.(type) { + switch item := item.(type) { case common.Hash: - codeHashes = append(codeHashes, item.(common.Hash)) + codeHashes = append(codeHashes, item) case string: - path := item.(string) - req, ok := s.nodeReqs[path] + req, ok := s.nodeReqs[item] if !ok { - log.Error("Missing node request", "path", path) + log.Error("Missing node request", "path", item) continue // System very wrong, shouldn't happen } - nodePaths = append(nodePaths, path) + nodePaths = append(nodePaths, item) nodeHashes = append(nodeHashes, req.hash) } } diff --git a/trie/sync_test.go b/trie/sync_test.go index 386e1995d7..b3fb10db14 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -36,7 +36,7 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *SecureTrie, map[st db := rawdb.NewMemoryDatabase() triedb := newTestDatabase(db, scheme) - trie, _ := NewSecure(TrieID(common.Hash{}), triedb) + trie, _ := NewSecure(TrieID(types.EmptyRootHash), triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -68,7 +68,9 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *SecureTrie, map[st panic(err) } - // Return the generated trie + // Re-create the trie based on the new state + trie, _ = NewSecure(TrieID(root), triedb) + return db, triedb, trie, content } @@ -98,7 +100,7 @@ func checkTrieConsistency(db ethdb.Database, scheme string, root common.Hash) er if err != nil { return nil // Consider a non existent state consistent } - it := trie.NodeIterator(nil) + it := trie.MustNodeIterator(nil) for it.Next(true) { } return it.Error() @@ -113,18 +115,18 @@ type trieElement struct { // Tests that an empty trie is not scheduled for syncing. func TestEmptySync(t *testing.T) { - dbA := NewDatabase(rawdb.NewMemoryDatabase()) - dbB := NewDatabase(rawdb.NewMemoryDatabase()) - //dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) - //dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) + dbA := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + dbB := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) + dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) emptyA := NewEmpty(dbA) emptyB, _ := New(TrieID(emptyRoot), dbB) - //emptyC := NewEmpty(dbC) - //emptyD, _ := New(TrieID(types.EmptyRootHash), dbD) + emptyC := NewEmpty(dbC) + emptyD, _ := New(TrieID(types.EmptyRootHash), dbD) - for i, trie := range []*Trie{emptyA, emptyB /*emptyC, emptyD*/} { - sync := NewSync(trie.Hash(), memorydb.New(), nil, NewSyncBloom(1, memorydb.New()), []*Database{dbA, dbB /*dbC, dbD*/}[i].Scheme()) + for i, trie := range []*Trie{emptyA, emptyB, emptyC, emptyD} { + sync := NewSync(trie.Hash(), memorydb.New(), nil, NewSyncBloom(1, memorydb.New()), []*Database{dbA, dbB, dbC, dbD}[i].Scheme()) if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, nodes, paths, codes) } @@ -138,10 +140,10 @@ func TestIterativeSync(t *testing.T) { testIterativeSync(t, 100, false, rawdb.HashScheme) testIterativeSync(t, 1, true, rawdb.HashScheme) testIterativeSync(t, 100, true, rawdb.HashScheme) - // testIterativeSync(t, 1, false, rawdb.PathScheme) - // testIterativeSync(t, 100, false, rawdb.PathScheme) - // testIterativeSync(t, 1, true, rawdb.PathScheme) - // testIterativeSync(t, 100, true, rawdb.PathScheme) + testIterativeSync(t, 1, false, rawdb.PathScheme) + testIterativeSync(t, 100, false, rawdb.PathScheme) + testIterativeSync(t, 1, true, rawdb.PathScheme) + testIterativeSync(t, 100, true, rawdb.PathScheme) } func testIterativeSync(t *testing.T, count int, bypath bool, scheme string) { @@ -168,7 +170,11 @@ func testIterativeSync(t *testing.T, count int, bypath bool, scheme string) { if !bypath { for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", srcTrie.Hash(), err) + } + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err) } @@ -212,7 +218,7 @@ func testIterativeSync(t *testing.T, count int, bypath bool, scheme string) { // partial results are returned, and the others sent only later. func TestIterativeDelayedSync(t *testing.T) { testIterativeDelayedSync(t, rawdb.HashScheme) - //testIterativeDelayedSync(t, rawdb.PathScheme) + testIterativeDelayedSync(t, rawdb.PathScheme) } func testIterativeDelayedSync(t *testing.T, scheme string) { @@ -238,7 +244,11 @@ func testIterativeDelayedSync(t *testing.T, scheme string) { results := make([]NodeSyncResult, len(elements)/2+1) for i, element := range elements[:len(results)] { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", srcTrie.Hash(), err) + } + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -275,8 +285,8 @@ func testIterativeDelayedSync(t *testing.T, scheme string) { func TestIterativeRandomSyncIndividual(t *testing.T) { testIterativeRandomSync(t, 1, rawdb.HashScheme) testIterativeRandomSync(t, 100, rawdb.HashScheme) - // testIterativeRandomSync(t, 1, rawdb.PathScheme) - // testIterativeRandomSync(t, 100, rawdb.PathScheme) + testIterativeRandomSync(t, 1, rawdb.PathScheme) + testIterativeRandomSync(t, 100, rawdb.PathScheme) } func testIterativeRandomSync(t *testing.T, count int, scheme string) { @@ -303,7 +313,11 @@ func testIterativeRandomSync(t *testing.T, count int, scheme string) { results := make([]NodeSyncResult, 0, len(queue)) for path, element := range queue { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", srcTrie.Hash(), err) + } + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -339,7 +353,7 @@ func testIterativeRandomSync(t *testing.T, count int, scheme string) { // partial results are returned (Even those randomly), others sent only later. func TestIterativeRandomDelayedSync(t *testing.T) { testIterativeRandomDelayedSync(t, rawdb.HashScheme) - // testIterativeRandomDelayedSync(t, rawdb.PathScheme) + testIterativeRandomDelayedSync(t, rawdb.PathScheme) } func testIterativeRandomDelayedSync(t *testing.T, scheme string) { @@ -366,7 +380,11 @@ func testIterativeRandomDelayedSync(t *testing.T, scheme string) { results := make([]NodeSyncResult, 0, len(queue)/2+1) for path, element := range queue { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", srcTrie.Hash(), err) + } + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -407,7 +425,7 @@ func testIterativeRandomDelayedSync(t *testing.T, scheme string) { // have such references. func TestDuplicateAvoidanceSync(t *testing.T) { testDuplicateAvoidanceSync(t, rawdb.HashScheme) - // testDuplicateAvoidanceSync(t, rawdb.PathScheme) + testDuplicateAvoidanceSync(t, rawdb.PathScheme) } func testDuplicateAvoidanceSync(t *testing.T, scheme string) { @@ -435,7 +453,12 @@ func testDuplicateAvoidanceSync(t *testing.T, scheme string) { results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", srcTrie.Hash(), err) + } + + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -475,11 +498,10 @@ func testDuplicateAvoidanceSync(t *testing.T, scheme string) { // the database. func TestIncompleteSyncHash(t *testing.T) { testIncompleteSync(t, rawdb.HashScheme) - // testIncompleteSync(t, rawdb.PathScheme) + testIncompleteSync(t, rawdb.PathScheme) } func testIncompleteSync(t *testing.T, scheme string) { - t.Parallel() // Create a random trie to copy _, srcDb, srcTrie, _ := makeTestTrie(scheme) @@ -509,7 +531,11 @@ func testIncompleteSync(t *testing.T, scheme string) { results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", srcTrie.Hash(), err) + } + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -562,7 +588,7 @@ func testIncompleteSync(t *testing.T, scheme string) { // depth. func TestSyncOrdering(t *testing.T) { testSyncOrdering(t, rawdb.HashScheme) - // testSyncOrdering(t, rawdb.PathScheme) + testSyncOrdering(t, rawdb.PathScheme) } func testSyncOrdering(t *testing.T, scheme string) { @@ -593,7 +619,12 @@ func testSyncOrdering(t *testing.T, scheme string) { results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", srcTrie.Hash(), err) + } + + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -656,7 +687,11 @@ func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(root).Node(owner, inner, element.hash) + reader, err := srcDb.Reader(root) + if err != nil { + t.Fatalf("failed to create reader for trie %x: %v", root, err) + } + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err) } diff --git a/trie/tracer_test.go b/trie/tracer_test.go index e44b8826d6..ca11e2ea92 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -61,7 +61,7 @@ func TestTrieTracer(t *testing.T) { // Tests if the trie diffs are tracked correctly. Tracer should capture // all non-leaf dirty nodes, no matter the node is embedded or not. func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) // Determine all new nodes are tracked @@ -104,7 +104,7 @@ func TestTrieTracerNoop(t *testing.T) { } func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range vals { trie.Update([]byte(val.k), []byte(val.v)) } @@ -128,7 +128,7 @@ func TestAccessList(t *testing.T) { func testAccessList(t *testing.T, vals []struct{ k, v string }) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) orig = trie.Copy() ) @@ -211,7 +211,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { // Tests origin values won't be tracked in Iterator or Prover func TestAccessListLeak(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) ) // Create trie from scratch @@ -226,14 +226,14 @@ func TestAccessListLeak(t *testing.T) { }{ { func(tr *Trie) { - it := tr.NodeIterator(nil) + it := tr.MustNodeIterator(nil) for it.Next(true) { } }, }, { func(tr *Trie) { - it := NewIterator(tr.NodeIterator(nil)) + it := NewIterator(tr.MustNodeIterator(nil)) for it.Next() { } }, @@ -262,7 +262,7 @@ func TestAccessListLeak(t *testing.T) { // in its parent due to the smaller size of the original tree node. func TestTinyTree(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) ) for _, val := range tiny { @@ -300,7 +300,7 @@ func compareSet(setA, setB map[string]struct{}) bool { func forNodes(tr *Trie) map[string][]byte { var ( - it = tr.NodeIterator(nil) + it = tr.MustNodeIterator(nil) nodes = make(map[string][]byte) ) for it.Next(true) { @@ -320,7 +320,7 @@ func iterNodes(db *Database, root common.Hash) map[string][]byte { func forHashedNodes(tr *Trie) map[string][]byte { var ( - it = tr.NodeIterator(nil) + it = tr.MustNodeIterator(nil) nodes = make(map[string][]byte) ) for it.Next(true) { diff --git a/trie/trie.go b/trie/trie.go index 51883e7bb8..05d136fb1b 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -45,6 +45,11 @@ var ( type Trie struct { owner common.Hash root node + + // Flag whether the commit operation is already performed. If so the + // trie is not usable(latest states is invisible). + committed bool + // Keep track of the number leafs which have been inserted since the last // hashing operation. This number will not directly map to the number of // actually unhashed nodes @@ -70,7 +75,7 @@ func (t *Trie) newFlag() nodeFlag { // zero hash or the sha3 hash of an empty string, then trie is initially // empty, otherwise, the root node must be present in database or returns // a MissingNodeError if not. -func New(id *ID, db NodeReader) (*Trie, error) { +func New(id *ID, db *Database) (*Trie, error) { reader, err := newTrieReader(id.StateRoot, id.Owner, db) if err != nil { return nil, err @@ -96,10 +101,24 @@ func NewEmpty(db *Database) *Trie { return tr } +// MustNodeIterator is a wrapper of NodeIterator and will omit any encountered +// error but just print out an error message. +func (t *Trie) MustNodeIterator(start []byte) NodeIterator { + it, err := t.NodeIterator(start) + if err != nil { + log.Error("Unhandled trie error in Trie.NodeIterator", "err", err) + } + return it +} + // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at // the key after the given start key. -func (t *Trie) NodeIterator(start []byte) NodeIterator { - return newNodeIterator(t, start) +func (t *Trie) NodeIterator(start []byte) (NodeIterator, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, ErrCommitted + } + return newNodeIterator(t, start), nil } // Get returns the value for key stored in the trie. @@ -116,6 +135,10 @@ func (t *Trie) Get(key []byte) []byte { // The value bytes must not be modified by the caller. // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryGet(key []byte) ([]byte, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, ErrCommitted + } value, newroot, didResolve, err := t.tryGet(t.root, keybytesToHex(key), 0) if err == nil && didResolve { t.root = newroot @@ -162,6 +185,10 @@ func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode // TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not // possible to use keybyte-encoding as the path might contain odd nibbles. func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, 0, ErrCommitted + } item, newroot, resolved, err := t.tryGetNode(t.root, compactToHex(path), 0) if err != nil { return nil, resolved, err @@ -266,6 +293,10 @@ func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return ErrCommitted + } t.unhashed++ k := keybytesToHex(key) if len(value) != 0 { @@ -372,6 +403,10 @@ func (t *Trie) Delete(key []byte) { // TryDelete removes any existing value for key from the trie. // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryDelete(key []byte) error { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return ErrCommitted + } t.unhashed++ k := keybytesToHex(key) _, n, err := t.delete(t.root, nil, k) @@ -586,6 +621,9 @@ func (t *Trie) Hash() common.Hash { func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { defer t.tracer.reset() + defer func() { + t.committed = true + }() // (a) The trie was empty and no update happens => return nil // (b) The trie was non-empty and all nodes are dropped => return // the node set includes all deleted nodes @@ -646,15 +684,17 @@ func (t *Trie) Reset() { t.owner = common.Hash{} t.unhashed = 0 t.tracer.reset() + t.committed = false } // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - root: t.root, - owner: t.owner, - unhashed: t.unhashed, - reader: t.reader, - tracer: t.tracer.copy(), + root: t.root, + owner: t.owner, + committed: t.committed, + unhashed: t.unhashed, + reader: t.reader, + tracer: t.tracer.copy(), } } diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 92cc8f54df..4c17a438ec 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -17,9 +17,10 @@ package trie import ( - "fmt" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie/triestate" ) // Reader wraps the Node and NodeBlob method of a backing trie store. @@ -37,7 +38,7 @@ type Reader interface { type NodeReader interface { // Reader returns a reader for accessing all trie nodes with provided // state root. Nil is returned in case the state is not available. - Reader(root common.Hash) Reader + Reader(root common.Hash) (Reader, error) } // trieReader is a wrapper of the underlying node reader. It's not safe @@ -50,9 +51,16 @@ type trieReader struct { // newTrieReader initializes the trie reader with the given node reader. func newTrieReader(stateRoot, owner common.Hash, db NodeReader) (*trieReader, error) { - reader := db.Reader(stateRoot) - if reader == nil { - return nil, fmt.Errorf("state not found #%x", stateRoot) + if stateRoot == (common.Hash{}) || stateRoot == types.EmptyRootHash { + if stateRoot == (common.Hash{}) { + log.Error("zero state root") + } + return &trieReader{owner: owner}, nil + } + + reader, err := db.Reader(stateRoot) + if err != nil { + return nil, &MissingNodeError{Owner: owner, NodeHash: stateRoot, err: err} } return &trieReader{owner: owner, reader: reader}, nil } @@ -82,3 +90,18 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) { } return blob, nil } + +// trieLoader implements triestate.TrieLoader for constructing tries. +type trieLoader struct { + db *Database +} + +// OpenTrie opens the main account trie. +func (l *trieLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { + return New(TrieID(root), l.db) +} + +// OpenStorageTrie opens the storage trie of an account. +func (l *trieLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { + return New(StorageTrieID(stateRoot, addrHash, root), l.db) +} diff --git a/trie/trie_test.go b/trie/trie_test.go index 062f465066..04e2851e6e 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -48,12 +48,12 @@ func init() { // Used for testing func newEmpty() *Trie { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) return trie } func TestEmptyTrie(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) res := trie.Hash() exp := emptyRoot if res != exp { @@ -62,7 +62,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) key := make([]byte, 32) value := []byte("test") trie.Update(key, value) @@ -72,7 +72,13 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - trie, err := New(TrieID(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")), NewDatabase(rawdb.NewMemoryDatabase())) + testMissingRoot(t, rawdb.HashScheme) + testMissingRoot(t, rawdb.PathScheme) +} + +func testMissingRoot(t *testing.T, scheme string) { + root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + trie, err := New(TrieID(root), newTestDatabase(rawdb.NewMemoryDatabase(), scheme)) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -83,9 +89,9 @@ func TestMissingRoot(t *testing.T) { func TestMissingNode(t *testing.T) { testMissingNode(t, false, rawdb.HashScheme) - //testMissingNode(t, false, rawdb.PathScheme) + testMissingNode(t, false, rawdb.PathScheme) testMissingNode(t, true, rawdb.HashScheme) - //testMissingNode(t, true, rawdb.PathScheme) + testMissingNode(t, true, rawdb.PathScheme) } func testMissingNode(t *testing.T, memonly bool, scheme string) { @@ -167,7 +173,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { } func TestInsert(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -193,7 +199,7 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -220,7 +226,7 @@ func TestGet(t *testing.T) { } func TestDelete(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -247,7 +253,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -271,7 +277,7 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -338,7 +344,7 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) trie.Update([]byte("key1"), []byte{99, 99, 99, 99}) trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() @@ -475,9 +481,9 @@ func verifyAccessList(old *Trie, new *Trie, set *trienode.NodeSet) error { func runRandTest(rt randTest) bool { var scheme = rawdb.HashScheme - //if rand.Intn(2) == 0 { - // scheme = rawdb.PathScheme - //} + if rand.Intn(2) == 0 { + scheme = rawdb.PathScheme + } var ( origin = types.EmptyRootHash triedb = newTestDatabase(rawdb.NewMemoryDatabase(), scheme) @@ -550,7 +556,7 @@ func runRandTest(rt randTest) bool { case opItercheckhash: checktr := NewEmpty(triedb) - it := NewIterator(tr.NodeIterator(nil)) + it := NewIterator(tr.MustNodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) } @@ -583,7 +589,7 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 func benchGet(b *testing.B, commit bool) { - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(triedb) if commit { _, tmpdb := tempDB() @@ -665,7 +671,7 @@ func BenchmarkCommitAfterHash(b *testing.B) { func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -692,8 +698,8 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - it := NewIterator(trie.NodeIterator(nil)) + checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) } @@ -705,7 +711,7 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -771,11 +777,17 @@ func (s *spongeDb) Stat(property string) (string, error) { panic("implement func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } func (s *spongeDb) Put(key []byte, value []byte) error { - valbrief := value + var ( + keybrief = key + valbrief = value + ) + if len(keybrief) > 8 { + keybrief = keybrief[:8] + } if len(valbrief) > 8 { valbrief = valbrief[:8] } - s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, key[:8], len(value), valbrief)) + s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, keybrief, len(value), valbrief)) s.sponge.Write(key) s.sponge.Write(value) return nil @@ -813,7 +825,7 @@ func TestCommitSequence(t *testing.T) { addresses, accounts := makeAccounts(tc.count) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -844,7 +856,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { prng := rand.New(rand.NewSource(int64(i))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -876,7 +888,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { prng := rand.New(rand.NewSource(int64(count))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} @@ -936,7 +948,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do. func TestCommitSequenceSmallRoot(t *testing.T) { s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} @@ -1116,7 +1128,7 @@ func BenchmarkDerefRootFixedSize(b *testing.B) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(triedb) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -1138,7 +1150,7 @@ func tempDB() (string, *Database) { if err != nil { panic(fmt.Sprintf("can't create temporary database: %v", err)) } - return dir, NewDatabase(rawdb.NewDatabase(diskdb)) + return dir, NewDatabase(rawdb.NewDatabase(diskdb), nil) } func getString(trie *Trie, k string) []byte { diff --git a/trie/triedb/hashdb/database.go b/trie/triedb/hashdb/database.go index 8d8701a60b..a86095916a 100644 --- a/trie/triedb/hashdb/database.go +++ b/trie/triedb/hashdb/database.go @@ -18,6 +18,7 @@ package hashdb import ( "errors" + "fmt" "reflect" "sync" "time" @@ -64,6 +65,20 @@ type ChildResolver interface { ForEach(node []byte, onChild func(common.Hash)) } +// Config contains the settings for database. +type Config struct { + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes +} + +// Defaults is the default setting for database if it's not specified. +// Notably, clean cache is disabled explicitly +var Defaults = &Config{ + // Explicitly set clean cache size to 0 to avoid creating fastcache, + // otherwise database must be closed when it's no longer needed to + // prevent memory leak. + CleanCacheSize: 0, +} + // Database is an intermediate write layer between the trie data structures and // the disk database. The aim is to accumulate trie writes in-memory and only // periodically flush a couple tries to disk, garbage collecting the remainder. @@ -119,21 +134,17 @@ func (n *cachedNode) forChildren(resolver ChildResolver, onChild func(hash commo resolver.ForEach(n.node, onChild) } -// Config defines all necessary options for database. -type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Journal string // Journal of clean cache to survive node restarts - Preimages bool // Flag whether the preimage of trie key is recorded -} - // New initializes the hash-based node database. -func New(diskdb ethdb.Database, size int, resolver ChildResolver) *Database { +func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Database { + if config == nil { + config = Defaults + } // Initialize the clean cache if the specified cache allowance // is non-zero. Note, the size is in bytes. var cleans *fastcache.Cache - if size > 0 { - cleans = fastcache.New(size) + if config.CleanCacheSize > 0 { + cleans = fastcache.New(config.CleanCacheSize) } db := &Database{ diskdb: diskdb, @@ -606,7 +617,14 @@ func (db *Database) Update(root common.Hash, parent common.Hash, block uint64, n } // Close closes the trie database and releases all held resources. -func (db *Database) Close() error { return nil } +func (db *Database) Close() error { + + if db.cleans != nil { + db.cleans.Reset() + db.cleans = nil + } + return nil +} // Size returns the current storage size of the memory cache in front of the // persistent database layer. @@ -627,8 +645,12 @@ func (db *Database) Scheme() string { } // Reader retrieves a node reader belonging to the given state root. -func (db *Database) Reader(root common.Hash) *reader { - return &reader{db: db} +func (db *Database) Reader(root common.Hash) (*reader, error) { + if _, err := db.Node(root); err != nil { + return nil, fmt.Errorf("state %#x is not available, %v", root, err) + } + + return &reader{db: db}, nil } // reader is a state reader of Database which implements the Reader interface. diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 588e477eb0..f02f2164ed 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -33,8 +33,27 @@ import ( "github.com/ethereum/go-ethereum/trie/triestate" ) -// maxDiffLayers is the maximum diff layers allowed in the layer tree. -const maxDiffLayers = 128 +const ( + // maxDiffLayers is the maximum diff layers allowed in the layer tree. + + maxDiffLayers = 128 + + // defaultCleanSize is the default memory allowance of clean cache. + defaultCleanSize = 16 * 1024 * 1024 + + // maxBufferSize is the maximum memory allowance of node buffer. + // Too large nodebuffer will cause the system to pause for a long + // time when write happens. Also, the largest batch that pebble can + // support is 4GB, node will panic if batch size exceeds this limit. + maxBufferSize = 256 * 1024 * 1024 + + // DefaultBufferSize is the default memory allowance of node buffer + // that aggregates the writes from above until it's flushed into the + // disk. It's meant to be used once the initial sync is finished. + // Do not increase the buffer size arbitrarily, otherwise the system + // pause time will increase when the database writes happen. + DefaultBufferSize = 64 * 1024 * 1024 +) // layer is the interface implemented by all state layers which includes some // public methods and some additional methods for internal usage. @@ -68,28 +87,29 @@ type layer interface { // Config contains the settings for database. type Config struct { - StateLimit uint64 // Number of recent blocks to maintain state history for - CleanSize int // Maximum memory allowance (in bytes) for caching clean nodes - DirtySize int // Maximum memory allowance (in bytes) for caching dirty nodes - ReadOnly bool // Flag whether the database is opened in read only mode. + StateHistory uint64 // Number of recent blocks to maintain state history for + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes + DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes + ReadOnly bool // Flag whether the database is opened in read only mode. } -var ( - // defaultCleanSize is the default memory allowance of clean cache. - defaultCleanSize = 16 * 1024 * 1024 - - // defaultBufferSize is the default memory allowance of node buffer - // that aggregates the writes from above until it's flushed into the - // disk. Do not increase the buffer size arbitrarily, otherwise the - // system pause time will increase when the database writes happen. - defaultBufferSize = 128 * 1024 * 1024 -) - // Defaults contains default settings for Ethereum mainnet. var Defaults = &Config{ - StateLimit: params.FullImmutabilityThreshold, - CleanSize: defaultCleanSize, - DirtySize: defaultBufferSize, + StateHistory: params.FullImmutabilityThreshold, + CleanCacheSize: defaultCleanSize, + DirtyCacheSize: DefaultBufferSize, +} + +// ReadOnly is the config in order to open database in read only mode. +var ReadOnly = &Config{ReadOnly: true} + +// sanitize checks the provided user configurations and changes anything that's +// unreasonable or unworkable. +func (c *Config) sanitize() { + if c.DirtyCacheSize > maxBufferSize { + log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(c.DirtyCacheSize), "updated", common.StorageSize(maxBufferSize)) + c.DirtyCacheSize = maxBufferSize + } } // Database is a multiple-layered structure for maintaining in-memory trie nodes. @@ -123,9 +143,10 @@ func New(diskdb ethdb.Database, config *Config) *Database { if config == nil { config = Defaults } + config.sanitize() db := &Database{ readOnly: config.ReadOnly, - bufferSize: config.DirtySize, + bufferSize: config.DirtyCacheSize, config: config, diskdb: diskdb, } @@ -140,7 +161,7 @@ func New(diskdb ethdb.Database, config *Config) *Database { // mechanism also ensures that at most one **non-readOnly** database // is opened at the same time to prevent accidental mutation. if ancient, err := diskdb.AncientDatadir(); err == nil && ancient != "" && !db.readOnly { - db.freezer, err = rawdb.NewStateHistoryFreezer(ancient, false) + db.freezer, err = rawdb.NewStateFreezer(ancient, false) if err != nil { log.Crit("Failed to open state history freezer", "err", err) } @@ -171,7 +192,7 @@ func (db *Database) Reader(root common.Hash) (layer, error) { // Update adds a new layer into the tree, if that can be linked to an existing // old parent. It is disallowed to insert a disk layer (the origin of all). Apart // from that this function will flatten the extra diff layers at bottom into disk -// to only keep 128 diff layers in memory by default. +// to only keep 128 diff layerReaders in memory by default. // // The passed in maps(nodes, states) will be retained to avoid copying everything. // Therefore, these maps must not be changed afterwards. @@ -379,7 +400,12 @@ func (db *Database) SetBufferSize(size int) error { db.lock.Lock() defer db.lock.Unlock() + if size > maxBufferSize { + log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(maxBufferSize)) + size = maxBufferSize + } db.bufferSize = size + return db.tree.bottom().setBufferSize(db.bufferSize) } @@ -398,7 +424,15 @@ func (db *Database) Close() error { db.lock.Lock() defer db.lock.Unlock() + // Set the database to read-only mode to prevent all + // following mutations. + db.readOnly = true + + // Release the memory held by clean cache. + db.tree.bottom().resetCache() + + // Close the attached state history freezer. if db.freezer == nil { return nil } diff --git a/trie/triedb/pathdb/database_test.go b/trie/triedb/pathdb/database_test.go index 89a5042ada..37d92ac268 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/trie/triedb/pathdb/database_test.go @@ -51,12 +51,13 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm } for key, val := range dirties { if len(val) == 0 { - h.Delete(key.Bytes()) + h.TryDelete(key.Bytes()) } else { - h.Update(key.Bytes(), val) + h.TryUpdate(key.Bytes(), val) } } - return h.Commit(false) + root, nodes, _ := h.Commit(false) + return root, nodes } const ( @@ -99,7 +100,7 @@ type tester struct { func newTester(t *testing.T) *tester { var ( disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - db = New(disk, &Config{CleanSize: 256 * 1024, DirtySize: 256 * 1024}) + db = New(disk, &Config{CleanCacheSize: 256 * 1024, DirtyCacheSize: 256 * 1024}) obj = &tester{ db: db, preimages: make(map[common.Hash]common.Address), diff --git a/trie/triedb/pathdb/difflayer_test.go b/trie/triedb/pathdb/difflayer_test.go index 513f9685de..6b33ae758a 100644 --- a/trie/triedb/pathdb/difflayer_test.go +++ b/trie/triedb/pathdb/difflayer_test.go @@ -29,7 +29,7 @@ import ( func emptyLayer() *diskLayer { return &diskLayer{ db: New(rawdb.NewMemoryDatabase(), nil), - buffer: newNodeBuffer(defaultBufferSize, nil, 0), + buffer: newNodeBuffer(DefaultBufferSize, nil, 0), } } diff --git a/trie/triedb/pathdb/disklayer.go b/trie/triedb/pathdb/disklayer.go index 34e2d62ba0..369f7a6cd5 100644 --- a/trie/triedb/pathdb/disklayer.go +++ b/trie/triedb/pathdb/disklayer.go @@ -48,8 +48,8 @@ func newDiskLayer(root common.Hash, id uint64, db *Database, cleans *fastcache.C // Initialize a clean cache if the memory allowance is not zero // or reuse the provided cache if it is not nil (inherited from // the original disk layer). - if cleans == nil && db.config.CleanSize != 0 { - cleans = fastcache.New(db.config.CleanSize) + if cleans == nil && db.config.CleanCacheSize != 0 { + cleans = fastcache.New(db.config.CleanCacheSize) } return &diskLayer{ root: root, @@ -177,7 +177,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // corresponding states(journal), the stored state history will // be truncated in the next restart. if dl.db.freezer != nil { - err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateLimit) + err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateHistory) if err != nil { return nil, err } @@ -278,6 +278,20 @@ func (dl *diskLayer) size() common.StorageSize { return common.StorageSize(dl.buffer.size) } +// resetCache releases the memory held by clean cache to prevent memory leak. +func (dl *diskLayer) resetCache() { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // Stale disk layer loses the ownership of clean cache. + if dl.stale { + return + } + if dl.cleans != nil { + dl.cleans.Reset() + } +} + // hasher is used to compute the sha256 hash of the provided data. type hasher struct{ sha crypto.KeccakState } diff --git a/trie/triedb/pathdb/history_test.go b/trie/triedb/pathdb/history_test.go index d76db53d56..2216524b46 100644 --- a/trie/triedb/pathdb/history_test.go +++ b/trie/triedb/pathdb/history_test.go @@ -248,7 +248,7 @@ func TestTruncateTailHistories(t *testing.T) { // openFreezer initializes the freezer instance for storing state histories. func openFreezer(datadir string, readOnly bool) (*rawdb.ResettableFreezer, error) { - return rawdb.NewStateHistoryFreezer(datadir, readOnly) + return rawdb.NewStateFreezer(datadir, readOnly) } func compareSet[k comparable](a, b map[k][]byte) bool { diff --git a/trie/triedb/pathdb/testutils.go b/trie/triedb/pathdb/testutils.go index 3feb6217d2..674f52fd16 100644 --- a/trie/triedb/pathdb/testutils.go +++ b/trie/triedb/pathdb/testutils.go @@ -57,7 +57,7 @@ func newTestHasher(owner common.Hash, root common.Hash, cleans map[common.Hash][ } // Get returns the value for key stored in the trie. -func (h *testHasher) Get(key []byte) ([]byte, error) { +func (h *testHasher) TryGet(key []byte) ([]byte, error) { hash := common.BytesToHash(key) val, ok := h.dirties[hash] if ok { @@ -67,20 +67,20 @@ func (h *testHasher) Get(key []byte) ([]byte, error) { } // Update associates key with value in the trie. -func (h *testHasher) Update(key, value []byte) error { +func (h *testHasher) TryUpdate(key, value []byte) error { h.dirties[common.BytesToHash(key)] = common.CopyBytes(value) return nil } // Delete removes any existing value for key from the trie. -func (h *testHasher) Delete(key []byte) error { +func (h *testHasher) TryDelete(key []byte) error { h.dirties[common.BytesToHash(key)] = nil return nil } // Commit computes the new hash of the states and returns the set with all // state changes. -func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { +func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { var ( nodes = make(map[common.Hash][]byte) set = trienode.NewNodeSet(h.owner) @@ -108,7 +108,7 @@ func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { if root == types.EmptyRootHash && h.root != types.EmptyRootHash { set.AddNode(nil, trienode.NewDeleted()) } - return root, set + return root, set, nil } // hash performs the hash computation upon the provided states. diff --git a/trie/triestate/state.go b/trie/triestate/state.go index 0e00b67d78..2b2f3720d9 100644 --- a/trie/triestate/state.go +++ b/trie/triestate/state.go @@ -34,17 +34,17 @@ import ( // tree or Verkle tree. type Trie interface { // Get returns the value for key stored in the trie. - Get(key []byte) ([]byte, error) + TryGet(key []byte) ([]byte, error) // Update associates key with value in the trie. - Update(key, value []byte) error + TryUpdate(key, value []byte) error // Delete removes any existing value for key from the trie. - Delete(key []byte) error + TryDelete(key []byte) error // Commit the trie and returns a set of dirty nodes generated along with // the new root hash. - Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) + Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) } // TrieLoader wraps functions to load tries. @@ -131,7 +131,10 @@ func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Addre return nil, fmt.Errorf("failed to revert state, err: %w", err) } } - root, result := tr.Commit(false) + root, result, err := tr.Commit(false) + if err != nil { + return nil, err + } if root != prevRoot { return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root) } @@ -157,7 +160,7 @@ func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error { } // The account may or may not existent in post-state, try to // load it and decode if it's found. - blob, err := ctx.accountTrie.Get(addrHash.Bytes()) + blob, err := ctx.accountTrie.TryGet(addrHash.Bytes()) if err != nil { return err } @@ -175,15 +178,18 @@ func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error { for key, val := range ctx.storages[addr] { var err error if len(val) == 0 { - err = st.Delete(key.Bytes()) + err = st.TryDelete(key.Bytes()) } else { - err = st.Update(key.Bytes(), val) + err = st.TryUpdate(key.Bytes(), val) } if err != nil { return err } } - root, result := st.Commit(false) + root, result, err := st.Commit(false) + if err != nil { + return err + } if root != prev.Root { return errors.New("failed to reset storage trie") } @@ -199,7 +205,7 @@ func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error { if err != nil { return err } - return ctx.accountTrie.Update(addrHash.Bytes(), full) + return ctx.accountTrie.TryUpdate(addrHash.Bytes(), full) } // deleteAccount the account was not present in prev-state, and is expected @@ -211,7 +217,7 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error { defer h.release() addrHash := h.hash(addr.Bytes()) - blob, err := ctx.accountTrie.Get(addrHash.Bytes()) + blob, err := ctx.accountTrie.TryGet(addrHash.Bytes()) if err != nil { return err } @@ -230,11 +236,14 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error { if len(val) != 0 { return errors.New("expect storage deletion") } - if err := st.Delete(key.Bytes()); err != nil { + if err := st.TryDelete(key.Bytes()); err != nil { return err } } - root, result := st.Commit(false) + root, result, err := st.Commit(false) + if err != nil { + return err + } if root != types.EmptyRootHash { return errors.New("failed to clear storage trie") } @@ -246,7 +255,7 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error { } } // Delete the post-state account from the main trie. - return ctx.accountTrie.Delete(addrHash.Bytes()) + return ctx.accountTrie.TryDelete(addrHash.Bytes()) } // hasher is used to compute the sha256 hash of the provided data.