Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: add the wrapper layer of versa #2636

Open
wants to merge 93 commits into
base: versa_base
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
93 commits
Select commit Hold shift + click to select a range
41ab497
feat: setup versa db
joeylichang Aug 9, 2024
d2e5c03
feat: add version commit
joeylichang Aug 12, 2024
435410f
chore: forbid versiondb rewind
joeylichang Aug 12, 2024
7f3aac3
feat: add mode for new caching db
joeylichang Aug 12, 2024
6cf5169
feat: add version scheme for startup
joeylichang Aug 12, 2024
e1a2b8e
feat: init genesis
joeylichang Aug 12, 2024
4635a45
feat: add caching db and trie copy
joeylichang Aug 12, 2024
3a34c73
feat: support has state on statedb and must fullsync under version
joeylichang Aug 12, 2024
6b7c807
chore: add debug code
joeylichang Aug 12, 2024
66bd590
chore: delete debug code and add log
joeylichang Aug 12, 2024
bb167a9
chore: add stack debug
joeylichang Aug 12, 2024
60df2fa
fix: open strorage trie param error
joeylichang Aug 12, 2024
6188aae
chore: add debug print statedb
joeylichang Aug 13, 2024
ed7aa3e
fix: storage tree value encode
joeylichang Aug 13, 2024
e8309ad
chore: delete debug print code
joeylichang Aug 13, 2024
d8c3726
fix: state object trie expire
joeylichang Aug 13, 2024
b00582c
chore: add state info debug log
joeylichang Aug 13, 2024
d162a63
chore: change panic to log
joeylichang Aug 13, 2024
83287ce
chore: add debug log
joeylichang Aug 13, 2024
ec2384a
fix: change check from tree root to tree handler
joeylichang Aug 13, 2024
04e5c17
chore: add debug panic
joeylichang Aug 13, 2024
f69ff95
fix: blockchain stateat use rw state
joeylichang Aug 13, 2024
657eced
chore: delete panic code
joeylichang Aug 13, 2024
2f0f91d
chore: delete debug log
joeylichang Aug 13, 2024
58dabc9
chore: add apply tx debug log
joeylichang Aug 13, 2024
324e4a4
chore: add panic log
joeylichang Aug 13, 2024
c572535
chore: add reset panic
joeylichang Aug 13, 2024
5b4e00c
fix: decode storage val
joeylichang Aug 13, 2024
f284358
chore: delete redundant code
joeylichang Aug 14, 2024
3a70b35
chore: dadd debug log for tx diff
joeylichang Aug 14, 2024
412c5ee
chore: add state obj debug info
joeylichang Aug 14, 2024
bfe07c5
chore: adjust state obj debug info
joeylichang Aug 14, 2024
be0a305
chore: add delete obj debug info
joeylichang Aug 14, 2024
26d7f62
chore: forbid prefetcher
joeylichang Aug 14, 2024
b73a98c
chore: forbid prefetcher
joeylichang Aug 14, 2024
0941eb6
chore: panic for snap
joeylichang Aug 14, 2024
023b092
chore: delete storage pool
joeylichang Aug 14, 2024
bd9c2ae
chore: add tx hash log
joeylichang Aug 14, 2024
a3f8c70
chore: fix build error
joeylichang Aug 14, 2024
8d7ee3a
chore: add parse state obj
joeylichang Aug 14, 2024
42b72fe
chore: add debug state after tx execute
joeylichang Aug 14, 2024
6dfcbc6
fix lint error
joeylichang Aug 14, 2024
bbef397
chore:add tree get debug info
joeylichang Aug 15, 2024
dec27d1
fix: add address check for get versa tree
joeylichang Aug 15, 2024
cf04af0
chore: add tx execute log
joeylichang Aug 15, 2024
7462078
chore: add system tx log
joeylichang Aug 15, 2024
c6d8d63
fix: get code error
joeylichang Aug 15, 2024
ac2d1a8
chore: add versa db error log
joeylichang Aug 15, 2024
2e38d06
chore: change panic to log error
joeylichang Aug 15, 2024
0a90118
chore: delete part log
joeylichang Aug 15, 2024
2e4ec39
chore: add owner and hash log
joeylichang Aug 15, 2024
7bce07a
chore: adjust state reset
joeylichang Aug 16, 2024
dc51003
feat: return version for repeat search account tree
joeylichang Aug 16, 2024
9d20658
chore: debug 24611
joeylichang Aug 16, 2024
77dea68
chore: add detete key debug log
joeylichang Aug 16, 2024
90de1d7
fix: version mismatch that add contract balance without update storag…
joeylichang Aug 16, 2024
8197064
fix: chang condition for ca
joeylichang Aug 16, 2024
3c4b452
fix: chang condition for ca
joeylichang Aug 16, 2024
b80d787
chore: add update storage debug info
joeylichang Aug 16, 2024
bf80891
chore: add update storage debug info
joeylichang Aug 16, 2024
84767d1
chore: add update storage debug info
joeylichang Aug 16, 2024
f67f494
chore: add 373559 debug breakpoint
joeylichang Aug 16, 2024
23c60a3
feat: add version state debug system
joeylichang Aug 18, 2024
e940e28
fix: access nil pointer
joeylichang Aug 18, 2024
90aa2e1
fix: change pebble to default db
joeylichang Aug 18, 2024
997d7b5
chore: delete debug log
joeylichang Aug 18, 2024
1cbdf0a
feat: add hash state db debug system
joeylichang Aug 19, 2024
7d5e6ce
feat: add version and hash state diff system
joeylichang Aug 19, 2024
55cdcfa
fix: repeat cmd error
joeylichang Aug 19, 2024
a0daf10
fix: genesis init panic
joeylichang Aug 19, 2024
aa9d518
fix: panic bug
joeylichang Aug 19, 2024
ef3f9df
fix: repeated cmd error
joeylichang Aug 19, 2024
f724941
fix: add protection for debug hash state
joeylichang Aug 19, 2024
3ad4700
fix: diff debug state error
joeylichang Aug 19, 2024
1a40f79
fix: set path mode breakpoint
joeylichang Aug 19, 2024
fa7a43a
fix: add calc hash point
joeylichang Aug 19, 2024
4764fdb
fix: the timing release debug state instance
joeylichang Aug 19, 2024
30b371d
chore: compatable empty hash
joeylichang Aug 19, 2024
d3854b4
fix: compare funx
joeylichang Aug 19, 2024
9c239db
fix: compare func
joeylichang Aug 19, 2024
74ab9ab
fix: changte the confidition for open storage tree
joeylichang Aug 19, 2024
3ad7c4e
fix: panic for updating code
joeylichang Aug 19, 2024
b8fa476
fix: get ca trie
joeylichang Aug 20, 2024
ea76d5f
chore: add debug inof for get trie version mismatch error
joeylichang Aug 20, 2024
a1d4b06
feat: check ca changeable
joeylichang Aug 20, 2024
b585984
fix: remove enable hash check
fynnss Aug 20, 2024
d5a438a
chore: add diff and disk version log
joeylichang Aug 20, 2024
d5e38ad
chore: adjust log confidition
joeylichang Aug 21, 2024
36044b0
chore: add open state version
joeylichang Aug 21, 2024
8db789b
fix: update trie error
joeylichang Aug 22, 2024
85a6b17
chore: add reopen same tree error debug info
joeylichang Aug 22, 2024
84da730
chore: add pprof service
joeylichang Aug 29, 2024
facbe6a
chore: add mgasps metrics
joeylichang Aug 29, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/geth/chaincmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -833,7 +833,7 @@ func dump(ctx *cli.Context) error {
triedb := utils.MakeTrieDatabase(ctx, stack, db, true, true, false) // always enable preimage lookup
defer triedb.Close()

state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb, false), nil)
if err != nil {
return err
}
Expand Down
138 changes: 138 additions & 0 deletions cmd/geth/dbcmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package main

import (
"bytes"
"encoding/json"
"errors"
"fmt"
"math"
Expand All @@ -34,6 +35,7 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/console/prompt"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
Expand Down Expand Up @@ -91,6 +93,9 @@ Remove blockchain and state databases`,
dbHbss2PbssCmd,
dbTrieGetCmd,
dbTrieDeleteCmd,
getVersionDBState,
getHashDBState,
diffDebugStateDB,
},
}
dbInspectCmd = &cli.Command{
Expand Down Expand Up @@ -286,8 +291,141 @@ WARNING: This is a low-level operation which may cause database corruption!`,
Description: `This commands will read current offset from kvdb, which is the current offset and starting BlockNumber
of ancientStore, will also displays the reserved number of blocks in ancientStore `,
}
getVersionDBState = &cli.Command{
Action: getDebugVersionState,
Name: "get-debug-version-state",
Flags: []cli.Flag{
utils.VersionStateDirFlag,
utils.BlockNumber,
},
}
getHashDBState = &cli.Command{
Action: getDebugHashState,
Name: "get-debug-hash-state",
Flags: []cli.Flag{
utils.HashStateDirFlag,
utils.BlockNumber,
},
}

diffDebugStateDB = &cli.Command{
Action: diffDebugState,
Name: "diff-debug-state",
Flags: []cli.Flag{
utils.VersionStateDirFlag,
utils.HashStateDirFlag,
utils.BlockNumber,
},
}
)

func diffDebugState(ctx *cli.Context) error {
if !ctx.IsSet(utils.VersionStateDirFlag.Name) {
return fmt.Errorf("please set `--versionstatedir` flag")
}
if !ctx.IsSet(utils.BlockNumber.Name) {
return fmt.Errorf("please set `--block` flag")
}
if !ctx.IsSet(utils.HashStateDirFlag.Name) {
return fmt.Errorf("please set `--hashstatedir` flag")
}
verDir := ctx.String(utils.VersionStateDirFlag.Name)
hasDir := ctx.String(utils.HashStateDirFlag.Name)
block := ctx.Int64(utils.BlockNumber.Name)

vdb, err := rawdb.Open(rawdb.OpenOptions{
ReadOnly: true,
Type: "pebble",
Directory: verDir,
})
if err != nil {
return err
}
verData, err := vdb.Get(state.DebugVersionStateKey(block))
if err != nil {
return err
}

verDebugState := &state.DebugVersionState{}
err = json.Unmarshal(verData, verDebugState)
if err != nil {
return nil
}

hdb, err := rawdb.Open(rawdb.OpenOptions{
ReadOnly: true,
Type: "pebble",
Directory: hasDir,
})
if err != nil {
return err
}
hashData, err := hdb.Get(state.DebugHashStateKey(block))
if err != nil {
return err
}
hasDebugState := &state.DebugHashState{}
err = json.Unmarshal(hashData, hasDebugState)
if err != nil {
return err
}

res := state.GenerateDebugStateDiff(verDebugState, hasDebugState)
fmt.Println(res)

return nil
}

func getDebugVersionState(ctx *cli.Context) error {
if !ctx.IsSet(utils.VersionStateDirFlag.Name) {
return fmt.Errorf("please set `--versionstatedir` flag")
}
if !ctx.IsSet(utils.BlockNumber.Name) {
return fmt.Errorf("please set `--block` flag")
}
dir := ctx.String(utils.VersionStateDirFlag.Name)
block := ctx.Int64(utils.BlockNumber.Name)
db, err := rawdb.Open(rawdb.OpenOptions{
ReadOnly: true,
Type: "pebble",
Directory: dir,
})
if err != nil {
return err
}
data, err := db.Get(state.DebugVersionStateKey(block))
if err != nil {
return err
}
fmt.Println(string(data))
return nil
}

func getDebugHashState(ctx *cli.Context) error {
if !ctx.IsSet(utils.HashStateDirFlag.Name) {
return fmt.Errorf("please set `--hashstatedir` flag")
}
if !ctx.IsSet(utils.BlockNumber.Name) {
return fmt.Errorf("please set `--block` flag")
}
dir := ctx.String(utils.HashStateDirFlag.Name)
block := ctx.Int64(utils.BlockNumber.Name)
db, err := rawdb.Open(rawdb.OpenOptions{
ReadOnly: true,
Type: "pebble",
Directory: dir,
})
if err != nil {
return err
}
data, err := db.Get(state.DebugHashStateKey(block))
if err != nil {
return err
}
fmt.Println(string(data))
return nil
}

func removeDB(ctx *cli.Context) error {
stack, config := makeConfigNode(ctx)

Expand Down
2 changes: 2 additions & 0 deletions cmd/geth/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,8 @@ func geth(ctx *cli.Context) error {
defer stack.Close()

startNode(ctx, stack, backend, false)
// TODO:: debug code , will be deleted in the future
debug.StartPProf("127.0.0.1:7060", !ctx.IsSet("metrics.addr"))
stack.Wait()
return nil
}
Expand Down
35 changes: 30 additions & 5 deletions cmd/utils/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ var (
}
StateSchemeFlag = &cli.StringFlag{
Name: "state.scheme",
Usage: "Scheme to use for storing ethereum state ('hash' or 'path')",
Usage: "Scheme to use for storing ethereum state ('hash', 'path', 'version')",
Category: flags.StateCategory,
}
PathDBSyncFlag = &cli.BoolFlag{
Expand Down Expand Up @@ -1135,6 +1135,25 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
Value: params.DefaultExtraReserveForBlobRequests,
Category: flags.MiscCategory,
}

BlockNumber = &cli.Int64Flag{
Name: "block",
Value: int64(0),
}

VersionStateDirFlag = &flags.DirectoryFlag{
Name: "versionstatedir",
Usage: "Data directory for the version databases and keystore",
Value: flags.DirectoryString(node.DefaultDataDir()),
Category: flags.EthCategory,
}

HashStateDirFlag = &flags.DirectoryFlag{
Name: "hashstatedir",
Usage: "Data directory for the version databases and keystore",
Value: flags.DirectoryString(node.DefaultDataDir()),
Category: flags.EthCategory,
}
)

var (
Expand Down Expand Up @@ -1953,11 +1972,17 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(StateHistoryFlag.Name) {
cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
}
scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
if err != nil {
Fatalf("%v", err)
if ctx.String(StateSchemeFlag.Name) != rawdb.VersionScheme {
scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
if err != nil {
Fatalf("%v", err)
}
cfg.StateScheme = scheme
} else {
// TODO:: compatible with cli line and configuration file, currently only supports cli.
cfg.StateScheme = rawdb.VersionScheme
}
cfg.StateScheme = scheme

// Parse transaction history flag, if user is still using legacy config
// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
Expand Down
Loading
Loading