Skip to content

Commit

Permalink
Merge pull request #130 from blxdyx/fix-trie-root-bugs
Browse files Browse the repository at this point in the history 
Fix bugs in trie hash computation
  • Loading branch information
@blxdyx
blxdyx authored Jun 19, 2023
2 parents 1ad23c2 + 2dcfdfd commit 3093e11
Show file tree
Hide file tree
Showing 3 changed files with 489 additions and 59 deletions.
30 changes: 22 additions & 8 deletions turbo/trie/structural_branch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,15 @@ func TestIHCursor(t *testing.T) {
k, _, _, _ := ih.AtPrefix([]byte{})
require.Equal(common.FromHex("0001"), k)
require.False(ih.SkipState)
require.Equal([]byte{}, ih.FirstNotCoveredPrefix())
firstPrefix, done := ih.FirstNotCoveredPrefix()
require.Equal([]byte{}, firstPrefix)
require.False(done)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("0100"), k)
require.True(ih.SkipState)
require.Equal(common.FromHex("02"), ih.FirstNotCoveredPrefix())
firstPrefix, done = ih.FirstNotCoveredPrefix()
require.Equal(common.FromHex("02"), firstPrefix)
require.False(done)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("010100"), k)
require.True(ih.SkipState)
Expand All @@ -94,29 +98,39 @@ func TestIHCursor(t *testing.T) {
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("010102"), k)
require.True(ih.SkipState)
require.Equal(common.FromHex("1120"), ih.FirstNotCoveredPrefix())
firstPrefix, done = ih.FirstNotCoveredPrefix()
require.Equal(common.FromHex("1120"), firstPrefix)
require.False(done)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("0102"), k)
require.True(ih.SkipState)
require.Equal(common.FromHex("1130"), ih.FirstNotCoveredPrefix())
firstPrefix, done = ih.FirstNotCoveredPrefix()
require.Equal(common.FromHex("1130"), firstPrefix)
require.False(done)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("020f"), k)
require.True(ih.SkipState)
require.Equal(common.FromHex("13"), ih.FirstNotCoveredPrefix())
firstPrefix, done = ih.FirstNotCoveredPrefix()
require.Equal(common.FromHex("13"), firstPrefix)
require.False(done)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("03000000"), k)
require.True(ih.SkipState)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("03000e00000002"), k)
require.True(ih.SkipState)
require.Equal(common.FromHex("3001"), ih.FirstNotCoveredPrefix())
firstPrefix, done = ih.FirstNotCoveredPrefix()
require.Equal(common.FromHex("3001"), firstPrefix)
require.False(done)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("03000e00000e02"), k)
require.True(ih.SkipState)
require.Equal(common.FromHex("30e00030"), ih.FirstNotCoveredPrefix())
firstPrefix, done = ih.FirstNotCoveredPrefix()
require.Equal(common.FromHex("30e00030"), firstPrefix)
require.False(done)
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("05000100"), k)
require.True(ih.SkipState)
require.False(ih.SkipState) // Must be false, in case a key exists starting with nibble 4
k, _, _, _ = ih.Next()
require.Equal(common.FromHex("05000f00"), k)
require.True(ih.SkipState)
Expand Down
137 changes: 86 additions & 51 deletions turbo/trie/trie_root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,15 +224,20 @@ func (l *FlatDBTrieLoader) CalcTrieRoot(tx kv.Tx, quit <-chan struct{}) (libcomm
if err != nil {
return EmptyRoot, err
}
var firstPrefix []byte
var done bool
if accTrie.SkipState {
goto SkipAccounts
}

for k, kHex, v, err1 := accs.Seek(accTrie.FirstNotCoveredPrefix()); k != nil; k, kHex, v, err1 = accs.Next() {
firstPrefix, done = accTrie.FirstNotCoveredPrefix()
if done {
goto SkipAccounts
}
for k, kHex, v, err1 := accs.Seek(firstPrefix); k != nil; k, kHex, v, err1 = accs.Next() {
if err1 != nil {
return EmptyRoot, err1
}
if keyIsBefore(ihK, kHex) || !bytes.HasPrefix(kHex, nil) { // read all accounts until next AccTrie
if keyIsBefore(ihK, kHex) { // read all accounts until next AccTrie
break
}
if err = l.accountValue.DecodeForStorage(v); err != nil {
Expand All @@ -256,7 +261,11 @@ func (l *FlatDBTrieLoader) CalcTrieRoot(tx kv.Tx, quit <-chan struct{}) (libcomm
goto SkipStorage
}

for vS, err3 := ss.SeekBothRange(accWithInc, storageTrie.FirstNotCoveredPrefix()); vS != nil; _, vS, err3 = ss.NextDup() {
firstPrefix, done = storageTrie.FirstNotCoveredPrefix()
if done {
goto SkipStorage
}
for vS, err3 := ss.SeekBothRange(accWithInc, firstPrefix); vS != nil; _, vS, err3 = ss.NextDup() {
if err3 != nil {
return EmptyRoot, err3
}
Expand Down Expand Up @@ -667,7 +676,7 @@ func (r *RootHashAggregator) saveValueAccount(isIH, hasTree bool, v *accounts.Ac
// has 2 basic operations: _preOrderTraversalStep and _preOrderTraversalStepNoInDepth
type AccTrieCursor struct {
SkipState bool
is, lvl int
lvl int
k, v [64][]byte // store up to 64 levels of key/value pairs in nibbles format
hasState [64]uint16 // says that records in dbutil.HashedAccounts exists by given prefix
hasTree [64]uint16 // says that records in dbutil.TrieOfAccounts exists by given prefix
Expand Down Expand Up @@ -728,9 +737,10 @@ func (c *AccTrieCursor) _preOrderTraversalStepNoInDepth() error {
return nil
}

func (c *AccTrieCursor) FirstNotCoveredPrefix() []byte {
c.firstNotCoveredPrefix = firstNotCoveredPrefix(c.prev, c.prefix, c.firstNotCoveredPrefix)
return c.firstNotCoveredPrefix
func (c *AccTrieCursor) FirstNotCoveredPrefix() ([]byte, bool) {
var ok bool
c.firstNotCoveredPrefix, ok = firstNotCoveredPrefix(c.prev, c.prefix, c.firstNotCoveredPrefix)
return c.firstNotCoveredPrefix, ok
}

func (c *AccTrieCursor) AtPrefix(prefix []byte) (k, v []byte, hasTree bool, err error) {
Expand Down Expand Up @@ -844,29 +854,25 @@ func (c *AccTrieCursor) _nextSiblingInMem() bool {
}

func (c *AccTrieCursor) _nextSiblingOfParentInMem() bool {
originalLvl := c.lvl
for c.lvl > 1 {
if c.k[c.lvl-1] == nil {
nonNilLvl := c.lvl - 1
for c.k[nonNilLvl] == nil && nonNilLvl > 1 {
nonNilLvl--
}
c.next = append(append(c.next[:0], c.k[c.lvl]...), uint8(c.childID[c.lvl]))
c.kBuf = append(append(c.kBuf[:0], c.k[nonNilLvl]...), uint8(c.childID[nonNilLvl]))
ok, err := c._seek(c.next, c.kBuf)
if err != nil {
panic(err)
}
if ok {
return true
}

c.lvl = nonNilLvl + 1
c.lvl--
if c.k[c.lvl] == nil {
continue
}
c.lvl--
c.next = append(append(c.next[:0], c.k[originalLvl]...), uint8(c.childID[originalLvl]))
c.kBuf = append(append(c.kBuf[:0], c.k[c.lvl]...), uint8(c.childID[c.lvl]))
ok, err := c._seek(c.next, c.kBuf)
if err != nil {
panic(err)
}
if ok {
return true
}
if c._nextSiblingInMem() {
return true
}
originalLvl = c.lvl
}
return false
}
Expand All @@ -877,10 +883,14 @@ func (c *AccTrieCursor) _nextSiblingInDB() error {
c.k[c.lvl] = nil
return nil
}
c.is++
if _, err := c._seek(c.next, []byte{}); err != nil {
return err
}
if c.k[c.lvl] == nil || !bytes.HasPrefix(c.next, c.k[c.lvl]) {
// If the cursor has moved beyond the next subtree, we need to check to make
// sure that any modified keys in between are processed.
c.SkipState = false
}
return nil
}

Expand All @@ -889,6 +899,17 @@ func (c *AccTrieCursor) _unmarshal(k, v []byte) {
if c.lvl >= len(k) {
from, to = len(k)+1, c.lvl+2
}
// Consider a trie DB with keys like: [0xa, 0xbb], then unmarshaling 0xbb
// needs to nil the existing 0xa key entry, as it is no longer a parent.
for i := from - 1; i > 0; i-- {
if c.k[i] == nil {
continue
}
if bytes.HasPrefix(k, c.k[i]) {
break
}
from = i
}
for i := from; i < to; i++ { // if first meet key is not 0 length, then nullify all shorter metadata
c.k[i], c.hasState[i], c.hasTree[i], c.hasHash[i], c.hashID[i], c.childID[i], c.deleted[i] = nil, 0, 0, 0, 0, 0, false
}
Expand Down Expand Up @@ -975,7 +996,7 @@ func (c *AccTrieCursor) _next() (k, v []byte, hasTree bool, err error) {

// StorageTrieCursor - holds logic related to iteration over AccTrie bucket
type StorageTrieCursor struct {
is, lvl int
lvl int
k, v [64][]byte
hasState, hasTree, hasHash [64]uint16
deleted [64]bool
Expand Down Expand Up @@ -1014,9 +1035,10 @@ func (c *StorageTrieCursor) PrevKey() []byte {
return c.prev
}

func (c *StorageTrieCursor) FirstNotCoveredPrefix() []byte {
c.firstNotCoveredPrefix = firstNotCoveredPrefix(c.prev, []byte{0, 0}, c.firstNotCoveredPrefix)
return c.firstNotCoveredPrefix
func (c *StorageTrieCursor) FirstNotCoveredPrefix() ([]byte, bool) {
var ok bool
c.firstNotCoveredPrefix, ok = firstNotCoveredPrefix(c.prev, []byte{0, 0}, c.firstNotCoveredPrefix)
return c.firstNotCoveredPrefix, ok
}

func (c *StorageTrieCursor) SeekToAccount(accWithInc []byte) (k, v []byte, hasTree bool, err error) {
Expand Down Expand Up @@ -1112,7 +1134,6 @@ func (c *StorageTrieCursor) _seek(seek, withinPrefix []byte) (bool, error) {
var k, v []byte
var err error
if len(seek) == 40 {
c.is++
k, v, err = c.c.Seek(seek)
} else {
// optimistic .Next call, can use result in 2 cases:
Expand All @@ -1124,7 +1145,6 @@ func (c *StorageTrieCursor) _seek(seek, withinPrefix []byte) (bool, error) {
// return false, err
//}
//if len(k) > c.lvl && c.childID[c.lvl] > int8(bits.TrailingZeros16(c.hasTree[c.lvl])) {
c.is++
k, v, err = c.c.Seek(seek)
//}
}
Expand Down Expand Up @@ -1208,28 +1228,25 @@ func (c *StorageTrieCursor) _nextSiblingInMem() bool {
}

func (c *StorageTrieCursor) _nextSiblingOfParentInMem() bool {
originalLvl := c.lvl
for c.lvl > 0 {
if c.k[c.lvl-1] == nil {
nonNilLvl := c.lvl - 1
for ; c.k[nonNilLvl] == nil && nonNilLvl > 0; nonNilLvl-- {
}
c.seek = append(append(c.seek[:40], c.k[c.lvl]...), uint8(c.childID[c.lvl]))
c.next = append(append(c.next[:0], c.k[nonNilLvl]...), uint8(c.childID[nonNilLvl]))
ok, err := c._seek(c.seek, c.next)
if err != nil {
panic(err)
}
if ok {
return true
}

c.lvl = nonNilLvl + 1
c.lvl--
if c.k[c.lvl] == nil {
continue
}
c.lvl--
c.seek = append(append(c.seek[:40], c.k[originalLvl]...), uint8(c.childID[originalLvl]))
c.next = append(append(c.next[:0], c.k[c.lvl]...), uint8(c.childID[c.lvl]))
ok, err := c._seek(c.seek, c.next)
if err != nil {
panic(err)
}
if ok {
return true
}
if c._nextSiblingInMem() {
return true
}
originalLvl = c.lvl
}
return false
}
Expand All @@ -1244,6 +1261,11 @@ func (c *StorageTrieCursor) _nextSiblingInDB() error {
if _, err := c._seek(c.seek, []byte{}); err != nil {
return err
}
if c.k[c.lvl] == nil || !bytes.HasPrefix(c.next, c.k[c.lvl]) {
// If the cursor has moved beyond the next subtree, we need to check to make
// sure that any modified keys in between are processed.
c.skipState = false
}
return nil
}

Expand Down Expand Up @@ -1285,6 +1307,17 @@ func (c *StorageTrieCursor) _unmarshal(k, v []byte) {
if c.lvl >= len(k) {
from, to = len(k)+1, c.lvl+2
}
// Consider a trie DB with keys like: [0xa, 0xbb], then unmarshaling 0xbb
// needs to nil the existing 0xa key entry, as it is no longer a parent.
for i := from - 1; i > 0; i-- {
if c.k[i] == nil {
continue
}
if bytes.HasPrefix(k[40:], c.k[i]) {
break
}
from = i
}
for i := from; i < to; i++ { // if first meet key is not 0 length, then nullify all shorter metadata
c.k[i], c.hasState[i], c.hasTree[i], c.hasHash[i], c.hashID[i], c.childID[i], c.deleted[i] = nil, 0, 0, 0, 0, 0, false
}
Expand Down Expand Up @@ -1356,17 +1389,19 @@ func isDenseSequence(prev []byte, next []byte) bool {

var isSequenceBuf = make([]byte, 256)

func firstNotCoveredPrefix(prev, prefix, buf []byte) []byte {
func firstNotCoveredPrefix(prev, prefix, buf []byte) ([]byte, bool) {
if len(prev) > 0 {
_ = dbutils.NextNibblesSubtree(prev, &buf)
if !dbutils.NextNibblesSubtree(prev, &buf) {
return buf, true
}
} else {
buf = append(buf[:0], prefix...)
}
if len(buf)%2 == 1 {
buf = append(buf, 0)
}
hexutil.CompressNibbles(buf, &buf)
return buf
return buf, false
}

type StateCursor struct {
Expand Down
Loading

0 comments on commit 3093e11

Please sign in to comment.