diff --git a/common/dbutils/history_index.go b/common/dbutils/history_index.go
index 7f7806d3d7fba93248507688c271c33b0e8dc527..660b197efd7ba3a9ea72f5f4c45eeb1e03e2ebbb 100644
--- a/common/dbutils/history_index.go
+++ b/common/dbutils/history_index.go
@@ -17,46 +17,45 @@ const (
MaxChunkSize = 1000
)
-func NewHistoryIndex() *HistoryIndexBytes {
- b := make(HistoryIndexBytes, LenBytes*2, 16)
- return &b
+func NewHistoryIndex() HistoryIndexBytes {
+ return make(HistoryIndexBytes, LenBytes*2, 16)
}
-func WrapHistoryIndex(b []byte) *HistoryIndexBytes {
+func WrapHistoryIndex(b []byte) HistoryIndexBytes {
index := HistoryIndexBytes(b)
if len(index) == 0 {
index = make(HistoryIndexBytes, LenBytes*2, 16)
}
- return &index
+ return index
}
type HistoryIndexBytes []byte
-func (hi *HistoryIndexBytes) Decode() ([]uint64, error) {
+func (hi HistoryIndexBytes) Decode() ([]uint64, error) {
if hi == nil {
return []uint64{}, nil
}
- if len(*hi) <= LenBytes*2 {
+ if len(hi) <= LenBytes*2 {
return []uint64{}, nil
}
- numOfElements := binary.LittleEndian.Uint32((*hi)[0:LenBytes])
- numOfUint32Elements := binary.LittleEndian.Uint32((*hi)[LenBytes : 2*LenBytes])
+ numOfElements := binary.LittleEndian.Uint32(hi[0:LenBytes])
+ numOfUint32Elements := binary.LittleEndian.Uint32(hi[LenBytes : 2*LenBytes])
decoded := make([]uint64, numOfElements)
for i := uint32(0); i < numOfElements; i++ {
if i < numOfUint32Elements {
- decoded[i] = uint64(binary.LittleEndian.Uint32((*hi)[LenBytes*2+i*4 : LenBytes*2+i*4+4]))
+ decoded[i] = uint64(binary.LittleEndian.Uint32(hi[LenBytes*2+i*4 : LenBytes*2+i*4+4]))
} else {
- decoded[i] = binary.LittleEndian.Uint64((*hi)[LenBytes*2+numOfUint32Elements*4+i*ItemLen : LenBytes*2+i*ItemLen+ItemLen])
+ decoded[i] = binary.LittleEndian.Uint64(hi[LenBytes*2+numOfUint32Elements*4+i*ItemLen : LenBytes*2+i*ItemLen+ItemLen])
}
}
return decoded, nil
}
-func (hi *HistoryIndexBytes) Append(v uint64) *HistoryIndexBytes {
- numOfElements := binary.LittleEndian.Uint32((*hi)[0:LenBytes])
- numOfUint32Elements := binary.LittleEndian.Uint32((*hi)[LenBytes : 2*LenBytes])
+func (hi HistoryIndexBytes) Append(v uint64) HistoryIndexBytes {
+ numOfElements := binary.LittleEndian.Uint32(hi[0:LenBytes])
+ numOfUint32Elements := binary.LittleEndian.Uint32(hi[LenBytes : 2*LenBytes])
var b []byte
if v < math.MaxUint32 {
b = make([]byte, 4)
@@ -67,9 +66,9 @@ func (hi *HistoryIndexBytes) Append(v uint64) *HistoryIndexBytes {
binary.LittleEndian.PutUint64(b, v)
}
- *hi = append(*hi, b...)
- binary.LittleEndian.PutUint32((*hi)[0:LenBytes], numOfElements+1)
- binary.LittleEndian.PutUint32((*hi)[LenBytes:2*LenBytes], numOfUint32Elements)
+ hi = append(hi, b...)
+ binary.LittleEndian.PutUint32(hi[0:LenBytes], numOfElements+1)
+ binary.LittleEndian.PutUint32(hi[LenBytes:2*LenBytes], numOfUint32Elements)
return hi
}
@@ -78,9 +77,9 @@ func (hi HistoryIndexBytes) Len() uint32 {
}
//most common operation is remove one from the tail
-func (hi *HistoryIndexBytes) Remove(v uint64) *HistoryIndexBytes {
- numOfElements := binary.LittleEndian.Uint32((*hi)[0:LenBytes])
- numOfUint32Elements := binary.LittleEndian.Uint32((*hi)[LenBytes : 2*LenBytes])
+func (hi HistoryIndexBytes) Remove(v uint64) HistoryIndexBytes {
+ numOfElements := binary.LittleEndian.Uint32(hi[0:LenBytes])
+ numOfUint32Elements := binary.LittleEndian.Uint32(hi[LenBytes : 2*LenBytes])
var currentElement uint64
var elemEnd uint32
@@ -90,17 +89,17 @@ Loop:
for i := numOfElements; i > 0; i-- {
if i > numOfUint32Elements {
elemEnd = LenBytes*2 + numOfUint32Elements*4 + (i-numOfUint32Elements)*8
- currentElement = binary.LittleEndian.Uint64((*hi)[elemEnd-8 : elemEnd])
+ currentElement = binary.LittleEndian.Uint64(hi[elemEnd-8 : elemEnd])
itemLen = 8
} else {
elemEnd = LenBytes*2 + i*4
- currentElement = uint64(binary.LittleEndian.Uint32((*hi)[elemEnd-4 : elemEnd]))
+ currentElement = uint64(binary.LittleEndian.Uint32(hi[elemEnd-4 : elemEnd]))
itemLen = 4
}
switch {
case currentElement == v:
- *hi = append((*hi)[:elemEnd-itemLen], (*hi)[elemEnd:]...)
+ hi = append(hi[:elemEnd-itemLen], hi[elemEnd:]...)
numOfElements--
if itemLen == 4 {
numOfUint32Elements--
@@ -111,8 +110,8 @@ Loop:
continue
}
}
- binary.LittleEndian.PutUint32((*hi)[0:LenBytes], numOfElements)
- binary.LittleEndian.PutUint32((*hi)[LenBytes:2*LenBytes], numOfUint32Elements)
+ binary.LittleEndian.PutUint32(hi[0:LenBytes], numOfElements)
+ binary.LittleEndian.PutUint32(hi[LenBytes:2*LenBytes], numOfUint32Elements)
return hi
}
@@ -185,14 +184,7 @@ func IndexChunkKey(key []byte, blockNumber uint64) []byte {
func IsIndexBucket(b []byte) bool {
return bytes.Equal(b, AccountsHistoryBucket) || bytes.Equal(b, StorageHistoryBucket)
}
-func IsFirstChunk(b []byte) bool {
- if len(b) == common.HashLength+8 {
- return binary.BigEndian.Uint64(b[common.HashLength:]) == ^uint64(0)
- } else if len(b) == common.HashLength*2+8 {
- return binary.BigEndian.Uint64(b[common.HashLength*2:]) == ^uint64(0)
- }
- panic("unexpected length" + strconv.Itoa(len(b)))
-}
+
func CheckNewIndexChunk(b []byte) bool {
return len(b)+8 > MaxChunkSize
}
diff --git a/common/dbutils/history_index_test.go b/common/dbutils/history_index_test.go
index 3b1932023a4973461966b6dd356cd380a1914d5d..89f04c613db546b8462d9d366f9fa38894a748d7 100644
--- a/common/dbutils/history_index_test.go
+++ b/common/dbutils/history_index_test.go
@@ -7,8 +7,7 @@ import (
)
func TestHistoryIndex_Search1(t *testing.T) {
- index := NewHistoryIndex()
- index.Append(3).Append(5).Append(8)
+ index := NewHistoryIndex().Append(3).Append(5).Append(8)
fmt.Println(index.Decode())
v, _ := index.Search(1)
if v != 3 {
@@ -53,7 +52,7 @@ func TestHistoryIndex_Search_EmptyIndex(t *testing.T) {
func TestHistoryIndex_Append(t *testing.T) {
index := NewHistoryIndex()
for i := uint64(1); i < 10; i++ {
- index.Append(i)
+ index = index.Append(i)
}
res, err := index.Decode()
@@ -68,7 +67,7 @@ func TestHistoryIndex_Append(t *testing.T) {
t.Fatal()
}
- index.Remove(9)
+ index = index.Remove(9)
res, err = index.Decode()
if err != nil {
t.Fatal(err)
@@ -82,7 +81,7 @@ func TestHistoryIndex_Append(t *testing.T) {
t.Fatal("Not equal")
}
- index.Remove(5)
+ index = index.Remove(5)
res, err = index.Decode()
if err != nil {
t.Fatal(err)
@@ -96,7 +95,7 @@ func TestHistoryIndex_Append(t *testing.T) {
t.Fatal("Not equal")
}
- index.Remove(1)
+ index = index.Remove(1)
res, err = index.Decode()
if err != nil {
t.Fatal(err)
diff --git a/core/generate_index.go b/core/generate_index.go
index a76cf830ad3a6b9db27037113d7b9235134517d7..cd85c25a44c82a003703dde25b70be2de1ff7f08 100644
--- a/core/generate_index.go
+++ b/core/generate_index.go
@@ -35,7 +35,7 @@ type IndexGenerator struct {
}
type IndexWithKey struct {
- Val *dbutils.HistoryIndexBytes
+ Val dbutils.HistoryIndexBytes
}
func (ig *IndexGenerator) changeSetWalker(blockNum uint64) func([]byte, []byte) error {
@@ -47,7 +47,7 @@ func (ig *IndexGenerator) changeSetWalker(blockNum uint64) func([]byte, []byte)
if err != nil && err != ethdb.ErrKeyNotFound {
return err
}
- var index *dbutils.HistoryIndexBytes
+ var index dbutils.HistoryIndexBytes
if len(indexBytes) == 0 {
index = dbutils.NewHistoryIndex()
@@ -58,18 +58,18 @@ func (ig *IndexGenerator) changeSetWalker(blockNum uint64) func([]byte, []byte)
}
indexes = append(indexes, IndexWithKey{
- Val: index,
+ Val: index,
})
ig.cache[string(k)] = indexes
}
lastIndex := indexes[len(indexes)-1]
- if dbutils.CheckNewIndexChunk(*lastIndex.Val) {
+ if dbutils.CheckNewIndexChunk(lastIndex.Val) {
lastIndex.Val = dbutils.NewHistoryIndex()
indexes = append(indexes, lastIndex)
ig.cache[string(k)] = indexes
}
- lastIndex.Val.Append(blockNum)
+ lastIndex.Val = lastIndex.Val.Append(blockNum)
return nil
}
}
@@ -109,7 +109,7 @@ func (ig *IndexGenerator) GenerateIndex() error {
return err
}
- if err := tuples.Append(ig.bucketToWrite, chunkKey, *val.Val); err != nil {
+ if err := tuples.Append(ig.bucketToWrite, chunkKey, val.Val); err != nil {
return err
}
diff --git a/core/state/database.go b/core/state/database.go
index 1b705d92d4a5ff481c9e884686a6f96041cf1e5e..a470ef670ff09163cc4f58fd5c8f3c3ce4fcab5f 100644
--- a/core/state/database.go
+++ b/core/state/database.go
@@ -945,12 +945,12 @@ func (tds *TrieDbState) deleteTimestamp(timestamp uint64) error {
if err != nil {
return err
}
- index.Remove(timestamp)
+ index = index.Remove(timestamp)
if index.Len() == 0 {
return tds.db.Delete(dbutils.AccountsHistoryBucket, chunkKey)
}
- return tds.db.Put(dbutils.AccountsHistoryBucket, chunkKey, *index)
+ return tds.db.Put(dbutils.AccountsHistoryBucket, chunkKey, index)
})
if innerErr != nil {
return innerErr
@@ -976,12 +976,12 @@ func (tds *TrieDbState) deleteTimestamp(timestamp uint64) error {
return err
}
- index.Remove(timestamp)
+ index = index.Remove(timestamp)
if index.Len() == 0 {
return tds.db.Delete(dbutils.StorageHistoryBucket, chunkKey)
}
- return tds.db.Put(dbutils.StorageHistoryBucket, chunkKey, *index)
+ return tds.db.Put(dbutils.StorageHistoryBucket, chunkKey, index)
})
if innerErr != nil {
return innerErr
diff --git a/core/state/db_state_writer.go b/core/state/db_state_writer.go
index 0019878e5df8eaf13744df240b844d69e7369011..5f558252527f5f4cf5e6980e896d30ece75890f4 100644
--- a/core/state/db_state_writer.go
+++ b/core/state/db_state_writer.go
@@ -171,28 +171,35 @@ func (dsw *DbStateWriter) WriteHistory() error {
func (dsw *DbStateWriter) writeIndex(changes *changeset.ChangeSet, bucket []byte) error {
for _, change := range changes.Changes {
- indexBytes, err := dsw.tds.db.GetIndexChunk(bucket, change.Key, dsw.tds.blockNr)
+ currentChunkKey := dbutils.IndexChunkKey(change.Key, ^uint64(0))
+ indexBytes, err := dsw.tds.db.Get(bucket, currentChunkKey)
if err != nil && err != ethdb.ErrKeyNotFound {
return fmt.Errorf("find chunk failed: %w", err)
}
- var index *dbutils.HistoryIndexBytes
+ var index dbutils.HistoryIndexBytes
if len(indexBytes) == 0 {
index = dbutils.NewHistoryIndex()
} else if dbutils.CheckNewIndexChunk(indexBytes) {
+ // Chunk overflow, need to write the "old" current chunk under its key derived from the last element
+ index = dbutils.WrapHistoryIndex(indexBytes)
+ indexKey, err := index.Key(change.Key)
+ if err != nil {
+ return err
+ }
+ // Flush the old chunk
+ if err := dsw.tds.db.Put(bucket, indexKey, index); err != nil {
+ return err
+ }
+ // Start a new chunk
index = dbutils.NewHistoryIndex()
} else {
index = dbutils.WrapHistoryIndex(indexBytes)
}
- index.Append(dsw.tds.blockNr)
-
- indexKey, err := index.Key(change.Key)
- if err != nil {
- return err
- }
+ index = index.Append(dsw.tds.blockNr)
- if err := dsw.tds.db.Put(bucket, indexKey, *index); err != nil {
+ if err := dsw.tds.db.Put(bucket, currentChunkKey, index); err != nil {
return err
}
}
diff --git a/ethdb/bolt_db.go b/ethdb/bolt_db.go
index b8bfb2bce432cbe860ef1c5cd96302e778adea19..4179a37a899dae6852f95613a21ae3a03772ea0b 100644
--- a/ethdb/bolt_db.go
+++ b/ethdb/bolt_db.go
@@ -374,6 +374,9 @@ func (db *BoltDatabase) walkAsOfThinAccounts(startkey []byte, fixedbits uint, ti
common.HashLength+8, /* part3start */
)
k, v := mainCursor.Seek(startkey)
+ for k != nil && len(k) > common.HashLength {
+ k, v = mainCursor.Next()
+ }
hK, tsEnc, _, hV := historyCursor.Seek()
if hK != nil {
ts := binary.BigEndian.Uint64(tsEnc)
@@ -431,20 +434,22 @@ func (db *BoltDatabase) walkAsOfThinAccounts(startkey []byte, fixedbits uint, ti
timestamp,
)
}
- var acc accounts.Account
- if err2 := acc.DecodeForStorage(data); err2 != nil {
- return err2
- }
- if acc.Incarnation > 0 && acc.IsEmptyCodeHash() {
- codeBucket := tx.Bucket(dbutils.ContractCodeBucket)
- codeHash, _ := codeBucket.Get(dbutils.GenerateStoragePrefix(common.BytesToHash(hK), acc.Incarnation))
- if len(codeHash) > 0 {
- acc.CodeHash = common.BytesToHash(codeHash)
+ if len(data) > 0 { // Skip accounts did not exist
+ var acc accounts.Account
+ if err2 := acc.DecodeForStorage(data); err2 != nil {
+ return err2
}
- data = make([]byte, acc.EncodingLengthForStorage())
- acc.EncodeForStorage(data)
+ if acc.Incarnation > 0 && acc.IsEmptyCodeHash() {
+ codeBucket := tx.Bucket(dbutils.ContractCodeBucket)
+ codeHash, _ := codeBucket.Get(dbutils.GenerateStoragePrefix(common.BytesToHash(hK), acc.Incarnation))
+ if len(codeHash) > 0 {
+ acc.CodeHash = common.BytesToHash(codeHash)
+ }
+ data = make([]byte, acc.EncodingLengthForStorage())
+ acc.EncodeForStorage(data)
+ }
+ goOn, err = walker(hK, data)
}
- goOn, err = walker(hK, data)
} else if cmp == 0 {
goOn, err = walker(k, v)
}
@@ -452,6 +457,9 @@ func (db *BoltDatabase) walkAsOfThinAccounts(startkey []byte, fixedbits uint, ti
if goOn {
if cmp <= 0 {
k, v = mainCursor.Next()
+ for k != nil && len(k) > common.HashLength {
+ k, v = mainCursor.Next()
+ }
}
if cmp >= 0 {
hK, tsEnc, _, hV = historyCursor.Next()