diff --git a/cmd/state/state.go b/cmd/state/state.go
index 86eaf8b7e812755c2f5b465f32ff8c4f4aa3c21b..005ed3f8065c0f9d0c2bfc20f76663d1a98ac7de 100644
--- a/cmd/state/state.go
+++ b/cmd/state/state.go
@@ -57,6 +57,7 @@ var triesize = flag.Int("triesize", 1024*1024, "maximum number of nodes in the s
var preroot = flag.Bool("preroot", false, "Attempt to compute hash of the trie without modifying it")
var snapshotInterval = flag.Uint64("snapshotInterval", 0, "how often to take snapshots (0 - never, 1 - every block, 1000 - every 1000th block, etc)")
var snapshotFrom = flag.Uint64("snapshotFrom", 0, "from which block to start snapshots")
+var witnessInterval = flag.Uint64("witnessInterval", 1, "after which block to extract witness (put a large number like 10000000 to disable)")
func check(e error) {
if e != nil {
@@ -1676,7 +1677,11 @@ func main() {
//nakedAccountChart()
//specExecChart1()
if *action == "stateless" {
- stateless(*chaindata, *statefile, *triesize, *preroot, *snapshotInterval, *snapshotFrom)
+ createDb := func(path string) (ethdb.Database, error) {
+ return ethdb.NewBoltDatabase(path)
+ }
+
+ stateless(*chaindata, *statefile, *triesize, *preroot, *snapshotInterval, *snapshotFrom, *witnessInterval, createDb)
}
if *action == "stateless_chart" {
stateless_chart_key_values("/Users/alexeyakhunov/mygit/go-ethereum/st_1/stateless.csv", []int{21, 20, 19, 18}, "breakdown.png", 2800000, 1)
diff --git a/cmd/state/state_snapshot.go b/cmd/state/state_snapshot.go
index 226cf883d8bb94e1e78a1c742ac3bb7f9fc07008..b249b2e6875babc6d1432aa4d8a7aa85cff90660 100644
--- a/cmd/state/state_snapshot.go
+++ b/cmd/state/state_snapshot.go
@@ -110,88 +110,87 @@ func constructSnapshot(ethDb ethdb.Database, blockNum uint64) {
check(err)
}
-func copyBucket(bucketName []byte, fromTx *bolt.Tx, toDB *bolt.DB) error {
- toTx, err := toDB.Begin(true)
- if err != nil {
- return err
- }
+type bucketWriter struct {
+ db ethdb.Database
+ bucket []byte
+ pending ethdb.DbWithPendingMutations
+ written uint64
+}
- toBucket, err := toTx.CreateBucket(bucketName, true)
- if err != nil {
- return err
+func (bw *bucketWriter) printStats() {
+ if bw.written == 0 {
+ fmt.Printf(" -- nothing to copy for bucket: '%s'...", string(bw.bucket))
+ } else {
+ fmt.Printf("\r -- commited %d records for bucket: '%s'...", bw.written, string(bw.bucket))
}
+}
- count := 0
+func (bw *bucketWriter) walker(k, v []byte) (bool, error) {
+ if bw.pending == nil {
+ bw.pending = bw.db.NewBatch()
+ }
- printStats := func() {
- fmt.Printf("\r -- commited %d records for bucket: '%s'...", count, string(bucketName))
+ if err := bw.pending.Put(bw.bucket, common.CopyBytes(k), common.CopyBytes(v)); err != nil {
+ return false, err
}
+ bw.written++
- if fromBucket := fromTx.Bucket(bucketName); fromBucket != nil {
- defer printStats()
+ if bw.pending.BatchSize() >= 100000 {
+ if _, err := bw.pending.Commit(); err != nil {
+ return false, err
+ }
- fromCursor := fromBucket.Cursor()
+ bw.printStats()
- for k, v := fromCursor.First(); k != nil; k, v = fromCursor.Next() {
- err = toBucket.Put(common.CopyBytes(k), common.CopyBytes(v))
- if err != nil {
- return err
- }
- count++
+ bw.pending = bw.db.NewBatch()
+ }
- if count%100000 == 0 {
- err = toTx.Commit()
- if err != nil {
- return err
- }
+ return true, nil
+}
- printStats()
+func (bw *bucketWriter) commit() error {
+ defer bw.printStats()
- toTx, err = toDB.Begin(true)
- if err != nil {
- return err
- }
- toBucket = toTx.Bucket(bucketName)
- }
- }
- } else {
- fmt.Printf(" -- nothing to copy for the bucket name: '%s'...", string(bucketName))
+ if bw.pending != nil {
+ _, err := bw.pending.Commit()
+ return err
}
- return toTx.Commit()
+ return nil
}
-func copyDatabase(fromDB *bolt.DB, toDB *bolt.DB) error {
- return fromDB.View(func(tx *bolt.Tx) error {
- fmt.Printf(" - copying AccountsBucket...\n")
- if err := copyBucket(dbutils.AccountsBucket, tx, toDB); err != nil {
- fmt.Println("FAIL")
- return err
- }
- fmt.Println("OK")
+func newBucketWriter(db ethdb.Database, bucket []byte) *bucketWriter {
+ return &bucketWriter{
+ db: db,
+ bucket: bucket,
+ pending: nil,
+ written: 0,
+ }
+}
+
+func copyDatabase(fromDB ethdb.Database, toDB ethdb.Database) error {
+ for _, bucket := range [][]byte{dbutils.AccountsBucket, dbutils.StorageBucket, dbutils.CodeBucket} {
+ fmt.Printf(" - copying bucket '%s'...\n", string(bucket))
+ writer := newBucketWriter(toDB, bucket)
- fmt.Printf(" - copying StorageBucket...\n")
- if err := copyBucket(dbutils.StorageBucket, tx, toDB); err != nil {
+ if err := fromDB.Walk(bucket, nil, 0, writer.walker); err != nil {
fmt.Println("FAIL")
return err
}
- fmt.Println("OK")
- fmt.Printf(" - copying CodeBucket...\n")
- if err := copyBucket(dbutils.CodeBucket, tx, toDB); err != nil {
+ if err := writer.commit(); err != nil {
fmt.Println("FAIL")
return err
}
fmt.Println("OK")
-
- return nil
- })
+ }
+ return nil
}
-func saveSnapshot(db *bolt.DB, filename string) {
+func saveSnapshot(db ethdb.Database, filename string, createDb CreateDbFunc) {
fmt.Printf("Saving snapshot to %s\n", filename)
- diskDb, err := bolt.Open(filename, 0600, &bolt.Options{})
+ diskDb, err := createDb(filename)
check(err)
defer diskDb.Close()
@@ -199,9 +198,10 @@ func saveSnapshot(db *bolt.DB, filename string) {
check(err)
}
-func loadSnapshot(db *bolt.DB, filename string) {
+func loadSnapshot(db ethdb.Database, filename string, createDb CreateDbFunc) {
fmt.Printf("Loading snapshot from %s\n", filename)
- diskDb, err := bolt.Open(filename, 0600, &bolt.Options{})
+
+ diskDb, err := createDb(filename)
check(err)
defer diskDb.Close()
@@ -308,7 +308,7 @@ func compare_snapshot(stateDb ethdb.Database, db *bolt.DB, filename string) {
check(err)
}
-func checkRoots(stateDb ethdb.Database, db *bolt.DB, rootHash common.Hash, blockNum uint64) {
+func checkRoots(stateDb ethdb.Database, rootHash common.Hash, blockNum uint64) {
startTime := time.Now()
t := trie.New(rootHash)
r := trie.NewResolver(0, true, blockNum)
@@ -324,26 +324,24 @@ func checkRoots(stateDb ethdb.Database, db *bolt.DB, rootHash common.Hash, block
startTime = time.Now()
var addrHash common.Hash
roots := make(map[common.Hash]common.Hash)
- err = db.View(func(tx *bolt.Tx) error {
- sb := tx.Bucket(dbutils.StorageBucket)
- b := tx.Bucket(dbutils.AccountsBucket)
- c := sb.Cursor()
- for k, _ := c.First(); k != nil; k, _ = c.Next() {
- copy(addrHash[:], k[:32])
- if _, ok := roots[addrHash]; !ok {
- if enc, _ := b.Get(addrHash[:]); enc == nil {
- roots[addrHash] = common.Hash{}
- } else {
- var account accounts.Account
- if err = account.DecodeForStorage(enc); err != nil {
- return err
- }
- roots[addrHash] = account.Root
+
+ err = stateDb.Walk(dbutils.StorageBucket, nil, 0, func(k, v []byte) (bool, error) {
+ copy(addrHash[:], k[:32])
+ if _, ok := roots[addrHash]; !ok {
+ if enc, _ := stateDb.Get(dbutils.AccountsBucket, addrHash[:]); enc == nil {
+ roots[addrHash] = common.Hash{}
+ } else {
+ var account accounts.Account
+ if err = account.DecodeForStorage(enc); err != nil {
+ return false, err
}
+ roots[addrHash] = account.Root
}
}
- return nil
+
+ return true, nil
})
+
if err != nil {
panic(err)
}
@@ -384,7 +382,9 @@ func stateSnapshot() error {
stateDb, db := ethdb.NewMemDatabase2()
defer stateDb.Close()
if _, err := os.Stat("statedb0"); err == nil {
- loadSnapshot(db, "statedb0")
+ loadSnapshot(stateDb, "statedb0", func(path string) (ethdb.Database, error) {
+ return ethdb.NewBoltDatabase(path)
+ })
if err := loadCodes(db, ethDb); err != nil {
return err
}
@@ -398,7 +398,7 @@ func stateSnapshot() error {
block := bc.GetBlockByNumber(blockNum)
fmt.Printf("Block number: %d\n", blockNum)
fmt.Printf("Block root hash: %x\n", block.Root())
- checkRoots(ethDb, ethDb.DB(), block.Root(), blockNum)
+ checkRoots(ethDb, block.Root(), blockNum)
return nil
}
@@ -416,5 +416,5 @@ func verifySnapshot(chaindata string) {
fmt.Printf("Block number: %d\n", currentBlockNr)
fmt.Printf("Block root hash: %x\n", currentBlock.Root())
preRoot := currentBlock.Root()
- checkRoots(ethDb, ethDb.DB(), preRoot, blockNum)
+ checkRoots(ethDb, preRoot, blockNum)
}
diff --git a/cmd/state/state_snapshot_test.go b/cmd/state/state_snapshot_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..72a36ac9976aa4934306a7939c45c7282c4b584e
--- /dev/null
+++ b/cmd/state/state_snapshot_test.go
@@ -0,0 +1,120 @@
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+
+ "github.com/ledgerwatch/turbo-geth/common/dbutils"
+ "github.com/ledgerwatch/turbo-geth/ethdb"
+)
+
+type testData map[string][]byte
+
+func generateData(prefix string) testData {
+ data := make(map[string][]byte)
+ for i := 0; i < 100; i++ {
+ data[fmt.Sprintf("key:%s-%d", prefix, i)] = []byte(fmt.Sprintf("val:%s-%d", prefix, i))
+ }
+
+ return testData(data)
+}
+
+func writeDataToDb(t *testing.T, db ethdb.Database, bucket []byte, data testData) {
+ for k, v := range data {
+ err := db.Put(bucket, []byte(k), v)
+ if err != nil {
+ t.Errorf("error while forming the source db: %v", err)
+ }
+ }
+}
+
+func checkDataInDb(t *testing.T, db ethdb.Database, bucket []byte, data testData) {
+ for k, v := range data {
+ val, err := db.Get(bucket, []byte(k))
+ if err != nil {
+ t.Errorf("error while requesting the dest db: %v", err)
+ }
+
+ if !bytes.Equal(v, val) {
+ t.Errorf("unexpected value for the key %s (got: %s expected: %x)", k, string(val), string(v))
+ }
+ }
+
+ err := db.Walk(bucket, nil, 0, func(k, v []byte) (bool, error) {
+ val, ok := data[string(k)]
+ if !ok {
+ t.Errorf("unexpected key in the database (not in the data): %s", string(k))
+ }
+
+ if !bytes.Equal(v, val) {
+ t.Errorf("unexpected value for the key %s (got: %s expected: %x)", k, string(val), string(v))
+ }
+
+ return true, nil
+ })
+
+ if err != nil {
+ t.Errorf("error while walking the dest db: %v", err)
+ }
+}
+
+func TestCopyDatabase(t *testing.T) {
+ doTestcase(t, map[string]testData{})
+
+ doTestcase(t, map[string]testData{
+ string(dbutils.AccountsBucket): generateData(string(dbutils.AccountsBucket)),
+ })
+
+ doTestcase(t, map[string]testData{
+ string(dbutils.StorageBucket): generateData(string(dbutils.StorageBucket)),
+ })
+
+ doTestcase(t, map[string]testData{
+ string(dbutils.CodeBucket): generateData(string(dbutils.CodeBucket)),
+ })
+
+ doTestcase(t, map[string]testData{
+ string(dbutils.AccountsBucket): generateData(string(dbutils.AccountsBucket)),
+ string(dbutils.StorageBucket): generateData(string(dbutils.StorageBucket)),
+ })
+
+ doTestcase(t, map[string]testData{
+ string(dbutils.StorageBucket): generateData(string(dbutils.StorageBucket)),
+ string(dbutils.CodeBucket): generateData(string(dbutils.CodeBucket)),
+ })
+
+ doTestcase(t, map[string]testData{
+ string(dbutils.AccountsBucket): generateData(string(dbutils.AccountsBucket)),
+ string(dbutils.CodeBucket): generateData(string(dbutils.CodeBucket)),
+ })
+
+ doTestcase(t, map[string]testData{
+ string(dbutils.AccountsBucket): generateData(string(dbutils.AccountsBucket)),
+ string(dbutils.StorageBucket): generateData(string(dbutils.StorageBucket)),
+ string(dbutils.CodeBucket): generateData(string(dbutils.CodeBucket)),
+ })
+
+}
+
+func doTestcase(t *testing.T, testCase map[string]testData) {
+ sourceDb := ethdb.NewMemDatabase()
+ defer sourceDb.Close()
+
+ destDb := ethdb.NewMemDatabase()
+ defer destDb.Close()
+
+ for bucket, data := range testCase {
+ writeDataToDb(t, sourceDb, []byte(bucket), data)
+ }
+
+ err := copyDatabase(sourceDb, destDb)
+
+ if err != nil {
+ t.Errorf("error while copying the db: %v", err)
+ }
+
+ for bucket, data := range testCase {
+ checkDataInDb(t, destDb, []byte(bucket), data)
+ }
+}
diff --git a/cmd/state/stateless.go b/cmd/state/stateless.go
index b8487c550e2104d1740c2e34b86daf3d59e80bed..90d1458b148d5a249a1bccc9fbe02871569c88c6 100644
--- a/cmd/state/stateless.go
+++ b/cmd/state/stateless.go
@@ -106,7 +106,17 @@ func writeStats(w io.Writer, blockNum uint64, blockProof trie.BlockProof) {
}
*/
-func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool, interval uint64, ignoreOlderThan uint64) {
+type CreateDbFunc func(string) (ethdb.Database, error)
+
+func stateless(chaindata string,
+ statefile string,
+ triesize int,
+ tryPreRoot bool,
+ interval uint64,
+ ignoreOlderThan uint64,
+ witnessThreshold uint64,
+ createDb CreateDbFunc) {
+
state.MaxTrieCacheGen = uint32(triesize)
startTime := time.Now()
sigs := make(chan os.Signal, 1)
@@ -118,7 +128,7 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
interruptCh <- true
}()
- ethDb, err := ethdb.NewBoltDatabase(chaindata)
+ ethDb, err := createDb(chaindata)
check(err)
defer ethDb.Close()
chainConfig := params.MainnetChainConfig
@@ -131,10 +141,9 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
engine := ethash.NewFullFaker()
bcb, err := core.NewBlockChain(ethDb, nil, chainConfig, engine, vm.Config{}, nil)
check(err)
- stateDb, err := ethdb.NewBoltDatabase(statefile)
+ stateDb, err := createDb(statefile)
check(err)
defer stateDb.Close()
- db := stateDb.DB()
blockNum := uint64(*block)
var preRoot common.Hash
if blockNum == 1 {
@@ -150,7 +159,7 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
fmt.Printf("Block number: %d\n", blockNum-1)
fmt.Printf("Block root hash: %x\n", block.Root())
preRoot = block.Root()
- checkRoots(stateDb, db, preRoot, blockNum-1)
+ checkRoots(stateDb, preRoot, blockNum-1)
}
batch := stateDb.NewBatch()
defer func() {
@@ -166,11 +175,10 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
tds.SetResolveReads(false)
tds.SetNoHistory(true)
interrupt := false
- var thresholdBlock uint64 = 1
var witness []byte
for !interrupt {
trace := false // blockNum == 545080
- tds.SetResolveReads(blockNum >= thresholdBlock)
+ tds.SetResolveReads(blockNum >= witnessThreshold)
block := bcb.GetBlockByNumber(blockNum)
if block == nil {
break
@@ -213,7 +221,7 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
return
}
witness = nil
- if blockNum >= thresholdBlock {
+ if blockNum >= witnessThreshold {
// Witness has to be extracted before the state trie is modified
witness, err = tds.ExtractWitness(trace)
if err != nil {
@@ -222,7 +230,7 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
}
}
finalRootFail := false
- if blockNum >= thresholdBlock && witness != nil { // witness == nil means the extraction fails
+ if blockNum >= witnessThreshold && witness != nil { // witness == nil means the extraction fails
var s *state.Stateless
s, err = state.NewStateless(preRoot, witness, blockNum-1, trace)
if err != nil {
@@ -280,6 +288,7 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
nextRoot := roots[len(roots)-1]
if nextRoot != block.Root() {
fmt.Printf("Root hash does not match for block %d, expected %x, was %x\n", blockNum, block.Root(), nextRoot)
+ return
}
tds.SetBlockNr(blockNum)
@@ -302,7 +311,8 @@ func stateless(chaindata string, statefile string, triesize int, tryPreRoot bool
if willSnapshot {
// Snapshots of the state will be written to the same directory as the state file
fmt.Printf("\nSaving snapshot at block %d, hash %x\n", blockNum, block.Root())
- saveSnapshot(db, fmt.Sprintf("%s_%d", statefile, blockNum))
+
+ saveSnapshot(stateDb, fmt.Sprintf("%s_%d", statefile, blockNum), createDb)
}
preRoot = header.Root