diff --git a/cmd/snapshots/generator/commands/copy_from_state.go b/cmd/snapshots/generator/commands/copy_from_state.go
index 8d79e61fe76b23a9a2d95268fd517a4a61e2a4b3..9053a84408514c453db7939aa191996d4183d7de 100644
--- a/cmd/snapshots/generator/commands/copy_from_state.go
+++ b/cmd/snapshots/generator/commands/copy_from_state.go
@@ -35,12 +35,19 @@ func CopyFromState(ctx context.Context, dbpath string, snapshotPath string, bloc
if err != nil {
return err
}
+ defer db.Close()
+
+ tx, err := db.RwKV().BeginRo(ctx)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
err = os.RemoveAll(snapshotPath)
if err != nil {
return err
}
- snkv := ethdb.NewLMDB().WithBucketsConfig(func(defaultBuckets dbutils.BucketsCfg) dbutils.BucketsCfg {
+ snkv := ethdb.NewMDBX().WithBucketsConfig(func(defaultBuckets dbutils.BucketsCfg) dbutils.BucketsCfg {
return dbutils.BucketsCfg{
dbutils.PlainStateBucket: dbutils.BucketsConfigs[dbutils.PlainStateBucket],
dbutils.PlainContractCodeBucket: dbutils.BucketsConfigs[dbutils.PlainContractCodeBucket],
@@ -49,98 +56,70 @@ func CopyFromState(ctx context.Context, dbpath string, snapshotPath string, bloc
}).Path(snapshotPath).MustOpen()
log.Info("Create snapshot db", "path", snapshotPath)
- sndb := ethdb.NewObjectDatabase(snkv).NewBatch()
+ logEvery := time.NewTicker(30 * time.Second)
+ defer logEvery.Stop()
tt := time.Now()
- tt2 := time.Now()
- max := 10000000
- i := 0
- err = db.Walk(dbutils.PlainStateBucket, []byte{}, 0, func(k, v []byte) (bool, error) {
- innerErr := sndb.Put(dbutils.PlainStateBucket, k, v)
- if innerErr != nil {
- return false, fmt.Errorf("put state err: %w", innerErr)
- }
- i++
- if i > max {
- i = 0
- innerErr = sndb.CommitAndBegin(ctx)
+ if err = snkv.Update(ctx, func(snTx ethdb.RwTx) error {
+ return tx.ForEach(dbutils.PlainStateBucket, []byte{}, func(k, v []byte) error {
+ innerErr := snTx.Put(dbutils.PlainStateBucket, k, v)
if innerErr != nil {
- return false, fmt.Errorf("commit state err: %w", innerErr)
+ return fmt.Errorf("put state err: %w", innerErr)
+ }
+ select {
+ case <-logEvery.C:
+ log.Info("progress", "bucket", dbutils.PlainStateBucket, "key", fmt.Sprintf("%x", k))
+ default:
}
- log.Info("Commit state", "batch", time.Since(tt2), "all", time.Since(tt))
- tt2 = time.Now()
- }
- return true, nil
- })
- if err != nil {
- return err
- }
- err = sndb.CommitAndBegin(ctx)
- if err != nil {
+ return nil
+ })
+ }); err != nil {
return err
}
+ log.Info("Copy state", "batch", "t", time.Since(tt))
log.Info("Copy plain state end", "t", time.Since(tt))
tt = time.Now()
- tt2 = time.Now()
- err = db.Walk(dbutils.PlainContractCodeBucket, []byte{}, 0, func(k, v []byte) (bool, error) {
- innerErr := sndb.Put(dbutils.PlainContractCodeBucket, k, v)
- if innerErr != nil {
- return false, fmt.Errorf("put contract code err: %w", innerErr)
- }
- i++
- if i > max {
- i = 0
- innerErr = sndb.CommitAndBegin(ctx)
+ if err = snkv.Update(ctx, func(sntx ethdb.RwTx) error {
+ return tx.ForEach(dbutils.PlainContractCodeBucket, []byte{}, func(k, v []byte) error {
+ innerErr := sntx.Put(dbutils.PlainContractCodeBucket, k, v)
if innerErr != nil {
- return false, fmt.Errorf("commit contract code err: %w", innerErr)
+ return fmt.Errorf("put contract code err: %w", innerErr)
}
- log.Info("Commit contract code", "batch", time.Since(tt2), "all", time.Since(tt))
- tt2 = time.Now()
- }
-
- return true, nil
- })
- if err != nil {
+ select {
+ case <-logEvery.C:
+ log.Info("progress", "bucket", dbutils.PlainContractCodeBucket, "key", fmt.Sprintf("%x", k))
+ default:
+ }
+ return nil
+ })
+ }); err != nil {
return err
}
log.Info("Copy contract code end", "t", time.Since(tt))
- err = sndb.Commit()
- if err != nil {
- return err
- }
tt = time.Now()
- tt2 = time.Now()
- err = db.Walk(dbutils.CodeBucket, []byte{}, 0, func(k, v []byte) (bool, error) {
- innerErr := sndb.Put(dbutils.CodeBucket, k, v)
- if innerErr != nil {
- return false, fmt.Errorf("put code err: %w", innerErr)
- }
- i++
- if i > max {
- i = 0
- innerErr = sndb.CommitAndBegin(ctx)
+ if err = snkv.Update(ctx, func(sntx ethdb.RwTx) error {
+ return tx.ForEach(dbutils.CodeBucket, []byte{}, func(k, v []byte) error {
+ innerErr := sntx.Put(dbutils.CodeBucket, k, v)
if innerErr != nil {
- return false, fmt.Errorf("commit code err: %w", innerErr)
+ return fmt.Errorf("put code err: %w", innerErr)
}
- log.Info("Commit code", "batch", time.Since(tt2), "all", time.Since(tt))
- tt2 = time.Now()
- }
-
- return true, nil
- })
- if err != nil {
- return err
- }
- log.Info("Copy code end", "t", time.Since(tt))
- err = sndb.Commit()
- if err != nil {
+ select {
+ case <-logEvery.C:
+ log.Info("progress", "bucket", dbutils.CodeBucket, "key", fmt.Sprintf("%x", k))
+ default:
+ }
+ return nil
+ })
+ }); err != nil {
return err
}
- sndb.Close()
+ log.Info("Copy code", "t", time.Since(tt))
+
db.Close()
+ snkv.Close()
tt = time.Now()
defer func() {
log.Info("Verify end", "t", time.Since(tt))
diff --git a/cmd/snapshots/generator/commands/generate_body_snapshot.go b/cmd/snapshots/generator/commands/generate_body_snapshot.go
index f713db46ec8ff4cb97d3811bb4c2fdad03e31a94..1d07258b0dde4e269798ce5d23ce6d1417d0d7d5 100644
--- a/cmd/snapshots/generator/commands/generate_body_snapshot.go
+++ b/cmd/snapshots/generator/commands/generate_body_snapshot.go
@@ -54,57 +54,53 @@ func BodySnapshot(ctx context.Context, dbPath, snapshotPath string, toBlock uint
}).Path(snapshotPath).MustOpen()
}
- snDB := ethdb.NewObjectDatabase(snKV)
tx, err := kv.BeginRo(context.Background())
if err != nil {
return err
}
defer tx.Rollback()
+ logEvery := time.NewTicker(30 * time.Second)
+ defer logEvery.Stop()
+
t := time.Now()
- chunkFile := 30000
- tuples := make(ethdb.MultiPutTuples, 0, chunkFile*3+100)
var hash common.Hash
+ if err := snKV.Update(ctx, func(sntx ethdb.RwTx) error {
+ for i := uint64(1); i <= toBlock; i++ {
+ if common.IsCanceled(ctx) {
+ return common.ErrStopped
+ }
- for i := uint64(1); i <= toBlock; i++ {
- if common.IsCanceled(ctx) {
- return common.ErrStopped
- }
-
- hash, err = rawdb.ReadCanonicalHash(tx, i)
- if err != nil {
- return fmt.Errorf("getting canonical hash for block %d: %v", i, err)
- }
- body := rawdb.ReadBodyRLP(tx, hash, i)
- tuples = append(tuples, []byte(dbutils.BlockBodyPrefix), dbutils.BlockBodyKey(i, hash), body)
- if len(tuples) >= chunkFile {
- log.Info("Committed", "block", i)
- if _, err = snDB.MultiPut(tuples...); err != nil {
- log.Crit("Multiput error", "err", err)
+ hash, err = rawdb.ReadCanonicalHash(tx, i)
+ if err != nil {
+ return fmt.Errorf("getting canonical hash for block %d: %v", i, err)
+ }
+ body := rawdb.ReadBodyRLP(tx, hash, i)
+ if err = sntx.Put(dbutils.BlockBodyPrefix, dbutils.BlockBodyKey(i, hash), body); err != nil {
return err
}
- tuples = tuples[:0]
+ select {
+ case <-logEvery.C:
+ log.Info("progress", "bucket", dbutils.BlockBodyPrefix, "block num", i)
+ default:
+ }
}
- }
- if len(tuples) > 0 {
- if _, err = snDB.MultiPut(tuples...); err != nil {
- log.Crit("Multiput error", "err", err)
+ err = sntx.Put(dbutils.BodiesSnapshotInfoBucket, []byte(dbutils.SnapshotBodyHeadNumber), big.NewInt(0).SetUint64(toBlock).Bytes())
+ if err != nil {
+ log.Crit("SnapshotBodyHeadNumber error", "err", err)
return err
}
- }
-
- err = snDB.Put(dbutils.BodiesSnapshotInfoBucket, []byte(dbutils.SnapshotBodyHeadNumber), big.NewInt(0).SetUint64(toBlock).Bytes())
- if err != nil {
- log.Crit("SnapshotBodyHeadNumber error", "err", err)
- return err
- }
- err = snDB.Put(dbutils.BodiesSnapshotInfoBucket, []byte(dbutils.SnapshotBodyHeadHash), hash.Bytes())
- if err != nil {
- log.Crit("SnapshotBodyHeadHash error", "err", err)
+ err = sntx.Put(dbutils.BodiesSnapshotInfoBucket, []byte(dbutils.SnapshotBodyHeadHash), hash.Bytes())
+ if err != nil {
+ log.Crit("SnapshotBodyHeadHash error", "err", err)
+ return err
+ }
+ return nil
+ }); err != nil {
return err
}
- snDB.Close()
+ snKV.Close()
if database == "lmdb" {
err = os.Remove(snapshotPath + "/lock.mdb")
} else {
diff --git a/cmd/snapshots/generator/commands/generate_state_snapshot.go b/cmd/snapshots/generator/commands/generate_state_snapshot.go
index 2a788180faaa5f19743838d4cabf8a203c13a92a..9720977d6ba072df738ab09591d00142147ddb34 100644
--- a/cmd/snapshots/generator/commands/generate_state_snapshot.go
+++ b/cmd/snapshots/generator/commands/generate_state_snapshot.go
@@ -66,8 +66,11 @@ func GenerateStateSnapshot(ctx context.Context, dbPath, snapshotPath string, toB
}
}).Path(snapshotPath).MustOpen()
}
- sndb := ethdb.NewObjectDatabase(snkv)
- mt := sndb.NewBatch()
+ writeTx, err := snkv.BeginRw(ctx)
+ if err != nil {
+ return err
+ }
+ defer writeTx.Rollback()
tx, err := kv.BeginRo(context.Background())
if err != nil {
@@ -93,13 +96,6 @@ func GenerateStateSnapshot(ctx context.Context, dbPath, snapshotPath string, toB
select {
case <-ctx.Done():
return false, errors.New("interrupted")
- case <-commitEvery.C:
- ttt := time.Now()
- innerErr := mt.CommitAndBegin(context.Background())
- if innerErr != nil {
- return false, innerErr
- }
- fmt.Println("Committed", time.Since(ttt))
default:
}
}
@@ -119,7 +115,7 @@ func GenerateStateSnapshot(ctx context.Context, dbPath, snapshotPath string, toB
j := 0
innerErr := state.WalkAsOfStorage(tx2, common.BytesToAddress(k), acc.Incarnation, common.Hash{}, toBlock+1, func(k1, k2 []byte, vv []byte) (bool, error) {
j++
- innerErr1 := mt.Put(dbutils.PlainStateBucket, dbutils.PlainGenerateCompositeStorageKey(k1, acc.Incarnation, k2), common.CopyBytes(vv))
+ innerErr1 := writeTx.Put(dbutils.PlainStateBucket, dbutils.PlainGenerateCompositeStorageKey(k1, acc.Incarnation, k2), common.CopyBytes(vv))
if innerErr1 != nil {
return false, innerErr1
}
@@ -145,10 +141,10 @@ func GenerateStateSnapshot(ctx context.Context, dbPath, snapshotPath string, toB
if err1 != nil {
return false, err1
}
- if err1 = mt.Put(dbutils.CodeBucket, codeHash, code); err1 != nil {
+ if err1 = writeTx.Put(dbutils.CodeBucket, codeHash, code); err1 != nil {
return false, err1
}
- if err1 = mt.Put(dbutils.PlainContractCodeBucket, storagePrefix, codeHash); err1 != nil {
+ if err1 = writeTx.Put(dbutils.PlainContractCodeBucket, storagePrefix, codeHash); err1 != nil {
return false, err1
}
}
@@ -156,7 +152,7 @@ func GenerateStateSnapshot(ctx context.Context, dbPath, snapshotPath string, toB
}
newAcc := make([]byte, acc.EncodingLengthForStorage())
acc.EncodeForStorage(newAcc)
- innerErr := mt.Put(dbutils.PlainStateBucket, common.CopyBytes(k), newAcc)
+ innerErr := writeTx.Put(dbutils.PlainStateBucket, common.CopyBytes(k), newAcc)
if innerErr != nil {
return false, innerErr
}
@@ -166,7 +162,7 @@ func GenerateStateSnapshot(ctx context.Context, dbPath, snapshotPath string, toB
if err != nil {
return err
}
- err = mt.Commit()
+ err = writeTx.Commit()
if err != nil {
return err
}