diff --git a/README.md b/README.md
index 490852d0f240aee28bab0573c94a38282d0e32fa..8cd165518de3ce3ccf04d6757c1b5d0002d8e599 100644
--- a/README.md
+++ b/README.md
@@ -232,7 +232,8 @@ or
XDG_DATA_HOME=/preferred/data/folder make docker-compose
```
-Makefile creates the initial directories for erigon, prometheus and grafana. The PID namespace is shared between erigon and rpcdaemon which is required to open Erigon's DB from another process (RPCDaemon local-mode).
+Makefile creates the initial directories for erigon, prometheus and grafana. The PID namespace is shared between erigon
+and rpcdaemon which is required to open Erigon's DB from another process (RPCDaemon local-mode).
See: https://github.com/ledgerwatch/erigon/pull/2392/files
Windows support for docker-compose is not ready yet. Please help us with .ps1 port
@@ -241,6 +242,10 @@ Windows support for docker-compose is not ready yet. Please help us with .ps1 po
`docker-compose up prometheus grafana`, [detailed docs](./cmd/prometheus/Readme.md).
+### Prune old data
+
+Disabled by default. To enable see `./build/bin/erigon --help` for flags `--prune`
+
FAQ
================
diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
index d19aef2649aca149898c17c4a6407440e3f0e725..307f21c2cbdca989766650a75c085bb5ab151f8f 100644
--- a/cmd/hack/hack.go
+++ b/cmd/hack/hack.go
@@ -29,7 +29,7 @@ import (
"github.com/wcharczuk/go-chart"
"github.com/wcharczuk/go-chart/util"
- "github.com/ledgerwatch/erigon/cmd/hack/db"
+ hackdb "github.com/ledgerwatch/erigon/cmd/hack/db"
"github.com/ledgerwatch/erigon/cmd/hack/flow"
"github.com/ledgerwatch/erigon/cmd/hack/tool"
"github.com/ledgerwatch/erigon/common"
@@ -1689,16 +1689,16 @@ func readCallTraces(chaindata string, block uint64) error {
var k []byte
var v []byte
count := 0
- for k, v, err = traceCursor.First(); k != nil && err == nil; k, v, err = traceCursor.Next() {
+ for k, v, err = traceCursor.First(); k != nil; k, v, err = traceCursor.Next() {
+ if err != nil {
+ return err
+ }
blockNum := binary.BigEndian.Uint64(k)
if blockNum == block {
fmt.Printf("%x\n", v)
}
count++
}
- if err != nil {
- return err
- }
fmt.Printf("Found %d records\n", count)
idxCursor, err2 := tx.Cursor(dbutils.CallToIndex)
if err2 != nil {
@@ -2019,7 +2019,7 @@ func scanReceipts3(chaindata string, block uint64) error {
var key [8]byte
var v []byte
binary.BigEndian.PutUint64(key[:], block)
- if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
+ if v, err = tx.GetOne(dbutils.Receipts, key[:]); err != nil {
return err
}
fmt.Printf("%x\n", v)
@@ -2041,20 +2041,16 @@ func scanReceipts2(chaindata string) error {
return err
}
defer tx.Rollback()
- if sm, smErr := ethdb.GetStorageModeFromDB(tx); smErr != nil {
- return smErr
- } else {
- if !sm.History {
- log.Warn("Could not perform this migration because history is not in storage mode")
- return nil
- }
+ blockNum, err := changeset.AvailableFrom(tx)
+ if err != nil {
+ return err
}
fixedCount := 0
logInterval := 30 * time.Second
logEvery := time.NewTicker(logInterval)
var key [8]byte
var v []byte
- for blockNum := uint64(1); true; blockNum++ {
+ for ; true; blockNum++ {
select {
default:
case <-logEvery.C:
@@ -2068,7 +2064,7 @@ func scanReceipts2(chaindata string) error {
break
}
binary.BigEndian.PutUint64(key[:], blockNum)
- if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
+ if v, err = tx.GetOne(dbutils.Receipts, key[:]); err != nil {
return err
}
var receipts types.Receipts
@@ -2103,21 +2099,21 @@ func scanReceipts(chaindata string, block uint64) error {
defer f.Close()
w := bufio.NewWriter(f)
defer w.Flush()
- dbdb := kv2.MustOpen(chaindata)
- defer dbdb.Close()
- tx, err := dbdb.BeginRw(context.Background())
+ db := kv2.MustOpen(chaindata)
+ defer db.Close()
+ tx, err := db.BeginRw(context.Background())
if err != nil {
return err
}
defer tx.Rollback()
- if sm, smErr := ethdb.GetStorageModeFromDB(tx); smErr != nil {
- return smErr
- } else {
- if !sm.History {
- log.Warn("Could not perform this migration because history is not in storage mode")
- return nil
- }
+ blockNum, err := changeset.AvailableFrom(tx)
+ if err != nil {
+ return err
}
+ if block > blockNum {
+ blockNum = block
+ }
+
genesisBlock, err := rawdb.ReadBlockByNumber(tx, 0)
if err != nil {
return err
@@ -2134,13 +2130,13 @@ func scanReceipts(chaindata string, block uint64) error {
logEvery := time.NewTicker(logInterval)
var key [8]byte
var v []byte
- for blockNum := block; true; blockNum++ {
+ for ; true; blockNum++ {
select {
default:
case <-logEvery.C:
log.Info("Commit", "block", blockNum, "fixed", fixedCount)
tx.Commit()
- if tx, err = dbdb.BeginRw(context.Background()); err != nil {
+ if tx, err = db.BeginRw(context.Background()); err != nil {
return err
}
}
@@ -2152,7 +2148,7 @@ func scanReceipts(chaindata string, block uint64) error {
break
}
binary.BigEndian.PutUint64(key[:], blockNum)
- if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
+ if v, err = tx.GetOne(dbutils.Receipts, key[:]); err != nil {
return err
}
var receipts types.Receipts
@@ -2191,7 +2187,7 @@ func scanReceipts(chaindata string, block uint64) error {
if err = cbor.Marshal(&buf, receipts); err != nil {
return err
}
- if err = tx.Put(dbutils.BlockReceiptsPrefix, common.CopyBytes(key[:]), common.CopyBytes(buf.Bytes())); err != nil {
+ if err = tx.Put(dbutils.Receipts, common.CopyBytes(key[:]), common.CopyBytes(buf.Bytes())); err != nil {
return err
}
fixedCount++
@@ -2225,7 +2221,7 @@ func scanReceipts(chaindata string, block uint64) error {
if err != nil {
return fmt.Errorf("encode block receipts for block %d: %v", blockNum, err)
}
- if err = tx.Put(dbutils.BlockReceiptsPrefix, key[:], buf.Bytes()); err != nil {
+ if err = tx.Put(dbutils.Receipts, key[:], buf.Bytes()); err != nil {
return fmt.Errorf("writing receipts for block %d: %v", blockNum, err)
}
if _, err = w.Write([]byte(fmt.Sprintf("%d\n", blockNum))); err != nil {
@@ -2368,10 +2364,10 @@ func main() {
err = extractHashes(*chaindata, uint64(*block), uint64(*blockTotal), *name)
case "defrag":
- err = db.Defrag()
+ err = hackdb.Defrag()
case "textInfo":
- err = db.TextInfo(*chaindata, &strings.Builder{})
+ err = hackdb.TextInfo(*chaindata, &strings.Builder{})
case "extractBodies":
err = extractBodies(*chaindata, uint64(*block))
diff --git a/cmd/integration/Readme.md b/cmd/integration/Readme.md
index 5f3e8cb8c6ea32e2150cf23519080a7dff3120bb..41fb8852026a49dc8ea5da50ebee439dba762baa 100644
--- a/cmd/integration/Readme.md
+++ b/cmd/integration/Readme.md
@@ -1,6 +1,8 @@
-Integration - tool to run Erigon stages in custom way: run/reset single stage, run all stages but reorg every X blocks, etc...
+Integration - tool to run Erigon stages in custom way: run/reset single stage, run all stages but reorg every X blocks,
+etc...
+
+## Examples
-## Examples
All commands require parameter `--datadir=<datadir>` - I will skip it for readability.
```
@@ -22,6 +24,14 @@ integration stage_exec --unwind=10
# Drop data of single stage
integration stage_exec --reset
integration stage_history --reset
+
+# Unwind single stage N blocks backward
+integration stage_exec --unwind=N
+integration stage_history --unwind=N
+
+# Run stage prune to block N
+integration stage_exec --prune.to=N
+integration stage_history --prune.to=N
...
# hack which allows to force clear unwind stack of all stages
@@ -31,6 +41,7 @@ clear_unwind_stack
## For testing run all stages in "N blocks forward M blocks re-org" loop
Pre-requirements of `state_stages` command:
+
- Headers/Bodies must be downloaded
- TxSenders stage must be executed
@@ -41,6 +52,7 @@ integration reset_state # drops all stages after Senders stage (including it's d
```
For example:
+
```
--unwind=1 --unwind.every=10 # 10 blocks forward, 1 block back, 10 blocks forward, ...
--unwind=10 --unwind.every=1 # 1 block forward, 10 blocks back, 1 blocks forward, ...
@@ -50,7 +62,7 @@ For example:
--chaindata.reference # When finish all cycles, does comparison to this db file.
```
-## "Wrong trie root" problem - temporary solution
+## "Wrong trie root" problem - temporary solution
```
make all
diff --git a/cmd/integration/commands/flags.go b/cmd/integration/commands/flags.go
index d91e91b2dab5855cefaa58583fea101c69291f19..f01750ddd082184dc0d6b8ab3f4a92dbd0b7aaf4 100644
--- a/cmd/integration/commands/flags.go
+++ b/cmd/integration/commands/flags.go
@@ -9,27 +9,24 @@ import (
)
var (
- chaindata string
- //database string
- databaseVerbosity int
- snapshotMode string
- snapshotDir string
- toChaindata string
- referenceChaindata string
- block uint64
- unwind uint64
- unwindEvery uint64
- batchSizeStr string
- reset bool
- bucket string
- datadir string
- migration string
- integritySlow bool
- integrityFast bool
- file string
- txtrace bool // Whether to trace the execution (should only be used together eith `block`)
- storageMode string
- chain string // Which chain to use (mainnet, ropsten, rinkeby, goerli, etc.)
+ chaindata string
+ databaseVerbosity int
+ snapshotMode, snapshotDir string
+ referenceChaindata string
+ block, pruneTo, unwind uint64
+ unwindEvery uint64
+ batchSizeStr string
+ reset bool
+ bucket string
+ datadir, toChaindata string
+ migration string
+ integrityFast, integritySlow bool
+ file string
+ txtrace bool // Whether to trace the execution (should only be used together eith `block`)
+ pruneFlag string
+ pruneH, pruneR, pruneT, pruneC uint64
+ experiments []string
+ chain string // Which chain to use (mainnet, ropsten, rinkeby, goerli, etc.)
)
func must(err error) {
@@ -74,6 +71,10 @@ func withUnwind(cmd *cobra.Command) {
cmd.Flags().Uint64Var(&unwind, "unwind", 0, "how much blocks unwind on each iteration")
}
+func withPruneTo(cmd *cobra.Command) {
+ cmd.Flags().Uint64Var(&pruneTo, "prune.to", 0, "how much blocks unwind on each iteration")
+}
+
func withUnwindEvery(cmd *cobra.Command) {
cmd.Flags().Uint64Var(&unwindEvery, "unwind.every", 0, "each iteration test will move forward `--unwind.every` blocks, then unwind `--unwind` blocks")
}
diff --git a/cmd/integration/commands/reset_state.go b/cmd/integration/commands/reset_state.go
index a661c52eb516ce36e7cef3df374f7be61ba10659..68d2b9d4ad03631bc26186c41110d3ec33350e65 100644
--- a/cmd/integration/commands/reset_state.go
+++ b/cmd/integration/commands/reset_state.go
@@ -12,6 +12,7 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
"github.com/spf13/cobra"
)
@@ -116,7 +117,7 @@ func resetExec(tx ethdb.RwTx, g *core.Genesis) error {
if err := tx.(ethdb.BucketMigrator).ClearBucket(dbutils.PlainContractCodeBucket); err != nil {
return err
}
- if err := tx.(ethdb.BucketMigrator).ClearBucket(dbutils.BlockReceiptsPrefix); err != nil {
+ if err := tx.(ethdb.BucketMigrator).ClearBucket(dbutils.Receipts); err != nil {
return err
}
if err := tx.(ethdb.BucketMigrator).ClearBucket(dbutils.Log); err != nil {
@@ -138,11 +139,7 @@ func resetExec(tx ethdb.RwTx, g *core.Genesis) error {
return err
}
- sm, err := ethdb.GetStorageModeFromDB(tx)
- if err != nil {
- return err
- }
- _, _, err = core.OverrideGenesisBlock(tx, g, sm.History)
+ _, _, err := core.OverrideGenesisBlock(tx, g)
if err != nil {
return err
}
@@ -222,11 +219,22 @@ func printStages(db ethdb.KVGetter) error {
w := new(tabwriter.Writer)
defer w.Flush()
w.Init(os.Stdout, 8, 8, 0, '\t', 0)
+ fmt.Fprintf(w, "Note: prune_at doesn't mean 'all data before were deleted' - it just mean stage.Prune function were run to this block. Because 1 stage may prune multiple data types to different prune distance.\n")
for _, stage := range stages.AllStages {
if progress, err = stages.GetStageProgress(db, stage); err != nil {
return err
}
- fmt.Fprintf(w, "%s \t %d\n", string(stage), progress)
+ prunedTo, err := stages.GetStagePruneProgress(db, stage)
+ if err != nil {
+ return err
+ }
+ fmt.Fprintf(w, "%s \t %d \t prune_at=%d\n", string(stage), progress, prunedTo)
+ }
+ pm, err := prune.Get(db)
+ if err != nil {
+ return err
}
+ fmt.Fprintf(w, "--\n")
+ fmt.Fprintf(w, "prune distance: %#v\n", pm)
return nil
}
diff --git a/cmd/integration/commands/snapshot_check.go b/cmd/integration/commands/snapshot_check.go
index 34a388847bff437b65c7be9ced39852a6d80a792..7f10235ac13fe33cac8f0d992f52a313a7057472 100644
--- a/cmd/integration/commands/snapshot_check.go
+++ b/cmd/integration/commands/snapshot_check.go
@@ -16,6 +16,7 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
kv2 "github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
"github.com/spf13/cobra"
@@ -90,7 +91,7 @@ var cmdSnapshotCheck = &cobra.Command{
if isNew {
if err := kv.Update(ctx, func(tx ethdb.RwTx) error {
- return ethdb.SetStorageModeIfNotExist(tx, ethdb.StorageMode{})
+ return prune.SetIfNotExist(tx, prune.DefaultMode)
}); err != nil {
return err
}
@@ -104,7 +105,7 @@ var cmdSnapshotCheck = &cobra.Command{
}
func snapshotCheck(ctx context.Context, db ethdb.RwKV, isNew bool, tmpDir string) (err error) {
- _, engine, chainConfig, vmConfig, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, engine, chainConfig, vmConfig, _, sync, _, _ := newSync(ctx, db, nil)
var snapshotBlock uint64 = 11_000_000
var lastBlockHeaderNumber, blockNum uint64
@@ -208,24 +209,24 @@ func snapshotCheck(ctx context.Context, db ethdb.RwKV, isNew bool, tmpDir string
)
if isNew {
- stage3 := stage(sync, tx, stages.Senders)
+ stage3 := stage(sync, tx, nil, stages.Senders)
err = stage3.Update(tx, lastBlockHeaderNumber)
if err != nil {
return err
}
- stage4 := stage(sync, tx, stages.Execution)
+ stage4 := stage(sync, tx, nil, stages.Execution)
err = stage4.Update(tx, snapshotBlock)
if err != nil {
return err
}
- stage5 := stage(sync, tx, stages.HashState)
+ stage5 := stage(sync, tx, nil, stages.HashState)
err = stage5.Update(tx, snapshotBlock)
if err != nil {
return err
}
- stage6 := stage(sync, tx, stages.IntermediateHashes)
+ stage6 := stage(sync, tx, nil, stages.IntermediateHashes)
err = stage6.Update(tx, snapshotBlock)
if err != nil {
return err
@@ -240,18 +241,18 @@ func snapshotCheck(ctx context.Context, db ethdb.RwKV, isNew bool, tmpDir string
if err != nil {
return err
}
- stage4 := stage(sync, tx, stages.Execution)
+ stage4 := stage(sync, tx, nil, stages.Execution)
stage4.BlockNumber = blockNumber - 1
log.Info("Stage4", "progress", stage4.BlockNumber)
err = stagedsync.SpawnExecuteBlocksStage(stage4, sync, tx, blockNumber, ctx,
- stagedsync.StageExecuteBlocksCfg(db, false, false, false, 0, batchSize, nil, chainConfig, engine, vmConfig, nil, false, tmpDir),
+ stagedsync.StageExecuteBlocksCfg(db, pm, batchSize, nil, chainConfig, engine, vmConfig, nil, false, tmpDir),
false)
if err != nil {
return fmt.Errorf("execution err %w", err)
}
- stage5 := stage(sync, tx, stages.HashState)
+ stage5 := stage(sync, tx, nil, stages.HashState)
stage5.BlockNumber = blockNumber - 1
log.Info("Stage5", "progress", stage5.BlockNumber)
err = stagedsync.SpawnHashStateStage(stage5, tx, stagedsync.StageHashStateCfg(db, tmpDir), ctx)
@@ -259,7 +260,7 @@ func snapshotCheck(ctx context.Context, db ethdb.RwKV, isNew bool, tmpDir string
return fmt.Errorf("spawnHashStateStage err %w", err)
}
- stage6 := stage(sync, tx, stages.IntermediateHashes)
+ stage6 := stage(sync, tx, nil, stages.IntermediateHashes)
stage6.BlockNumber = blockNumber - 1
log.Info("Stage6", "progress", stage6.BlockNumber)
if _, err = stagedsync.SpawnIntermediateHashesStage(stage5, nil /* Unwinder */, tx, stagedsync.StageTrieCfg(db, true, true, tmpDir), ctx); err != nil {
diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go
index bd8f86b082711d5684d6120f0c2a89ac9b265aea..540980158ab2afbde491a90eeadd582b96d92238 100644
--- a/cmd/integration/commands/stages.go
+++ b/cmd/integration/commands/stages.go
@@ -23,6 +23,7 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/ethdb/remote/remotedbserver"
"github.com/ledgerwatch/erigon/log"
"github.com/ledgerwatch/erigon/migrations"
@@ -232,9 +233,9 @@ var cmdRunMigrations = &cobra.Command{
},
}
-var cmdSetStorageMode = &cobra.Command{
- Use: "set_storage_mode",
- Short: "Override storage mode (if you know what you are doing)",
+var cmdSetPrune = &cobra.Command{
+ Use: "set_prune",
+ Short: "Override existing --prune flag value (if you know what you are doing)",
RunE: func(cmd *cobra.Command, args []string) error {
db := openDB(chaindata, true)
defer db.Close()
@@ -265,6 +266,7 @@ func init() {
withReset(cmdStageExec)
withBlock(cmdStageExec)
withUnwind(cmdStageExec)
+ withPruneTo(cmdStageExec)
withBatchSize(cmdStageExec)
withTxTrace(cmdStageExec)
withChain(cmdStageExec)
@@ -275,6 +277,7 @@ func init() {
withReset(cmdStageHashState)
withBlock(cmdStageHashState)
withUnwind(cmdStageHashState)
+ withPruneTo(cmdStageHashState)
withBatchSize(cmdStageHashState)
withChain(cmdStageHashState)
@@ -284,6 +287,7 @@ func init() {
withReset(cmdStageTrie)
withBlock(cmdStageTrie)
withUnwind(cmdStageTrie)
+ withPruneTo(cmdStageTrie)
withIntegrityChecks(cmdStageTrie)
withChain(cmdStageTrie)
@@ -293,6 +297,7 @@ func init() {
withReset(cmdStageHistory)
withBlock(cmdStageHistory)
withUnwind(cmdStageHistory)
+ withPruneTo(cmdStageHistory)
withChain(cmdStageHistory)
rootCmd.AddCommand(cmdStageHistory)
@@ -301,6 +306,7 @@ func init() {
withReset(cmdLogIndex)
withBlock(cmdLogIndex)
withUnwind(cmdLogIndex)
+ withPruneTo(cmdLogIndex)
withChain(cmdLogIndex)
rootCmd.AddCommand(cmdLogIndex)
@@ -309,6 +315,7 @@ func init() {
withReset(cmdCallTraces)
withBlock(cmdCallTraces)
withUnwind(cmdCallTraces)
+ withPruneTo(cmdCallTraces)
withChain(cmdCallTraces)
rootCmd.AddCommand(cmdCallTraces)
@@ -317,6 +324,7 @@ func init() {
withBlock(cmdStageTxLookup)
withUnwind(cmdStageTxLookup)
withDatadir(cmdStageTxLookup)
+ withPruneTo(cmdStageTxLookup)
withChain(cmdStageTxLookup)
rootCmd.AddCommand(cmdStageTxLookup)
@@ -333,10 +341,15 @@ func init() {
withChain(cmdRunMigrations)
rootCmd.AddCommand(cmdRunMigrations)
- withDatadir(cmdSetStorageMode)
- withChain(cmdSetStorageMode)
- cmdSetStorageMode.Flags().StringVar(&storageMode, "storage-mode", "htre", "Storage mode to override database")
- rootCmd.AddCommand(cmdSetStorageMode)
+ withDatadir(cmdSetPrune)
+ withChain(cmdSetPrune)
+ cmdSetPrune.Flags().StringVar(&pruneFlag, "prune", "hrtc", "")
+ cmdSetPrune.Flags().Uint64Var(&pruneH, "--prune.history.older", 0, "")
+ cmdSetPrune.Flags().Uint64Var(&pruneR, "--prune.receipt.older", 0, "")
+ cmdSetPrune.Flags().Uint64Var(&pruneT, "--prune.txindex.older", 0, "")
+ cmdSetPrune.Flags().Uint64Var(&pruneC, "--prune.calltrace.older", 0, "")
+ cmdSetPrune.Flags().StringSliceVar(&experiments, "experiments", nil, "Storage mode to override database")
+ rootCmd.AddCommand(cmdSetPrune)
}
func stageBodies(db ethdb.RwKV, ctx context.Context) error {
@@ -382,7 +395,7 @@ func stageSenders(db ethdb.RwKV, ctx context.Context) error {
return tx.Commit()
}
- s := stage(sync, tx, stages.Senders)
+ s := stage(sync, tx, nil, stages.Senders)
log.Info("Stage", "name", s.ID, "progress", s.BlockNumber)
cfg := stagedsync.StageSendersCfg(db, chainConfig, tmpdir)
@@ -403,7 +416,7 @@ func stageSenders(db ethdb.RwKV, ctx context.Context) error {
}
func stageExec(db ethdb.RwKV, ctx context.Context) error {
- sm, engine, chainConfig, vmConfig, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, engine, chainConfig, vmConfig, _, sync, _, _ := newSync(ctx, db, nil)
if reset {
genesis, _ := byChain()
@@ -421,16 +434,17 @@ func stageExec(db ethdb.RwKV, ctx context.Context) error {
var batchSize datasize.ByteSize
must(batchSize.UnmarshalText([]byte(batchSizeStr)))
- var s *stagedsync.StageState
- if err := db.View(ctx, func(tx ethdb.Tx) error {
- s = stage(sync, tx, stages.Execution)
- return nil
- }); err != nil {
- return err
- }
+ s := stage(sync, nil, db, stages.Execution)
log.Info("Stage", "name", s.ID, "progress", s.BlockNumber)
- cfg := stagedsync.StageExecuteBlocksCfg(db, sm.Receipts, sm.CallTraces, sm.TEVM, 0, batchSize, nil, chainConfig, engine, vmConfig, nil, false, tmpDBPath)
+ if pruneTo > 0 {
+ pm.History = prune.Distance(s.BlockNumber - pruneTo)
+ pm.Receipts = prune.Distance(s.BlockNumber - pruneTo)
+ pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo)
+ pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo)
+ }
+
+ cfg := stagedsync.StageExecuteBlocksCfg(db, pm, batchSize, nil, chainConfig, engine, vmConfig, nil, false, tmpDBPath)
if unwind > 0 {
u := sync.NewUnwindState(stages.Execution, s.BlockNumber-unwind, s.BlockNumber)
err := stagedsync.UnwindExecutionStage(u, s, nil, ctx, cfg, false)
@@ -440,6 +454,18 @@ func stageExec(db ethdb.RwKV, ctx context.Context) error {
return nil
}
+ if pruneTo > 0 {
+ p, err := sync.PruneStageState(stages.Execution, s.BlockNumber, nil, db)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneExecutionStage(p, nil, cfg, ctx, false)
+ if err != nil {
+ return err
+ }
+ return nil
+ }
+
err := stagedsync.SpawnExecuteBlocksStage(s, sync, nil, block, ctx, cfg, false)
if err != nil {
return err
@@ -448,49 +474,59 @@ func stageExec(db ethdb.RwKV, ctx context.Context) error {
}
func stageTrie(db ethdb.RwKV, ctx context.Context) error {
- _, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
tmpdir := path.Join(datadir, etl.TmpDirName)
+ tx, err := db.BeginRw(ctx)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
if reset {
- if err := db.Update(ctx, func(tx ethdb.RwTx) error { return stagedsync.ResetIH(tx) }); err != nil {
- return err
- }
+ return stagedsync.ResetIH(tx)
}
- var execStage, s *stagedsync.StageState
- if err := db.View(ctx, func(tx ethdb.Tx) error {
- execStage = stage(sync, tx, stages.Execution)
- s = stage(sync, tx, stages.IntermediateHashes)
- return nil
- }); err != nil {
- return err
+ execStage := stage(sync, tx, nil, stages.Execution)
+ s := stage(sync, tx, nil, stages.IntermediateHashes)
+
+ if pruneTo > 0 {
+ pm.History = prune.Distance(s.BlockNumber - pruneTo)
+ pm.Receipts = prune.Distance(s.BlockNumber - pruneTo)
+ pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo)
+ pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo)
}
+ fmt.Printf("distance: %d\n", pm.History)
log.Info("Stage4", "progress", execStage.BlockNumber)
log.Info("Stage5", "progress", s.BlockNumber)
cfg := stagedsync.StageTrieCfg(db, true, true, tmpdir)
if unwind > 0 {
u := sync.NewUnwindState(stages.IntermediateHashes, s.BlockNumber-unwind, s.BlockNumber)
- if err := stagedsync.UnwindIntermediateHashesStage(u, s, nil, cfg, ctx); err != nil {
+ if err := stagedsync.UnwindIntermediateHashesStage(u, s, tx, cfg, ctx); err != nil {
+ return err
+ }
+ } else if pruneTo > 0 {
+ p, err := sync.PruneStageState(stages.IntermediateHashes, s.BlockNumber, tx, db)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneIntermediateHashesStage(p, tx, cfg, ctx)
+ if err != nil {
return err
}
} else {
- if _, err := stagedsync.SpawnIntermediateHashesStage(s, nil /* Unwinder */, nil, cfg, ctx); err != nil {
+ if _, err := stagedsync.SpawnIntermediateHashesStage(s, nil /* Unwinder */, tx, cfg, ctx); err != nil {
return err
}
}
- if err := db.View(ctx, func(tx ethdb.Tx) error {
- integrity.Trie(tx, integritySlow, ctx)
- return nil
- }); err != nil {
- return err
- }
+ integrity.Trie(tx, integritySlow, ctx)
return nil
}
func stageHashState(db ethdb.RwKV, ctx context.Context) error {
tmpdir := path.Join(datadir, etl.TmpDirName)
- _, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
tx, err := db.BeginRw(ctx)
if err != nil {
@@ -506,7 +542,14 @@ func stageHashState(db ethdb.RwKV, ctx context.Context) error {
return tx.Commit()
}
- s := stage(sync, tx, stages.HashState)
+ s := stage(sync, tx, nil, stages.HashState)
+ if pruneTo > 0 {
+ pm.History = prune.Distance(s.BlockNumber - pruneTo)
+ pm.Receipts = prune.Distance(s.BlockNumber - pruneTo)
+ pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo)
+ pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo)
+ }
+
log.Info("Stage", "name", s.ID, "progress", s.BlockNumber)
cfg := stagedsync.StageHashStateCfg(db, tmpdir)
if unwind > 0 {
@@ -515,6 +558,15 @@ func stageHashState(db ethdb.RwKV, ctx context.Context) error {
if err != nil {
return err
}
+ } else if pruneTo > 0 {
+ p, err := sync.PruneStageState(stages.HashState, s.BlockNumber, tx, nil)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneHashStateStage(p, tx, cfg, ctx)
+ if err != nil {
+ return err
+ }
} else {
err = stagedsync.SpawnHashStateStage(s, tx, cfg, ctx)
if err != nil {
@@ -527,7 +579,7 @@ func stageHashState(db ethdb.RwKV, ctx context.Context) error {
func stageLogIndex(db ethdb.RwKV, ctx context.Context) error {
tmpdir := path.Join(datadir, etl.TmpDirName)
- _, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
tx, err := db.BeginRw(ctx)
if err != nil {
return err
@@ -543,17 +595,33 @@ func stageLogIndex(db ethdb.RwKV, ctx context.Context) error {
}
execAt := progress(tx, stages.Execution)
- s := stage(sync, tx, stages.LogIndex)
+ s := stage(sync, tx, nil, stages.LogIndex)
+ if pruneTo > 0 {
+ pm.History = prune.Distance(s.BlockNumber - pruneTo)
+ pm.Receipts = prune.Distance(s.BlockNumber - pruneTo)
+ pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo)
+ pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo)
+ }
+
log.Info("Stage exec", "progress", execAt)
log.Info("Stage", "name", s.ID, "progress", s.BlockNumber)
- cfg := stagedsync.StageLogIndexCfg(db, tmpdir)
+ cfg := stagedsync.StageLogIndexCfg(db, pm, tmpdir)
if unwind > 0 {
u := sync.NewUnwindState(stages.LogIndex, s.BlockNumber-unwind, s.BlockNumber)
err = stagedsync.UnwindLogIndex(u, s, tx, cfg, ctx)
if err != nil {
return err
}
+ } else if pruneTo > 0 {
+ p, err := sync.PruneStageState(stages.LogIndex, s.BlockNumber, nil, db)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneLogIndex(p, tx, cfg, ctx)
+ if err != nil {
+ return err
+ }
} else {
if err := stagedsync.SpawnLogIndex(s, tx, cfg, ctx); err != nil {
return err
@@ -565,7 +633,7 @@ func stageLogIndex(db ethdb.RwKV, ctx context.Context) error {
func stageCallTraces(kv ethdb.RwKV, ctx context.Context) error {
tmpdir := path.Join(datadir, etl.TmpDirName)
- _, _, _, _, _, sync, _, _ := newSync(ctx, kv, nil)
+ pm, _, _, _, _, sync, _, _ := newSync(ctx, kv, nil)
tx, err := kv.BeginRw(ctx)
if err != nil {
return err
@@ -583,7 +651,13 @@ func stageCallTraces(kv ethdb.RwKV, ctx context.Context) error {
must(batchSize.UnmarshalText([]byte(batchSizeStr)))
execStage := progress(tx, stages.Execution)
- s := stage(sync, tx, stages.CallTraces)
+ s := stage(sync, tx, nil, stages.CallTraces)
+ if pruneTo > 0 {
+ pm.History = prune.Distance(s.BlockNumber - pruneTo)
+ pm.Receipts = prune.Distance(s.BlockNumber - pruneTo)
+ pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo)
+ pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo)
+ }
log.Info("ID exec", "progress", execStage)
if block != 0 {
s.BlockNumber = block
@@ -591,7 +665,7 @@ func stageCallTraces(kv ethdb.RwKV, ctx context.Context) error {
}
log.Info("ID call traces", "progress", s.BlockNumber)
- cfg := stagedsync.StageCallTracesCfg(kv, block, tmpdir)
+ cfg := stagedsync.StageCallTracesCfg(kv, pm, block, tmpdir)
if unwind > 0 {
u := sync.NewUnwindState(stages.CallTraces, s.BlockNumber-unwind, s.BlockNumber)
@@ -599,6 +673,15 @@ func stageCallTraces(kv ethdb.RwKV, ctx context.Context) error {
if err != nil {
return err
}
+ } else if pruneTo > 0 {
+ p, err := sync.PruneStageState(stages.CallTraces, s.BlockNumber, tx, nil)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneCallTraces(p, tx, cfg, ctx)
+ if err != nil {
+ return err
+ }
} else {
if err := stagedsync.SpawnCallTraces(s, tx, cfg, ctx); err != nil {
return err
@@ -609,7 +692,7 @@ func stageCallTraces(kv ethdb.RwKV, ctx context.Context) error {
func stageHistory(db ethdb.RwKV, ctx context.Context) error {
tmpdir := path.Join(datadir, etl.TmpDirName)
- _, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
tx, err := db.BeginRw(ctx)
if err != nil {
return err
@@ -624,13 +707,19 @@ func stageHistory(db ethdb.RwKV, ctx context.Context) error {
return tx.Commit()
}
execStage := progress(tx, stages.Execution)
- stageStorage := stage(sync, tx, stages.StorageHistoryIndex)
- stageAcc := stage(sync, tx, stages.AccountHistoryIndex)
+ stageStorage := stage(sync, tx, nil, stages.StorageHistoryIndex)
+ stageAcc := stage(sync, tx, nil, stages.AccountHistoryIndex)
+ if pruneTo > 0 {
+ pm.History = prune.Distance(stageAcc.BlockNumber - pruneTo)
+ pm.Receipts = prune.Distance(stageAcc.BlockNumber - pruneTo)
+ pm.CallTraces = prune.Distance(stageAcc.BlockNumber - pruneTo)
+ pm.TxIndex = prune.Distance(stageAcc.BlockNumber - pruneTo)
+ }
log.Info("ID exec", "progress", execStage)
log.Info("ID acc history", "progress", stageAcc.BlockNumber)
log.Info("ID storage history", "progress", stageStorage.BlockNumber)
- cfg := stagedsync.StageHistoryCfg(db, tmpdir)
+ cfg := stagedsync.StageHistoryCfg(db, pm, tmpdir)
if unwind > 0 { //nolint:staticcheck
u := sync.NewUnwindState(stages.StorageHistoryIndex, stageStorage.BlockNumber-unwind, stageStorage.BlockNumber)
if err := stagedsync.UnwindStorageHistoryIndex(u, stageStorage, tx, cfg, ctx); err != nil {
@@ -640,6 +729,23 @@ func stageHistory(db ethdb.RwKV, ctx context.Context) error {
if err := stagedsync.UnwindAccountHistoryIndex(u, stageAcc, tx, cfg, ctx); err != nil {
return err
}
+ } else if pruneTo > 0 {
+ pa, err := sync.PruneStageState(stages.AccountHistoryIndex, stageAcc.BlockNumber, tx, db)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneAccountHistoryIndex(pa, tx, cfg, ctx)
+ if err != nil {
+ return err
+ }
+ ps, err := sync.PruneStageState(stages.StorageHistoryIndex, stageStorage.BlockNumber, tx, db)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneAccountHistoryIndex(ps, tx, cfg, ctx)
+ if err != nil {
+ return err
+ }
} else {
if err := stagedsync.SpawnAccountHistoryIndex(stageAcc, tx, cfg, ctx); err != nil {
return err
@@ -654,7 +760,7 @@ func stageHistory(db ethdb.RwKV, ctx context.Context) error {
func stageTxLookup(db ethdb.RwKV, ctx context.Context) error {
tmpdir := path.Join(datadir, etl.TmpDirName)
- _, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, _, _, _, _, sync, _, _ := newSync(ctx, db, nil)
tx, err := db.BeginRw(ctx)
if err != nil {
@@ -669,16 +775,31 @@ func stageTxLookup(db ethdb.RwKV, ctx context.Context) error {
}
return tx.Commit()
}
- s := stage(sync, tx, stages.TxLookup)
+ s := stage(sync, tx, nil, stages.TxLookup)
+ if pruneTo > 0 {
+ pm.History = prune.Distance(s.BlockNumber - pruneTo)
+ pm.Receipts = prune.Distance(s.BlockNumber - pruneTo)
+ pm.CallTraces = prune.Distance(s.BlockNumber - pruneTo)
+ pm.TxIndex = prune.Distance(s.BlockNumber - pruneTo)
+ }
log.Info("Stage", "name", s.ID, "progress", s.BlockNumber)
- cfg := stagedsync.StageTxLookupCfg(db, tmpdir)
+ cfg := stagedsync.StageTxLookupCfg(db, pm, tmpdir)
if unwind > 0 {
u := sync.NewUnwindState(stages.TxLookup, s.BlockNumber-unwind, s.BlockNumber)
err = stagedsync.UnwindTxLookup(u, s, tx, cfg, ctx)
if err != nil {
return err
}
+ } else if pruneTo > 0 {
+ p, err := sync.PruneStageState(stages.TxPool, s.BlockNumber, tx, nil)
+ if err != nil {
+ return err
+ }
+ err = stagedsync.PruneTxLookup(p, tx, cfg, ctx)
+ if err != nil {
+ return err
+ }
} else {
err = stagedsync.SpawnTxLookup(s, tx, cfg, ctx)
if err != nil {
@@ -743,15 +864,14 @@ func byChain() (*core.Genesis, *params.ChainConfig) {
return genesis, chainConfig
}
-func newSync(ctx context.Context, db ethdb.RwKV, miningConfig *params.MiningConfig) (ethdb.StorageMode, consensus.Engine, *params.ChainConfig, *vm.Config, *core.TxPool, *stagedsync.Sync, *stagedsync.Sync, stagedsync.MiningState) {
+func newSync(ctx context.Context, db ethdb.RwKV, miningConfig *params.MiningConfig) (prune.Mode, consensus.Engine, *params.ChainConfig, *vm.Config, *core.TxPool, *stagedsync.Sync, *stagedsync.Sync, stagedsync.MiningState) {
tmpdir := path.Join(datadir, etl.TmpDirName)
snapshotDir = path.Join(datadir, "erigon", "snapshot")
- var sm ethdb.StorageMode
-
+ var pm prune.Mode
var err error
if err = db.View(context.Background(), func(tx ethdb.Tx) error {
- sm, err = ethdb.GetStorageModeFromDB(tx)
+ pm, err = prune.Get(tx)
if err != nil {
return err
}
@@ -759,7 +879,7 @@ func newSync(ctx context.Context, db ethdb.RwKV, miningConfig *params.MiningConf
}); err != nil {
panic(err)
}
- vmConfig := &vm.Config{NoReceipts: !sm.Receipts}
+ vmConfig := &vm.Config{}
genesis, chainConfig := byChain()
var engine consensus.Engine
@@ -773,7 +893,7 @@ func newSync(ctx context.Context, db ethdb.RwKV, miningConfig *params.MiningConf
txPool := core.NewTxPool(ethconfig.Defaults.TxPool, chainConfig, db)
- chainConfig, genesisBlock, genesisErr := core.CommitGenesisBlock(db, genesis, sm.History)
+ chainConfig, genesisBlock, genesisErr := core.CommitGenesisBlock(db, genesis)
if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok {
panic(genesisErr)
}
@@ -800,7 +920,7 @@ func newSync(ctx context.Context, db ethdb.RwKV, miningConfig *params.MiningConf
txPoolP2PServer.TxFetcher = fetcher.NewTxFetcher(txPool.Has, txPool.AddRemotes, fetchTx)
cfg := ethconfig.Defaults
- cfg.StorageMode = sm
+ cfg.Prune = pm
cfg.BatchSize = batchSize
if miningConfig != nil {
cfg.Miner = *miningConfig
@@ -830,7 +950,7 @@ func newSync(ctx context.Context, db ethdb.RwKV, miningConfig *params.MiningConf
stagedsync.MiningPruneOrder,
)
- return sm, engine, chainConfig, vmConfig, txPool, sync, miningSync, miner
+ return pm, engine, chainConfig, vmConfig, txPool, sync, miningSync, miner
}
func progress(tx ethdb.KVGetter, stage stages.SyncStage) uint64 {
@@ -841,8 +961,8 @@ func progress(tx ethdb.KVGetter, stage stages.SyncStage) uint64 {
return res
}
-func stage(st *stagedsync.Sync, tx ethdb.Tx, stage stages.SyncStage) *stagedsync.StageState {
- res, err := st.StageState(stage, tx, nil)
+func stage(st *stagedsync.Sync, tx ethdb.Tx, db ethdb.RoKV, stage stages.SyncStage) *stagedsync.StageState {
+ res, err := st.StageState(stage, tx, db)
if err != nil {
panic(err)
}
@@ -850,19 +970,19 @@ func stage(st *stagedsync.Sync, tx ethdb.Tx, stage stages.SyncStage) *stagedsync
}
func overrideStorageMode(db ethdb.RwKV) error {
- sm, err := ethdb.StorageModeFromString(storageMode)
+ pm, err := prune.FromCli(pruneFlag, pruneH, pruneR, pruneT, pruneC, experiments)
if err != nil {
return err
}
return db.Update(context.Background(), func(tx ethdb.RwTx) error {
- if err = ethdb.OverrideStorageMode(tx, sm); err != nil {
+ if err = prune.Override(tx, pm); err != nil {
return err
}
- sm, err = ethdb.GetStorageModeFromDB(tx)
+ pm, err = prune.Get(tx)
if err != nil {
return err
}
- log.Info("Storage mode in DB", "mode", sm.ToString())
+ log.Info("Storage mode in DB", "mode", pm.ToString())
return nil
})
}
diff --git a/cmd/integration/commands/state_stages.go b/cmd/integration/commands/state_stages.go
index b0f7aec3fec917c5f16691e879c226d895368476..1fa961a484b2c54a62149db5f73373292709eb50 100644
--- a/cmd/integration/commands/state_stages.go
+++ b/cmd/integration/commands/state_stages.go
@@ -139,7 +139,7 @@ func init() {
}
func syncBySmallSteps(db ethdb.RwKV, miningConfig params.MiningConfig, ctx context.Context) error {
- sm, engine, chainConfig, vmConfig, txPool, stateStages, miningStages, miner := newSync(ctx, db, &miningConfig)
+ pm, engine, chainConfig, vmConfig, txPool, stateStages, miningStages, miner := newSync(ctx, db, &miningConfig)
tx, err := db.BeginRw(ctx)
if err != nil {
@@ -178,7 +178,7 @@ func syncBySmallSteps(db ethdb.RwKV, miningConfig params.MiningConfig, ctx conte
stages.TxPool, // TODO: enable TxPool stage
stages.Finish)
- execCfg := stagedsync.StageExecuteBlocksCfg(db, sm.Receipts, sm.CallTraces, sm.TEVM, 0, batchSize, changeSetHook, chainConfig, engine, vmConfig, nil, false, tmpDir)
+ execCfg := stagedsync.StageExecuteBlocksCfg(db, pm, batchSize, changeSetHook, chainConfig, engine, vmConfig, nil, false, tmpDir)
execUntilFunc := func(execToBlock uint64) func(firstCycle bool, stageState *stagedsync.StageState, unwinder stagedsync.Unwinder, tx ethdb.RwTx) error {
return func(firstCycle bool, s *stagedsync.StageState, unwinder stagedsync.Unwinder, tx ethdb.RwTx) error {
@@ -274,7 +274,7 @@ func syncBySmallSteps(db ethdb.RwKV, miningConfig params.MiningConfig, ctx conte
}
if integrityFast {
- if err := checkChanges(expectedAccountChanges, tx, expectedStorageChanges, execAtBlock, sm.History); err != nil {
+ if err := checkChanges(expectedAccountChanges, tx, expectedStorageChanges, execAtBlock, pm.History.PruneTo(execToBlock)); err != nil {
return err
}
integrity.Trie(tx, integritySlow, ctx)
@@ -370,7 +370,11 @@ func syncBySmallSteps(db ethdb.RwKV, miningConfig params.MiningConfig, ctx conte
return nil
}
-func checkChanges(expectedAccountChanges map[uint64]*changeset.ChangeSet, tx ethdb.Tx, expectedStorageChanges map[uint64]*changeset.ChangeSet, execAtBlock uint64, historyEnabled bool) error {
+func checkChanges(expectedAccountChanges map[uint64]*changeset.ChangeSet, tx ethdb.Tx, expectedStorageChanges map[uint64]*changeset.ChangeSet, execAtBlock, prunedTo uint64) error {
+ checkHistoryFrom := execAtBlock
+ if prunedTo > checkHistoryFrom {
+ checkHistoryFrom = prunedTo
+ }
for blockN := range expectedAccountChanges {
if err := checkChangeSet(tx, blockN, expectedAccountChanges[blockN], expectedStorageChanges[blockN]); err != nil {
return err
@@ -379,13 +383,11 @@ func checkChanges(expectedAccountChanges map[uint64]*changeset.ChangeSet, tx eth
delete(expectedStorageChanges, blockN)
}
- if historyEnabled {
- if err := checkHistory(tx, dbutils.AccountChangeSetBucket, execAtBlock); err != nil {
- return err
- }
- if err := checkHistory(tx, dbutils.StorageChangeSetBucket, execAtBlock); err != nil {
- return err
- }
+ if err := checkHistory(tx, dbutils.AccountChangeSetBucket, checkHistoryFrom); err != nil {
+ return err
+ }
+ if err := checkHistory(tx, dbutils.StorageChangeSetBucket, checkHistoryFrom); err != nil {
+ return err
}
return nil
}
@@ -418,16 +420,16 @@ func loopIh(db ethdb.RwKV, ctx context.Context, unwind uint64) error {
if err = sync.Run(db, tx, false); err != nil {
return err
}
- execStage := stage(sync, tx, stages.HashState)
+ execStage := stage(sync, tx, nil, stages.HashState)
to := execStage.BlockNumber - unwind
_ = sync.SetCurrentStage(stages.HashState)
u := &stagedsync.UnwindState{ID: stages.HashState, UnwindPoint: to}
- if err = stagedsync.UnwindHashStateStage(u, stage(sync, tx, stages.HashState), tx, stagedsync.StageHashStateCfg(db, tmpdir), ctx); err != nil {
+ if err = stagedsync.UnwindHashStateStage(u, stage(sync, tx, nil, stages.HashState), tx, stagedsync.StageHashStateCfg(db, tmpdir), ctx); err != nil {
return err
}
_ = sync.SetCurrentStage(stages.IntermediateHashes)
u = &stagedsync.UnwindState{ID: stages.IntermediateHashes, UnwindPoint: to}
- if err = stagedsync.UnwindIntermediateHashesStage(u, stage(sync, tx, stages.IntermediateHashes), tx, stagedsync.StageTrieCfg(db, true, true, tmpdir), ctx); err != nil {
+ if err = stagedsync.UnwindIntermediateHashesStage(u, stage(sync, tx, nil, stages.IntermediateHashes), tx, stagedsync.StageTrieCfg(db, true, true, tmpdir), ctx); err != nil {
return err
}
must(tx.Commit())
@@ -471,7 +473,7 @@ func loopIh(db ethdb.RwKV, ctx context.Context, unwind uint64) error {
}
func loopExec(db ethdb.RwKV, ctx context.Context, unwind uint64) error {
- _, engine, chainConfig, vmConfig, _, sync, _, _ := newSync(ctx, db, nil)
+ pm, engine, chainConfig, vmConfig, _, sync, _, _ := newSync(ctx, db, nil)
tx, err := db.BeginRw(ctx)
if err != nil {
@@ -490,7 +492,7 @@ func loopExec(db ethdb.RwKV, ctx context.Context, unwind uint64) error {
from := progress(tx, stages.Execution)
to := from + unwind
- cfg := stagedsync.StageExecuteBlocksCfg(db, true, false, false, 0, batchSize, nil, chainConfig, engine, vmConfig, nil, false, tmpDBPath)
+ cfg := stagedsync.StageExecuteBlocksCfg(db, pm, batchSize, nil, chainConfig, engine, vmConfig, nil, false, tmpDBPath)
// set block limit of execute stage
sync.MockExecFunc(stages.Execution, func(firstCycle bool, stageState *stagedsync.StageState, unwinder stagedsync.Unwinder, tx ethdb.RwTx) error {
diff --git a/cmd/pics/state.go b/cmd/pics/state.go
index 9600ab2224e1780948bf2538c18e148f86f7a407..7d7f58ee5f85cebf411e91c8a172333a75e0db76 100644
--- a/cmd/pics/state.go
+++ b/cmd/pics/state.go
@@ -69,7 +69,7 @@ import (
}*/
var bucketLabels = map[string]string{
- dbutils.BlockReceiptsPrefix: "Receipts",
+ dbutils.Receipts: "Receipts",
dbutils.Log: "Event Logs",
dbutils.AccountsHistoryBucket: "History Of Accounts",
dbutils.StorageHistoryBucket: "History Of Storage",
diff --git a/cmd/rpcdaemon/README.md b/cmd/rpcdaemon/README.md
index 2523e50b4a30531f23ed41b2b7d1386e0571fe74..277fd82f7467fa682b5bb71df7ba3c8c0d8da47b 100644
--- a/cmd/rpcdaemon/README.md
+++ b/cmd/rpcdaemon/README.md
@@ -4,6 +4,7 @@
* [Running remotely](#running-remotely)
* [Testing](#testing)
- [FAQ](#faq)
+ * [Relations between prune options and rpc methods](#relations-between-prune-options-and-rpc-method)
* [RPC Implementation Status](#rpc-implementation-status)
* [Securing the communication between RPC daemon and Erigon instance via TLS and authentication](#securing-the-communication-between-rpc-daemon-and-erigon-instance-via-tls-and-authentication)
* [Ethstats](#ethstats)
@@ -89,6 +90,21 @@ to test the RPC.
## FAQ
+### Relations between prune options and RPC methods
+
+Next options available (by `--prune` flag):
+
+```
+* h - prune history (ChangeSets, HistoryIndices - used to access historical state)
+* r - prune receipts (Receipts, Logs, LogTopicIndex, LogAddressIndex - used by eth_getLogs and similar RPC methods)
+* t - prune tx lookup (used to get transaction by hash)
+* c - prune call traces (used by trace_* methods)
+```
+
+By default data pruned after 90K blocks, can change it by flags like `--prune.history.after=100_000`
+
+Some methods, if not found historical data in DB, can fallback to old blocks re-execution - but it require `h`.
+
### RPC Implementation Status
The following table shows the current implementation status of Erigon's RPC daemon.
diff --git a/cmd/rpcdaemon/commands/get_chain_config_test.go b/cmd/rpcdaemon/commands/get_chain_config_test.go
index 6c314310d024348be9c7a032bbad40930caadcdb..4f7d0cf636e45ce0881b53d7505289092dec2de1 100644
--- a/cmd/rpcdaemon/commands/get_chain_config_test.go
+++ b/cmd/rpcdaemon/commands/get_chain_config_test.go
@@ -10,7 +10,7 @@ import (
func TestGetChainConfig(t *testing.T) {
db := kv.NewTestKV(t)
- config, _, err := core.CommitGenesisBlock(db, core.DefaultGenesisBlock(), false)
+ config, _, err := core.CommitGenesisBlock(db, core.DefaultGenesisBlock())
if err != nil {
t.Fatalf("setting up genensis block: %v", err)
}
diff --git a/cmd/snapshots/debug/debug_test.go b/cmd/snapshots/debug/debug_test.go
index b1792269945b2cc52f5b646b70111dbbe8451a5c..ec0e6eca11b3d1b2a7251eeafdd98cd72c53ffb2 100644
--- a/cmd/snapshots/debug/debug_test.go
+++ b/cmd/snapshots/debug/debug_test.go
@@ -57,7 +57,7 @@ func TestMatreshkaStream(t *testing.T) {
t.Fatal(err)
}
- chainConfig, _, genesisErr := core.CommitGenesisBlock(kv, core.DefaultGenesisBlock(), true)
+ chainConfig, _, genesisErr := core.CommitGenesisBlock(kv, core.DefaultGenesisBlock())
if genesisErr != nil {
t.Fatal(err)
}
diff --git a/cmd/state/generate/regenerate_tx_lookup.go b/cmd/state/generate/regenerate_tx_lookup.go
index d49e9c3be6c3a5e002aad294a3d6277373303841..6047c2265d1482ba1dd9d28c0ebc5b71625d3709 100644
--- a/cmd/state/generate/regenerate_tx_lookup.go
+++ b/cmd/state/generate/regenerate_tx_lookup.go
@@ -10,6 +10,7 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
)
@@ -34,13 +35,21 @@ func RegenerateTxLookup(chaindata string) error {
close(quitCh)
}()
+ pm, err := prune.Get(tx)
+ if err != nil {
+ return err
+ }
lastExecutedBlock, err := stages.GetStageProgress(tx, stages.Execution)
if err != nil {
//There could be headers without block in the end
log.Error("Cant get last executed block", "err", err)
}
log.Info("TxLookup generation started", "start time", startTime)
- err = stagedsync.TxLookupTransform("txlookup", tx, dbutils.EncodeBlockNumber(0), dbutils.EncodeBlockNumber(lastExecutedBlock+1), quitCh, stagedsync.StageTxLookupCfg(db, os.TempDir()))
+ err = stagedsync.TxLookupTransform("txlookup", tx,
+ dbutils.EncodeBlockNumber(pm.TxIndex.PruneTo(lastExecutedBlock)),
+ dbutils.EncodeBlockNumber(lastExecutedBlock+1),
+ quitCh,
+ stagedsync.StageTxLookupCfg(db, pm, os.TempDir()))
if err != nil {
return err
}
diff --git a/common/changeset/changeset.go b/common/changeset/changeset.go
index d6cf1937a3b44e9153ffa6db31dac884ef546d41..3e2db9f779e6e56a5c3703fa1f3802c5206f5ac5 100644
--- a/common/changeset/changeset.go
+++ b/common/changeset/changeset.go
@@ -8,6 +8,7 @@ import (
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/dbutils"
+ "github.com/ledgerwatch/erigon/common/math"
"github.com/ledgerwatch/erigon/ethdb"
)
@@ -103,11 +104,6 @@ func (s *ChangeSet) String() string {
}
// Encoded Method
-
-func Len(b []byte) int {
- return int(binary.BigEndian.Uint32(b[0:4]))
-}
-
func FromDBFormat(dbKey, dbValue []byte) (uint64, []byte, []byte) {
if len(dbKey) == 8 {
return DecodeAccounts(dbKey, dbValue)
@@ -116,6 +112,37 @@ func FromDBFormat(dbKey, dbValue []byte) (uint64, []byte, []byte) {
}
}
+func AvailableFrom(tx ethdb.Tx) (uint64, error) {
+ c, err := tx.Cursor(dbutils.AccountChangeSetBucket)
+ if err != nil {
+ return math.MaxUint64, err
+ }
+ defer c.Close()
+ k, _, err := c.First()
+ if err != nil {
+ return math.MaxUint64, err
+ }
+ if len(k) == 0 {
+ return math.MaxUint64, nil
+ }
+ return binary.BigEndian.Uint64(k), nil
+}
+func AvailableStorageFrom(tx ethdb.Tx) (uint64, error) {
+ c, err := tx.Cursor(dbutils.StorageChangeSetBucket)
+ if err != nil {
+ return math.MaxUint64, err
+ }
+ defer c.Close()
+ k, _, err := c.First()
+ if err != nil {
+ return math.MaxUint64, err
+ }
+ if len(k) == 0 {
+ return math.MaxUint64, nil
+ }
+ return binary.BigEndian.Uint64(k), nil
+}
+
func Walk(db ethdb.Tx, bucket string, startkey []byte, fixedbits int, walker func(blockN uint64, k, v []byte) (bool, error)) error {
var blockN uint64
c, err := db.Cursor(bucket)
diff --git a/common/dbutils/bucket.go b/common/dbutils/bucket.go
index ebea3fe55f9eb6a86a2f0baf4540e95d8025958e..5d3acafa110b987f4e93f747d8be7f2232ac3dc0 100644
--- a/common/dbutils/bucket.go
+++ b/common/dbutils/bucket.go
@@ -111,6 +111,27 @@ It allows:
see also: docs/programmers_guide/db_walkthrough.MD#table-change-sets
+
+addr+max(shard)
+
+addr+123
+addr+234
+addr+999
+
+prune(150):
+cs.Walk(0-150,k,v {
+ etl.Collect(k)
+})
+
+elt.Load(k {
+ c.Seek(k:150)
+ for c.Prev() { c.Del() }
+})
+
+elt.Load(k {
+ for addr,n=c.Seek(k); addr==k && n<150 ;c.Next() { c.Del() }
+})
+
AccountsHistoryBucket:
key - address + shard_id_u64
value - roaring bitmap - list of block where it changed
@@ -199,10 +220,10 @@ const (
HeadersBucket = "Header" // block_num_u64 + hash -> header (RLP)
HeaderTDBucket = "HeadersTotalDifficulty" // block_num_u64 + hash -> td (RLP)
- BlockBodyPrefix = "BlockBody" // block_num_u64 + hash -> block body
- EthTx = "BlockTransaction" // tbl_sequence_u64 -> rlp(tx)
- BlockReceiptsPrefix = "Receipt" // block_num_u64 -> canonical block receipts (non-canonical are not stored)
- Log = "TransactionLog" // block_num_u64 + txId -> logs of transaction
+ BlockBodyPrefix = "BlockBody" // block_num_u64 + hash -> block body
+ EthTx = "BlockTransaction" // tbl_sequence_u64 -> rlp(tx)
+ Receipts = "Receipt" // block_num_u64 -> canonical block receipts (non-canonical are not stored)
+ Log = "TransactionLog" // block_num_u64 + txId -> logs of transaction
// Stores bitmap indices - in which block numbers saw logs of given 'address' or 'topic'
// [addr or topic] + [2 bytes inverted shard number] -> bitmap(blockN)
@@ -262,17 +283,14 @@ const (
// Keys
var (
- //StorageModeHistory - does node save history.
- StorageModeHistory = []byte("smHistory")
- //StorageModeReceipts - does node save receipts.
- StorageModeReceipts = []byte("smReceipts")
- //StorageModeTxIndex - does node save transactions index.
- StorageModeTxIndex = []byte("smTxIndex")
- //StorageModeCallTraces - does not build index of call traces
- StorageModeCallTraces = []byte("smCallTraces")
//StorageModeTEVM - does not translate EVM to TEVM
StorageModeTEVM = []byte("smTEVM")
+ PruneDistanceHistory = []byte("pruneHistory")
+ PruneDistanceReceipts = []byte("pruneReceipts")
+ PruneDistanceTxIndex = []byte("pruneTxIndex")
+ PruneDistanceCallTraces = []byte("pruneCallTraces")
+
DBSchemaVersionKey = []byte("dbVersion")
BittorrentPeerID = "peerID"
@@ -292,7 +310,7 @@ var Buckets = []string{
ContractCodeBucket,
HeaderNumberBucket,
BlockBodyPrefix,
- BlockReceiptsPrefix,
+ Receipts,
TxLookupPrefix,
BloomBitsPrefix,
ConfigPrefix,
diff --git a/common/etl/collector.go b/common/etl/collector.go
index 5b1ba1c84935c91ae497503e5a7e28896e0e851e..9a47bbe23bbe7d5f64d10e08033a63a30f218ea3 100644
--- a/common/etl/collector.go
+++ b/common/etl/collector.go
@@ -22,7 +22,7 @@ import (
const TmpDirName = "etl-temp"
type LoadNextFunc func(originalK, k, v []byte) error
-type LoadFunc func(k []byte, value []byte, table CurrentTableReader, next LoadNextFunc) error
+type LoadFunc func(k, v []byte, table CurrentTableReader, next LoadNextFunc) error
// Collector performs the job of ETL Transform, but can also be used without "E" (Extract) part
// as a Collect Transform Load
diff --git a/core/genesis.go b/core/genesis.go
index 2000716e72926706fa593e452936a8384f42d4d2..c59482699e90c61d9212efb6736c4947372d4d81 100644
--- a/core/genesis.go
+++ b/core/genesis.go
@@ -170,13 +170,13 @@ func (e *GenesisMismatchError) Error() string {
// error is a *params.ConfigCompatError and the new, unwritten config is returned.
//
// The returned chain configuration is never nil.
-func CommitGenesisBlock(db ethdb.RwKV, genesis *Genesis, history bool) (*params.ChainConfig, *types.Block, error) {
+func CommitGenesisBlock(db ethdb.RwKV, genesis *Genesis) (*params.ChainConfig, *types.Block, error) {
tx, err := db.BeginRw(context.Background())
if err != nil {
return nil, nil, err
}
defer tx.Rollback()
- c, b, err := WriteGenesisBlock(tx, genesis, history)
+ c, b, err := WriteGenesisBlock(tx, genesis)
if err != nil {
return c, b, err
}
@@ -187,15 +187,15 @@ func CommitGenesisBlock(db ethdb.RwKV, genesis *Genesis, history bool) (*params.
return c, b, nil
}
-func MustCommitGenesisBlock(db ethdb.RwKV, genesis *Genesis, history bool) (*params.ChainConfig, *types.Block) {
- c, b, err := CommitGenesisBlock(db, genesis, history)
+func MustCommitGenesisBlock(db ethdb.RwKV, genesis *Genesis) (*params.ChainConfig, *types.Block) {
+ c, b, err := CommitGenesisBlock(db, genesis)
if err != nil {
panic(err)
}
return c, b
}
-func OverrideGenesisBlock(db ethdb.RwTx, genesis *Genesis, history bool) (*params.ChainConfig, *types.Block, error) {
+func OverrideGenesisBlock(db ethdb.RwTx, genesis *Genesis) (*params.ChainConfig, *types.Block, error) {
stored, err := rawdb.ReadCanonicalHash(db, 0)
if err != nil {
return nil, nil, err
@@ -208,10 +208,10 @@ func OverrideGenesisBlock(db ethdb.RwTx, genesis *Genesis, history bool) (*param
if err != nil {
return nil, nil, err
}
- return WriteGenesisBlock(db, genesis, history)
+ return WriteGenesisBlock(db, genesis)
}
-func WriteGenesisBlock(db ethdb.RwTx, genesis *Genesis, history bool) (*params.ChainConfig, *types.Block, error) {
+func WriteGenesisBlock(db ethdb.RwTx, genesis *Genesis) (*params.ChainConfig, *types.Block, error) {
if genesis != nil && genesis.Config == nil {
return params.AllEthashProtocolChanges, nil, ErrGenesisNoConfig
}
@@ -227,7 +227,7 @@ func WriteGenesisBlock(db ethdb.RwTx, genesis *Genesis, history bool) (*params.C
genesis = DefaultGenesisBlock()
custom = false
}
- block, _, err1 := genesis.Write(db, history)
+ block, _, err1 := genesis.Write(db)
if err1 != nil {
return genesis.Config, nil, err1
}
@@ -405,7 +405,7 @@ func (g *Genesis) ToBlock() (*types.Block, *state.IntraBlockState, error) {
return types.NewBlock(head, nil, nil, nil), statedb, nil
}
-func (g *Genesis) WriteGenesisState(tx ethdb.RwTx, history bool) (*types.Block, *state.IntraBlockState, error) {
+func (g *Genesis) WriteGenesisState(tx ethdb.RwTx) (*types.Block, *state.IntraBlockState, error) {
block, statedb, err := g.ToBlock()
if err != nil {
return nil, nil, err
@@ -433,17 +433,14 @@ func (g *Genesis) WriteGenesisState(tx ethdb.RwTx, history bool) (*types.Block,
if err := blockWriter.WriteChangeSets(); err != nil {
return nil, statedb, fmt.Errorf("cannot write change sets: %v", err)
}
- // Optionally write history
- if history {
- if err := blockWriter.WriteHistory(); err != nil {
- return nil, statedb, fmt.Errorf("cannot write history: %v", err)
- }
+ if err := blockWriter.WriteHistory(); err != nil {
+ return nil, statedb, fmt.Errorf("cannot write history: %v", err)
}
return block, statedb, nil
}
func (g *Genesis) MustWrite(tx ethdb.RwTx, history bool) (*types.Block, *state.IntraBlockState) {
- b, s, err := g.Write(tx, history)
+ b, s, err := g.Write(tx)
if err != nil {
panic(err)
}
@@ -452,8 +449,8 @@ func (g *Genesis) MustWrite(tx ethdb.RwTx, history bool) (*types.Block, *state.I
// Commit writes the block and state of a genesis specification to the database.
// The block is committed as the canonical head block.
-func (g *Genesis) Write(tx ethdb.RwTx, history bool) (*types.Block, *state.IntraBlockState, error) {
- block, statedb, err2 := g.WriteGenesisState(tx, history)
+func (g *Genesis) Write(tx ethdb.RwTx) (*types.Block, *state.IntraBlockState, error) {
+ block, statedb, err2 := g.WriteGenesisState(tx)
if err2 != nil {
return block, statedb, err2
}
@@ -494,7 +491,7 @@ func (g *Genesis) Commit(db ethdb.Database, history bool) (*types.Block, error)
return nil, err
}
defer tx.Rollback()
- block, _, err := g.Write(tx.(ethdb.HasTx).Tx().(ethdb.RwTx), history)
+ block, _, err := g.Write(tx.(ethdb.HasTx).Tx().(ethdb.RwTx))
if err != nil {
return block, err
}
diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go
index c50926af1b91516b01eed9e292244f21cfd1fc1a..79df3f1e8a276b24e2e74b68baf035f459e12262 100644
--- a/core/rawdb/accessors_chain.go
+++ b/core/rawdb/accessors_chain.go
@@ -20,6 +20,7 @@ import (
"bytes"
"encoding/binary"
"fmt"
+ "math"
"math/big"
"github.com/ledgerwatch/erigon/common"
@@ -510,7 +511,7 @@ func DeleteTd(db ethdb.Deleter, hash common.Hash, number uint64) error {
// HasReceipts verifies the existence of all the transaction receipts belonging
// to a block.
func HasReceipts(db ethdb.Has, hash common.Hash, number uint64) bool {
- if has, err := db.Has(dbutils.BlockReceiptsPrefix, dbutils.ReceiptsKey(number)); !has || err != nil {
+ if has, err := db.Has(dbutils.Receipts, dbutils.ReceiptsKey(number)); !has || err != nil {
return false
}
return true
@@ -521,7 +522,7 @@ func HasReceipts(db ethdb.Has, hash common.Hash, number uint64) bool {
// should not be used. Use ReadReceipts instead if the metadata is needed.
func ReadRawReceipts(db ethdb.Tx, blockNum uint64) types.Receipts {
// Retrieve the flattened receipt slice
- data, err := db.GetOne(dbutils.BlockReceiptsPrefix, dbutils.ReceiptsKey(blockNum))
+ data, err := db.GetOne(dbutils.Receipts, dbutils.ReceiptsKey(blockNum))
if err != nil {
log.Error("ReadRawReceipts failed", "err", err)
}
@@ -616,7 +617,7 @@ func WriteReceipts(tx ethdb.Putter, number uint64, receipts types.Receipts) erro
return fmt.Errorf("encode block receipts for block %d: %v", number, err)
}
- if err = tx.Put(dbutils.BlockReceiptsPrefix, dbutils.ReceiptsKey(number), buf.Bytes()); err != nil {
+ if err = tx.Put(dbutils.Receipts, dbutils.ReceiptsKey(number), buf.Bytes()); err != nil {
return fmt.Errorf("writing receipts for block %d: %v", number, err)
}
return nil
@@ -625,6 +626,7 @@ func WriteReceipts(tx ethdb.Putter, number uint64, receipts types.Receipts) erro
// AppendReceipts stores all the transaction receipts belonging to a block.
func AppendReceipts(tx ethdb.RwTx, blockNumber uint64, receipts types.Receipts) error {
buf := bytes.NewBuffer(make([]byte, 0, 1024))
+
for txId, r := range receipts {
//fmt.Printf("1: %d,%x\n", txId, r.TxHash)
if len(r.Logs) == 0 {
@@ -648,7 +650,7 @@ func AppendReceipts(tx ethdb.RwTx, blockNumber uint64, receipts types.Receipts)
return fmt.Errorf("encode block receipts for block %d: %v", blockNumber, err)
}
- if err = tx.Append(dbutils.BlockReceiptsPrefix, dbutils.ReceiptsKey(blockNumber), buf.Bytes()); err != nil {
+ if err = tx.Append(dbutils.Receipts, dbutils.ReceiptsKey(blockNumber), buf.Bytes()); err != nil {
return fmt.Errorf("writing receipts for block %d: %v", blockNumber, err)
}
return nil
@@ -656,7 +658,7 @@ func AppendReceipts(tx ethdb.RwTx, blockNumber uint64, receipts types.Receipts)
// DeleteReceipts removes all receipt data associated with a block hash.
func DeleteReceipts(db ethdb.RwTx, number uint64) error {
- if err := db.Delete(dbutils.BlockReceiptsPrefix, dbutils.ReceiptsKey(number), nil); err != nil {
+ if err := db.Delete(dbutils.Receipts, dbutils.ReceiptsKey(number), nil); err != nil {
return fmt.Errorf("receipts delete failed: %d, %w", number, err)
}
@@ -672,8 +674,8 @@ func DeleteReceipts(db ethdb.RwTx, number uint64) error {
// DeleteNewerReceipts removes all receipt for given block number or newer
func DeleteNewerReceipts(db ethdb.RwTx, number uint64) error {
- if err := db.ForEach(dbutils.BlockReceiptsPrefix, dbutils.ReceiptsKey(number), func(k, v []byte) error {
- return db.Delete(dbutils.BlockReceiptsPrefix, k, nil)
+ if err := db.ForEach(dbutils.Receipts, dbutils.ReceiptsKey(number), func(k, v []byte) error {
+ return db.Delete(dbutils.Receipts, k, nil)
}); err != nil {
return err
}
@@ -688,6 +690,22 @@ func DeleteNewerReceipts(db ethdb.RwTx, number uint64) error {
return nil
}
+func ReceiptsAvailableFrom(tx ethdb.Tx) (uint64, error) {
+ c, err := tx.Cursor(dbutils.Receipts)
+ if err != nil {
+ return math.MaxUint64, err
+ }
+ defer c.Close()
+ k, _, err := c.First()
+ if err != nil {
+ return math.MaxUint64, err
+ }
+ if len(k) == 0 {
+ return math.MaxUint64, nil
+ }
+ return binary.BigEndian.Uint64(k), nil
+}
+
// ReadBlock retrieves an entire block corresponding to the hash, assembling it
// back from the stored header and body. If either the header or body could not
// be retrieved nil is returned.
diff --git a/eth/backend.go b/eth/backend.go
index 6905c11668ef5ff0e9685781d318b62428f1b63f..675dbdb47cbffc8588930f8ac935845b6e9c86fa 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -54,6 +54,7 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/ethdb/remote/remotedbserver"
"github.com/ledgerwatch/erigon/log"
"github.com/ledgerwatch/erigon/node"
@@ -173,7 +174,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
}
}
- chainConfig, genesis, genesisErr := core.CommitGenesisBlock(chainKv, config.Genesis, config.StorageMode.History)
+ chainConfig, genesis, genesisErr := core.CommitGenesisBlock(chainKv, config.Genesis)
if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok {
return nil, genesisErr
}
@@ -213,7 +214,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
log.Info("Initialising Ethereum protocol", "network", config.NetworkID)
if err := chainKv.Update(context.Background(), func(tx ethdb.RwTx) error {
- if err := ethdb.SetStorageModeIfNotExist(tx, config.StorageMode); err != nil {
+ if err := prune.SetIfNotExist(tx, config.Prune); err != nil {
return err
}
@@ -221,19 +222,19 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
return err
}
- sm, err := ethdb.GetStorageModeFromDB(tx)
+ pm, err := prune.Get(tx)
if err != nil {
return err
}
- if config.StorageMode.Initialised {
+ if config.Prune.Initialised {
// If storage mode is not explicitly specified, we take whatever is in the database
- if !reflect.DeepEqual(sm, config.StorageMode) {
- return errors.New("mode is " + config.StorageMode.ToString() + " original mode is " + sm.ToString())
+ if !reflect.DeepEqual(pm, config.Prune) {
+ return errors.New("prune is " + config.Prune.ToString() + " original prune is " + pm.ToString())
}
} else {
- config.StorageMode = sm
+ config.Prune = pm
}
- log.Info("Effective", "storage mode", config.StorageMode)
+ log.Info("Effective", "prune", config.Prune.ToString())
return nil
}); err != nil {
diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go
index 4ef42e85f83498079f94fd4151005a38b8ef7202..e6b2697b672fa8b8751a4686381618b5a6644cff 100644
--- a/eth/ethconfig/config.go
+++ b/eth/ethconfig/config.go
@@ -29,6 +29,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/ledgerwatch/erigon/consensus/aura"
"github.com/ledgerwatch/erigon/consensus/aura/consensusconfig"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/consensus"
@@ -37,7 +38,6 @@ import (
"github.com/ledgerwatch/erigon/consensus/ethash"
"github.com/ledgerwatch/erigon/core"
"github.com/ledgerwatch/erigon/eth/gasprice"
- "github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/log"
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
@@ -69,8 +69,8 @@ var Defaults = Config{
DatasetsOnDisk: 2,
DatasetsLockMmap: false,
},
- NetworkID: 1,
- StorageMode: ethdb.DefaultStorageMode,
+ NetworkID: 1,
+ Prune: prune.DefaultMode,
Miner: params.MiningConfig{
GasFloor: 8000000,
GasCeil: 8000000,
@@ -133,8 +133,8 @@ type Config struct {
P2PEnabled bool
- StorageMode ethdb.StorageMode
- BatchSize datasize.ByteSize // Batch size for execution stage
+ Prune prune.Mode
+ BatchSize datasize.ByteSize // Batch size for execution stage
Snapshot Snapshot
diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go
index 7ab3bbc2cc0b91f2b2e06c43d5f94315c48c4045..7111ac822c2da49ecb4bd1eb749d23d2178e772c 100644
--- a/eth/ethconfig/gen_config.go
+++ b/eth/ethconfig/gen_config.go
@@ -7,7 +7,7 @@ import (
"github.com/ledgerwatch/erigon/consensus/ethash"
"github.com/ledgerwatch/erigon/core"
"github.com/ledgerwatch/erigon/eth/gasprice"
- "github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/params"
)
@@ -32,7 +32,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.NetworkID = c.NetworkID
enc.EthDiscoveryURLs = c.EthDiscoveryURLs
enc.Whitelist = c.Whitelist
- enc.StorageMode = c.StorageMode.ToString()
+ enc.StorageMode = c.Prune.ToString()
enc.Miner = c.Miner
enc.Ethash = c.Ethash
enc.TxPool = c.TxPool
@@ -50,6 +50,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
EthDiscoveryURLs []string
Whitelist map[uint64]common.Hash `toml:"-"`
Mode *string
+ PruneH *uint64
+ PruneR *uint64
+ PruneT *uint64
+ PruneC *uint64
+ Experiments *[]string
OnlyAnnounce *bool
SkipBcVersionCheck *bool `toml:"-"`
DatabaseHandles *int `toml:"-"`
@@ -78,11 +83,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
c.Whitelist = dec.Whitelist
}
if dec.Mode != nil {
- mode, err := ethdb.StorageModeFromString(*dec.Mode)
+ mode, err := prune.FromCli(*dec.Mode, *dec.PruneH, *dec.PruneR, *dec.PruneT, *dec.PruneC, *dec.Experiments)
if err != nil {
return err
}
- c.StorageMode = mode
+ c.Prune = mode
}
if dec.Miner != nil {
c.Miner = *dec.Miner
diff --git a/eth/stagedsync/default_stages.go b/eth/stagedsync/default_stages.go
index f881db0e2fc25ac40a9db18c18b35c228719ba9a..1fa1431077de33714366bb06439429e5f307b210 100644
--- a/eth/stagedsync/default_stages.go
+++ b/eth/stagedsync/default_stages.go
@@ -5,10 +5,11 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
)
func DefaultStages(ctx context.Context,
- sm ethdb.StorageMode,
+ sm prune.Mode,
headers HeadersCfg,
blockHashCfg BlockHashesCfg,
snapshotHeaders SnapshotHeadersCfg,
@@ -56,10 +57,9 @@ func DefaultStages(ctx context.Context,
},
},
{
- ID: stages.CreateHeadersSnapshot,
- Description: "Create headers snapshot",
- Disabled: true,
- DisabledDescription: "Enable by --snapshot.layout",
+ ID: stages.CreateHeadersSnapshot,
+ Description: "Create headers snapshot",
+ Disabled: true,
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnHeadersSnapshotGenerationStage(s, tx, snapshotHeaders, firstCycle, ctx)
},
@@ -84,10 +84,9 @@ func DefaultStages(ctx context.Context,
},
},
{
- ID: stages.CreateBodiesSnapshot,
- Description: "Create bodies snapshot",
- Disabled: true,
- DisabledDescription: "Enable by --snapshot.layout",
+ ID: stages.CreateBodiesSnapshot,
+ Description: "Create bodies snapshot",
+ Disabled: true,
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnBodiesSnapshotGenerationStage(s, tx, snapshotBodies, ctx)
},
@@ -127,8 +126,8 @@ func DefaultStages(ctx context.Context,
{
ID: stages.Translation,
Description: "Transpile marked EVM contracts to TEVM",
- Disabled: !sm.TEVM,
- DisabledDescription: "Enable by adding `e` to --storage-mode",
+ Disabled: !sm.Experiments.TEVM,
+ DisabledDescription: "Enable by adding `tevm` to --experiments",
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnTranspileStage(s, tx, 0, trans, ctx)
},
@@ -185,7 +184,6 @@ func DefaultStages(ctx context.Context,
ID: stages.CallTraces,
Description: "Generate call traces index",
DisabledDescription: "Work In Progress",
- Disabled: !sm.CallTraces,
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnCallTraces(s, tx, callTraces, ctx)
},
@@ -197,10 +195,8 @@ func DefaultStages(ctx context.Context,
},
},
{
- ID: stages.AccountHistoryIndex,
- Description: "Generate account history index",
- Disabled: !sm.History,
- DisabledDescription: "Enable by adding `h` to --storage-mode",
+ ID: stages.AccountHistoryIndex,
+ Description: "Generate account history index",
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnAccountHistoryIndex(s, tx, history, ctx)
},
@@ -212,10 +208,8 @@ func DefaultStages(ctx context.Context,
},
},
{
- ID: stages.StorageHistoryIndex,
- Description: "Generate storage history index",
- Disabled: !sm.History,
- DisabledDescription: "Enable by adding `h` to --storage-mode",
+ ID: stages.StorageHistoryIndex,
+ Description: "Generate storage history index",
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnStorageHistoryIndex(s, tx, history, ctx)
},
@@ -227,10 +221,8 @@ func DefaultStages(ctx context.Context,
},
},
{
- ID: stages.LogIndex,
- Description: "Generate receipt logs index",
- Disabled: !sm.Receipts,
- DisabledDescription: "Enable by adding `r` to --storage-mode",
+ ID: stages.LogIndex,
+ Description: "Generate receipt logs index",
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnLogIndex(s, tx, logIndex, ctx)
},
@@ -242,10 +234,8 @@ func DefaultStages(ctx context.Context,
},
},
{
- ID: stages.TxLookup,
- Description: "Generate tx lookup index",
- Disabled: !sm.TxIndex,
- DisabledDescription: "Enable by adding `t` to --storage-mode",
+ ID: stages.TxLookup,
+ Description: "Generate tx lookup index",
Forward: func(firstCycle bool, s *StageState, u Unwinder, tx ethdb.RwTx) error {
return SpawnTxLookup(s, tx, txLookup, ctx)
},
diff --git a/eth/stagedsync/stage.go b/eth/stagedsync/stage.go
index dd430c4aadf31d606bab59bf4b71762303d5e9ad..92bd50ccfebadddc6563bb5837b2b0574c832bed 100644
--- a/eth/stagedsync/stage.go
+++ b/eth/stagedsync/stage.go
@@ -82,10 +82,13 @@ func (u *UnwindState) Done(db ethdb.Putter) error {
}
type PruneState struct {
- ID stages.SyncStage
- PrunePoint uint64 // PrunePoint is the block to prune to.
- CurrentBlockNumber uint64
- state *Sync
+ ID stages.SyncStage
+ ForwardProgress uint64 // progress of stage forward move
+ PruneProgress uint64 // progress of stage prune move. after sync cycle it become equal to ForwardProgress by Done() method
+ state *Sync
}
-func (u *PruneState) LogPrefix() string { return u.state.LogPrefix() }
+func (s *PruneState) LogPrefix() string { return s.state.LogPrefix() }
+func (s *PruneState) Done(db ethdb.Putter) error {
+ return stages.SaveStagePruneProgress(db, s.ID, s.ForwardProgress)
+}
diff --git a/eth/stagedsync/stage_call_traces.go b/eth/stagedsync/stage_call_traces.go
index 4b0f3482a880bf3ff6633b0b3c0f5d04ab90273c..8dfd23b4ceae31ff6867906da9d2809bb0e5f0e7 100644
--- a/eth/stagedsync/stage_call_traces.go
+++ b/eth/stagedsync/stage_call_traces.go
@@ -8,6 +8,7 @@ import (
"math"
"math/big"
"runtime"
+ "sort"
"time"
"github.com/RoaringBitmap/roaring/roaring64"
@@ -20,23 +21,27 @@ import (
"github.com/ledgerwatch/erigon/core/vm/stack"
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/ethdb/bitmapdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
"github.com/ledgerwatch/erigon/params"
)
type CallTracesCfg struct {
db ethdb.RwKV
+ prune prune.Mode
ToBlock uint64 // not setting this params means no limit
tmpdir string
}
func StageCallTracesCfg(
db ethdb.RwKV,
+ prune prune.Mode,
toBlock uint64,
tmpdir string,
) CallTracesCfg {
return CallTracesCfg{
db: db,
+ prune: prune,
ToBlock: toBlock,
tmpdir: tmpdir,
}
@@ -103,7 +108,10 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin
var k, v []byte
prev := startBlock
- for k, v, err = traceCursor.Seek(dbutils.EncodeBlockNumber(startBlock)); k != nil && err == nil; k, v, err = traceCursor.Next() {
+ for k, v, err = traceCursor.Seek(dbutils.EncodeBlockNumber(startBlock)); k != nil; k, v, err = traceCursor.Next() {
+ if err != nil {
+ return err
+ }
blockNum := binary.BigEndian.Uint64(k)
if blockNum > endBlock {
break
@@ -158,9 +166,6 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin
}
}
}
- if err != nil {
- return fmt.Errorf("%s: failed to move cursor: %w", logPrefix, err)
- }
if err = flushBitmaps64(collectorFrom, froms); err != nil {
return err
}
@@ -170,7 +175,10 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin
// Clean up before loading call traces to reclaim space
var prunedMin uint64 = math.MaxUint64
var prunedMax uint64 = 0
- for k, _, err = traceCursor.First(); k != nil && err == nil; k, _, err = traceCursor.NextNoDup() {
+ for k, _, err = traceCursor.First(); k != nil; k, _, err = traceCursor.NextNoDup() {
+ if err != nil {
+ return err
+ }
blockNum := binary.BigEndian.Uint64(k)
if blockNum+params.FullImmutabilityThreshold <= endBlock {
break
@@ -194,9 +202,6 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin
prunedMax = blockNum
}
}
- if err != nil {
- return fmt.Errorf("%s: failed to move cleanup cursor: %w", logPrefix, err)
- }
if prunedMax != 0 && prunedMax > prunedMin+16 {
log.Info(fmt.Sprintf("[%s] Pruned call trace intermediate table", logPrefix), "from", prunedMin, "to", prunedMax)
}
@@ -267,10 +272,9 @@ func UnwindCallTraces(u *UnwindState, s *StageState, tx ethdb.RwTx, cfg CallTrac
}
defer tx.Rollback()
}
- quitCh := ctx.Done()
logPrefix := u.LogPrefix()
- if err := DoUnwindCallTraces(logPrefix, tx, s.BlockNumber, u.UnwindPoint, quitCh, cfg); err != nil {
+ if err := DoUnwindCallTraces(logPrefix, tx, s.BlockNumber, u.UnwindPoint, ctx, cfg); err != nil {
return fmt.Errorf("[%s] %w", logPrefix, err)
}
@@ -287,7 +291,7 @@ func UnwindCallTraces(u *UnwindState, s *StageState, tx ethdb.RwTx, cfg CallTrac
return nil
}
-func DoUnwindCallTraces(logPrefix string, db ethdb.RwTx, from, to uint64, quitCh <-chan struct{}, cfg CallTracesCfg) error {
+func DoUnwindCallTraces(logPrefix string, db ethdb.RwTx, from, to uint64, ctx context.Context, cfg CallTracesCfg) error {
froms := map[string]struct{}{}
tos := map[string]struct{}{}
@@ -298,10 +302,14 @@ func DoUnwindCallTraces(logPrefix string, db ethdb.RwTx, from, to uint64, quitCh
if err != nil {
return fmt.Errorf("%s: failed to create cursor for call traces: %w", logPrefix, err)
}
+ defer traceCursor.Close()
var k, v []byte
prev := to + 1
- for k, v, err = traceCursor.Seek(dbutils.EncodeBlockNumber(to + 1)); k != nil && err == nil; k, v, err = traceCursor.Next() {
+ for k, v, err = traceCursor.Seek(dbutils.EncodeBlockNumber(to + 1)); k != nil; k, v, err = traceCursor.Next() {
+ if err != nil {
+ return err
+ }
blockNum := binary.BigEndian.Uint64(k)
if blockNum >= from {
break
@@ -317,7 +325,6 @@ func DoUnwindCallTraces(logPrefix string, db ethdb.RwTx, from, to uint64, quitCh
tos[mapKey] = struct{}{}
}
select {
- default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
@@ -328,11 +335,11 @@ func DoUnwindCallTraces(logPrefix string, db ethdb.RwTx, from, to uint64, quitCh
"blk/second", speed,
"alloc", common.StorageSize(m.Alloc),
"sys", common.StorageSize(m.Sys))
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
}
}
- if err != nil {
- return fmt.Errorf("%s: failed to move cursor: %w", logPrefix, err)
- }
if err := truncateBitmaps64(db, dbutils.CallFromIndex, froms, to); err != nil {
return err
@@ -400,6 +407,8 @@ func (ct *CallTracer) CaptureAccountWrite(account common.Address) error {
}
func PruneCallTraces(s *PruneState, tx ethdb.RwTx, cfg CallTracesCfg, ctx context.Context) (err error) {
+ logPrefix := s.LogPrefix()
+
useExternalTx := tx != nil
if !useExternalTx {
tx, err = cfg.db.BeginRw(ctx)
@@ -409,6 +418,15 @@ func PruneCallTraces(s *PruneState, tx ethdb.RwTx, cfg CallTracesCfg, ctx contex
defer tx.Rollback()
}
+ if cfg.prune.CallTraces.Enabled() {
+ if err = pruneCallTraces(tx, logPrefix, cfg.tmpdir, cfg.prune.History.PruneTo(s.ForwardProgress), ctx); err != nil {
+ return err
+ }
+ }
+ if err := s.Done(tx); err != nil {
+ return err
+ }
+
if !useExternalTx {
if err = tx.Commit(); err != nil {
return err
@@ -416,3 +434,122 @@ func PruneCallTraces(s *PruneState, tx ethdb.RwTx, cfg CallTracesCfg, ctx contex
}
return nil
}
+
+func pruneCallTraces(tx ethdb.RwTx, logPrefix, tmpDir string, pruneTo uint64, ctx context.Context) error {
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+
+ froms := map[string]struct{}{}
+ tos := map[string]struct{}{}
+
+ {
+ traceCursor, err := tx.CursorDupSort(dbutils.CallTraceSet)
+ if err != nil {
+ return fmt.Errorf("%s: failed to create cursor for call traces: %w", logPrefix, err)
+ }
+ defer traceCursor.Close()
+
+ var k, v []byte
+ for k, v, err = traceCursor.First(); k != nil; k, v, err = traceCursor.Next() {
+ if err != nil {
+ return err
+ }
+ blockNum := binary.BigEndian.Uint64(k)
+ if blockNum >= pruneTo {
+ break
+ }
+ if len(v) != common.AddressLength+1 {
+ return fmt.Errorf("%s: wrong size of value in CallTraceSet: %x (size %d)", logPrefix, v, len(v))
+ }
+ mapKey := string(v[:common.AddressLength])
+ if v[common.AddressLength]&1 > 0 {
+ froms[mapKey] = struct{}{}
+ }
+ if v[common.AddressLength]&2 > 0 {
+ tos[mapKey] = struct{}{}
+ }
+ select {
+ case <-logEvery.C:
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ log.Info(fmt.Sprintf("[%s] Progress", logPrefix), "number", blockNum, "alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys))
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ }
+
+ }
+
+ {
+ sorted := make([]string, 0, len(froms))
+ for k := range froms {
+ sorted = append(sorted, k)
+ }
+ sort.Strings(sorted)
+ c, err := tx.RwCursor(dbutils.CallFromIndex)
+ if err != nil {
+ return err
+ }
+ defer c.Close()
+
+ for _, fromS := range sorted {
+ from := []byte(fromS)
+ for k, _, err := c.Seek(from); k != nil; k, _, err = c.Next() {
+ if err != nil {
+ return err
+ }
+ blockNum := binary.BigEndian.Uint64(k[common.AddressLength:])
+ if !bytes.HasPrefix(k, from) || blockNum >= pruneTo {
+ break
+ }
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", dbutils.CallFromIndex, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ if err = c.DeleteCurrent(); err != nil {
+ return fmt.Errorf("failed delete, block=%d: %w", blockNum, err)
+ }
+ }
+ }
+ }
+ {
+ sorted := make([]string, 0, len(tos))
+ for k := range tos {
+ sorted = append(sorted, k)
+ }
+ sort.Strings(sorted)
+ c, err := tx.RwCursor(dbutils.CallToIndex)
+ if err != nil {
+ return err
+ }
+ defer c.Close()
+
+ for _, toS := range sorted {
+ to := []byte(toS)
+ for k, _, err := c.Seek(to); k != nil; k, _, err = c.Next() {
+ if err != nil {
+ return err
+ }
+ blockNum := binary.BigEndian.Uint64(k[common.AddressLength:])
+ if !bytes.HasPrefix(k, to) || blockNum >= pruneTo {
+ break
+ }
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", dbutils.CallToIndex, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ if err = c.DeleteCurrent(); err != nil {
+ return fmt.Errorf("failed delete, block=%d: %w", blockNum, err)
+ }
+ }
+ }
+ }
+ return nil
+}
diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go
index 1ea3d919d337338ff3dcbe7ea05cf709d6e9a646..ab76158bb0786f6f91f24897be1b019ae40f383f 100644
--- a/eth/stagedsync/stage_execute.go
+++ b/eth/stagedsync/stage_execute.go
@@ -4,13 +4,13 @@ import (
"context"
"encoding/binary"
"fmt"
- "math"
"runtime"
"sort"
"time"
"github.com/c2h5oh/datasize"
"github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/changeset"
@@ -44,27 +44,21 @@ type HasChangeSetWriter interface {
type ChangeSetHook func(blockNum uint64, wr *state.ChangeSetWriter)
type ExecuteBlockCfg struct {
- db ethdb.RwKV
- batchSize datasize.ByteSize
- changeSetHook ChangeSetHook
- chainConfig *params.ChainConfig
- engine consensus.Engine
- vmConfig *vm.Config
- tmpdir string
- writeReceipts bool
- writeCallTraces bool
- writeTEVM bool
- pruningDistance uint64
- stateStream bool
- accumulator *shards.Accumulator
+ db ethdb.RwKV
+ batchSize datasize.ByteSize
+ prune prune.Mode
+ changeSetHook ChangeSetHook
+ chainConfig *params.ChainConfig
+ engine consensus.Engine
+ vmConfig *vm.Config
+ tmpdir string
+ stateStream bool
+ accumulator *shards.Accumulator
}
func StageExecuteBlocksCfg(
kv ethdb.RwKV,
- writeReceipts bool,
- writeCallTraces bool,
- writeTEVM bool,
- pruningDistance uint64,
+ prune prune.Mode,
batchSize datasize.ByteSize,
changeSetHook ChangeSetHook,
chainConfig *params.ChainConfig,
@@ -75,19 +69,16 @@ func StageExecuteBlocksCfg(
tmpdir string,
) ExecuteBlockCfg {
return ExecuteBlockCfg{
- db: kv,
- writeReceipts: writeReceipts,
- writeCallTraces: writeCallTraces,
- writeTEVM: writeTEVM,
- pruningDistance: pruningDistance,
- batchSize: batchSize,
- changeSetHook: changeSetHook,
- chainConfig: chainConfig,
- engine: engine,
- vmConfig: vmConfig,
- tmpdir: tmpdir,
- accumulator: accumulator,
- stateStream: stateStream,
+ db: kv,
+ prune: prune,
+ batchSize: batchSize,
+ changeSetHook: changeSetHook,
+ chainConfig: chainConfig,
+ engine: engine,
+ vmConfig: vmConfig,
+ tmpdir: tmpdir,
+ accumulator: accumulator,
+ stateStream: stateStream,
}
}
@@ -105,7 +96,10 @@ func executeBlock(
tx ethdb.RwTx,
batch ethdb.Database,
cfg ExecuteBlockCfg,
+ vmConfig vm.Config, // emit copy, because will modify it
writeChangesets bool,
+ writeReceipts bool,
+ writeCallTraces bool,
checkTEVM func(contractHash common.Hash) (bool, error),
initialCycle bool,
) error {
@@ -114,19 +108,16 @@ func executeBlock(
// where the magic happens
getHeader := func(hash common.Hash, number uint64) *types.Header { return rawdb.ReadHeader(tx, hash, number) }
- var callTracer *CallTracer
- if cfg.writeCallTraces {
- callTracer = NewCallTracer(checkTEVM)
- cfg.vmConfig.Debug = true
- cfg.vmConfig.Tracer = callTracer
- }
- receipts, err := core.ExecuteBlockEphemerally(cfg.chainConfig, cfg.vmConfig, getHeader, cfg.engine, block, stateReader, stateWriter, epochReader{tx: tx}, checkTEVM)
+ callTracer := NewCallTracer(checkTEVM)
+ vmConfig.Debug = true
+ vmConfig.Tracer = callTracer
+ receipts, err := core.ExecuteBlockEphemerally(cfg.chainConfig, &vmConfig, getHeader, cfg.engine, block, stateReader, stateWriter, epochReader{tx: tx}, checkTEVM)
if err != nil {
return err
}
- if cfg.writeReceipts {
+ if writeReceipts {
if err = rawdb.AppendReceipts(tx, blockNum, receipts); err != nil {
return err
}
@@ -138,7 +129,7 @@ func executeBlock(
}
}
- if cfg.writeCallTraces {
+ if writeCallTraces {
callTracer.tos[block.Coinbase()] = false
for _, uncle := range block.Uncles() {
callTracer.tos[uncle.Coinbase] = false
@@ -172,7 +163,7 @@ func executeBlock(
v[common.AddressLength] |= 2
}
// TEVM marking still untranslated contracts
- if cfg.vmConfig.EnableTEMV {
+ if vmConfig.EnableTEMV {
if created = callTracer.tos[addr]; created {
v[common.AddressLength] |= 4
}
@@ -284,19 +275,17 @@ Loop:
lastLogTx += uint64(block.Transactions().Len())
- writeChangesets := true
- if cfg.pruningDistance > 0 && to-blockNum > cfg.pruningDistance {
- writeChangesets = false
- }
-
var checkTEVMCode func(contractHash common.Hash) (bool, error)
if cfg.vmConfig.EnableTEMV {
checkTEVMCode = ethdb.GetCheckTEVM(tx)
}
- if err = executeBlock(block, tx, batch, cfg, writeChangesets, checkTEVMCode, initialCycle); err != nil {
- log.Error(fmt.Sprintf("[%s] Execution failed", logPrefix), "number", blockNum, "hash", block.Hash().String(), "error", err)
+ writeReceipts := blockNum > cfg.prune.Receipts.PruneTo(to)
+ writeChangeSets := blockNum > cfg.prune.History.PruneTo(to)
+ writeCallTraces := blockNum > cfg.prune.CallTraces.PruneTo(to)
+ if err = executeBlock(block, tx, batch, cfg, *cfg.vmConfig, writeChangeSets, writeReceipts, writeCallTraces, checkTEVMCode, initialCycle); err != nil {
+ log.Error(fmt.Sprintf("[%s] Execution failed", logPrefix), "block", blockNum, "hash", block.Hash().String(), "error", err)
u.UnwindTo(blockNum-1, block.Hash())
break Loop
}
@@ -344,15 +333,6 @@ Loop:
if err = batch.Commit(); err != nil {
return fmt.Errorf("%s: failed to write batch commit: %v", logPrefix, err)
}
- // Prune changesets if needed
- if cfg.pruningDistance > 0 {
- if err = pruneDupSortedBucket(tx, logPrefix, "account changesets", dbutils.AccountChangeSetBucket, to, cfg.pruningDistance, logEvery.C); err != nil {
- return err
- }
- if err = pruneDupSortedBucket(tx, logPrefix, "storage changesets", dbutils.StorageChangeSetBucket, to, cfg.pruningDistance, logEvery.C); err != nil {
- return err
- }
- }
if !useExternalTx {
if err = tx.Commit(); err != nil {
@@ -364,46 +344,31 @@ Loop:
return stoppedErr
}
-func pruneDupSortedBucket(tx ethdb.RwTx, logPrefix string, name string, tableName string, endBlock uint64, pruningDistance uint64, logChannel <-chan time.Time) error {
- changeSetCursor, err := tx.RwCursorDupSort(tableName)
+func pruneChangeSets(tx ethdb.RwTx, logPrefix string, table string, pruneTo uint64, logEvery *time.Ticker, ctx context.Context) error {
+ c, err := tx.RwCursorDupSort(table)
if err != nil {
- return fmt.Errorf("%s: failed to create cursor for pruning %s: %v", logPrefix, name, err)
+ return fmt.Errorf("failed to create cursor for pruning %w", err)
}
- defer changeSetCursor.Close()
-
- var prunedMin uint64 = math.MaxUint64
- var prunedMax uint64 = 0
- var k []byte
+ defer c.Close()
- for k, _, err = changeSetCursor.First(); k != nil && err == nil; k, _, err = changeSetCursor.NextNoDup() {
+ for k, _, err := c.First(); k != nil; k, _, err = c.NextNoDup() {
+ if err != nil {
+ return fmt.Errorf("failed to move %s cleanup cursor: %w", table, err)
+ }
blockNum := binary.BigEndian.Uint64(k)
- if endBlock-blockNum <= pruningDistance {
+ if blockNum >= pruneTo {
break
}
select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", table, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
default:
- case <-logChannel:
- var m runtime.MemStats
- runtime.ReadMemStats(&m)
- log.Info(fmt.Sprintf("[%s] Pruning", logPrefix), "table", tableName, "number", blockNum,
- "alloc", common.StorageSize(m.Alloc),
- "sys", common.StorageSize(m.Sys))
- }
- if err = changeSetCursor.DeleteCurrentDuplicates(); err != nil {
- return fmt.Errorf("%s: failed to remove %s for block %d: %v", logPrefix, name, blockNum, err)
}
- if blockNum < prunedMin {
- prunedMin = blockNum
+ if err = c.DeleteCurrentDuplicates(); err != nil {
+ return fmt.Errorf("failed to remove for block %d: %w", blockNum, err)
}
- if blockNum > prunedMax {
- prunedMax = blockNum
- }
- }
- if err != nil {
- return fmt.Errorf("%s: failed to move %s cleanup cursor: %w", logPrefix, tableName, err)
- }
- if prunedMax != 0 && prunedMax > prunedMin+16 {
- log.Info(fmt.Sprintf("[%s] Pruned", logPrefix), "table", tableName, "from", prunedMin, "to", prunedMax)
}
return nil
}
@@ -559,28 +524,24 @@ func unwindExecutionStage(u *UnwindState, s *StageState, tx ethdb.RwTx, quit <-c
return fmt.Errorf("[%s] %w", logPrefix, err)
}
- if cfg.writeReceipts {
- if err := rawdb.DeleteNewerReceipts(tx, u.UnwindPoint+1); err != nil {
- return fmt.Errorf("%s: walking receipts: %v", logPrefix, err)
- }
+ if err := rawdb.DeleteNewerReceipts(tx, u.UnwindPoint+1); err != nil {
+ return fmt.Errorf("%s: walking receipts: %v", logPrefix, err)
}
- if cfg.writeCallTraces {
- // Truncate CallTraceSet
- keyStart := dbutils.EncodeBlockNumber(u.UnwindPoint + 1)
- c, err := tx.RwCursorDupSort(dbutils.CallTraceSet)
+ // Truncate CallTraceSet
+ keyStart := dbutils.EncodeBlockNumber(u.UnwindPoint + 1)
+ c, err := tx.RwCursorDupSort(dbutils.CallTraceSet)
+ if err != nil {
+ return err
+ }
+ defer c.Close()
+ for k, _, err := c.Seek(keyStart); k != nil; k, _, err = c.NextNoDup() {
if err != nil {
return err
}
- defer c.Close()
- for k, _, err := c.Seek(keyStart); k != nil; k, _, err = c.NextNoDup() {
- if err != nil {
- return err
- }
- err = c.DeleteCurrentDuplicates()
- if err != nil {
- return err
- }
+ err = c.DeleteCurrentDuplicates()
+ if err != nil {
+ return err
}
}
@@ -604,7 +565,8 @@ func min(a, b uint64) uint64 {
return b
}
-func PruneExecutionStage(p *PruneState, tx ethdb.RwTx, cfg ExecuteBlockCfg, ctx context.Context, initialCycle bool) (err error) {
+func PruneExecutionStage(s *PruneState, tx ethdb.RwTx, cfg ExecuteBlockCfg, ctx context.Context, initialCycle bool) (err error) {
+ logPrefix := s.LogPrefix()
useExternalTx := tx != nil
if !useExternalTx {
tx, err = cfg.db.BeginRw(ctx)
@@ -614,7 +576,32 @@ func PruneExecutionStage(p *PruneState, tx ethdb.RwTx, cfg ExecuteBlockCfg, ctx
defer tx.Rollback()
}
- logPrefix := p.LogPrefix()
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+
+ if cfg.prune.History.Enabled() {
+ if err = pruneChangeSets(tx, logPrefix, dbutils.AccountChangeSetBucket, cfg.prune.History.PruneTo(s.ForwardProgress), logEvery, ctx); err != nil {
+ return err
+ }
+ if err = pruneChangeSets(tx, logPrefix, dbutils.StorageChangeSetBucket, cfg.prune.History.PruneTo(s.ForwardProgress), logEvery, ctx); err != nil {
+ return err
+ }
+ }
+
+ if cfg.prune.Receipts.Enabled() {
+ if err = pruneReceipts(tx, logPrefix, cfg.prune.Receipts.PruneTo(s.ForwardProgress), logEvery, ctx); err != nil {
+ return err
+ }
+ }
+ if cfg.prune.CallTraces.Enabled() {
+ if err = pruneCallTracesSet(tx, logPrefix, cfg.prune.CallTraces.PruneTo(s.ForwardProgress), logEvery, ctx); err != nil {
+ return err
+ }
+ }
+
+ if err = s.Done(tx); err != nil {
+ return err
+ }
if !useExternalTx {
if err = tx.Commit(); err != nil {
return fmt.Errorf("%s: failed to write db commit: %v", logPrefix, err)
@@ -622,3 +609,88 @@ func PruneExecutionStage(p *PruneState, tx ethdb.RwTx, cfg ExecuteBlockCfg, ctx
}
return nil
}
+
+func pruneReceipts(tx ethdb.RwTx, logPrefix string, pruneTo uint64, logEvery *time.Ticker, ctx context.Context) error {
+ c, err := tx.RwCursor(dbutils.Receipts)
+ if err != nil {
+ return fmt.Errorf("failed to create cursor for pruning %w", err)
+ }
+ defer c.Close()
+
+ for k, _, err := c.First(); k != nil; k, _, err = c.Next() {
+ if err != nil {
+ return err
+ }
+
+ blockNum := binary.BigEndian.Uint64(k)
+ if blockNum >= pruneTo {
+ break
+ }
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", dbutils.Receipts, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ if err = c.DeleteCurrent(); err != nil {
+ return fmt.Errorf("failed to remove for block %d: %w", blockNum, err)
+ }
+ }
+
+ c, err = tx.RwCursor(dbutils.Log)
+ if err != nil {
+ return fmt.Errorf("failed to create cursor for pruning %w", err)
+ }
+ defer c.Close()
+
+ for k, _, err := c.First(); k != nil; k, _, err = c.Next() {
+ if err != nil {
+ return err
+ }
+ blockNum := binary.BigEndian.Uint64(k)
+ if blockNum >= pruneTo {
+ break
+ }
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", dbutils.Log, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ if err = c.DeleteCurrent(); err != nil {
+ return fmt.Errorf("failed to remove for block %d: %w", blockNum, err)
+ }
+ }
+ return nil
+}
+
+func pruneCallTracesSet(tx ethdb.RwTx, logPrefix string, pruneTo uint64, logEvery *time.Ticker, ctx context.Context) error {
+ c, err := tx.RwCursorDupSort(dbutils.CallTraceSet)
+ if err != nil {
+ return fmt.Errorf("failed to create cursor for pruning %w", err)
+ }
+ defer c.Close()
+
+ for k, _, err := c.First(); k != nil; k, _, err = c.NextNoDup() {
+ if err != nil {
+ return err
+ }
+ blockNum := binary.BigEndian.Uint64(k)
+ if blockNum >= pruneTo {
+ break
+ }
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", dbutils.CallTraceSet, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ if err = c.DeleteCurrentDuplicates(); err != nil {
+ return fmt.Errorf("failed to remove for block %d: %w", blockNum, err)
+ }
+ }
+ return nil
+}
diff --git a/eth/stagedsync/stage_execute_test.go b/eth/stagedsync/stage_execute_test.go
index 23289b780a33855d1dbd33872faef901c17a927b..eb1efdcd8c4bf69ae5743ccd3dd72079a7e9055d 100644
--- a/eth/stagedsync/stage_execute_test.go
+++ b/eth/stagedsync/stage_execute_test.go
@@ -4,13 +4,16 @@ import (
"context"
"testing"
+ "github.com/ledgerwatch/erigon/common/changeset"
"github.com/ledgerwatch/erigon/common/dbutils"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
+ "github.com/stretchr/testify/assert"
)
func TestUnwindExecutionStagePlainStatic(t *testing.T) {
- ctx := context.Background()
+ ctx, assert := context.Background(), assert.New(t)
_, tx1 := kv.NewTestTx(t)
_, tx2 := kv.NewTestTx(t)
@@ -18,21 +21,18 @@ func TestUnwindExecutionStagePlainStatic(t *testing.T) {
generateBlocks(t, 1, 100, plainWriterGen(tx2), staticCodeStaticIncarnations)
err := stages.SaveStageProgress(tx2, stages.Execution, 100)
- if err != nil {
- t.Errorf("error while saving progress: %v", err)
- }
+ assert.NoError(err)
+
u := &UnwindState{ID: stages.Execution, UnwindPoint: 50}
s := &StageState{ID: stages.Execution, BlockNumber: 100}
- err = UnwindExecutionStage(u, s, tx2, ctx, ExecuteBlockCfg{writeReceipts: true}, false)
- if err != nil {
- t.Errorf("error while unwinding state: %v", err)
- }
+ err = UnwindExecutionStage(u, s, tx2, ctx, ExecuteBlockCfg{}, false)
+ assert.NoError(err)
compareCurrentState(t, tx1, tx2, dbutils.PlainStateBucket, dbutils.PlainContractCodeBucket, dbutils.ContractTEVMCodeBucket)
}
func TestUnwindExecutionStagePlainWithIncarnationChanges(t *testing.T) {
- ctx := context.Background()
+ ctx, assert := context.Background(), assert.New(t)
_, tx1 := kv.NewTestTx(t)
_, tx2 := kv.NewTestTx(t)
@@ -40,15 +40,12 @@ func TestUnwindExecutionStagePlainWithIncarnationChanges(t *testing.T) {
generateBlocks(t, 1, 100, plainWriterGen(tx2), changeCodeWithIncarnations)
err := stages.SaveStageProgress(tx2, stages.Execution, 100)
- if err != nil {
- t.Errorf("error while saving progress: %v", err)
- }
+ assert.NoError(err)
+
u := &UnwindState{ID: stages.Execution, UnwindPoint: 50}
s := &StageState{ID: stages.Execution, BlockNumber: 100}
- err = UnwindExecutionStage(u, s, tx2, ctx, ExecuteBlockCfg{writeReceipts: true}, false)
- if err != nil {
- t.Errorf("error while unwinding state: %v", err)
- }
+ err = UnwindExecutionStage(u, s, tx2, ctx, ExecuteBlockCfg{}, false)
+ assert.NoError(err)
compareCurrentState(t, tx1, tx2, dbutils.PlainStateBucket, dbutils.PlainContractCodeBucket)
}
@@ -68,10 +65,57 @@ func TestUnwindExecutionStagePlainWithCodeChanges(t *testing.T) {
}
u := &UnwindState{ID: stages.Execution, UnwindPoint: 50}
s := &StageState{ID: stages.Execution, BlockNumber: 100}
- err = UnwindExecutionStage(u, s, tx2, ctx, ExecuteBlockCfg{writeReceipts: true}, false)
+ err = UnwindExecutionStage(u, s, tx2, ctx, ExecuteBlockCfg{}, false)
if err != nil {
t.Errorf("error while unwinding state: %v", err)
}
compareCurrentState(t, tx1, tx2, dbutils.PlainStateBucket, dbutils.PlainContractCodeBucket)
}
+
+func TestPruneExecution(t *testing.T) {
+ ctx, assert := context.Background(), assert.New(t)
+ _, tx := kv.NewTestTx(t)
+
+ generateBlocks(t, 1, 50, plainWriterGen(tx), changeCodeIndepenentlyOfIncarnations)
+ err := stages.SaveStageProgress(tx, stages.Execution, 50)
+ assert.NoError(err)
+
+ available, err := changeset.AvailableFrom(tx)
+ assert.NoError(err)
+ assert.Equal(uint64(1), available)
+
+ s := &PruneState{ID: stages.Execution, ForwardProgress: 50}
+ // check pruning distance > than current stage progress
+ err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 100, Receipts: 101, CallTraces: 200}}, ctx, false)
+ assert.NoError(err)
+
+ available, err = changeset.AvailableFrom(tx)
+ assert.NoError(err)
+ assert.Equal(uint64(1), available)
+ available, err = changeset.AvailableStorageFrom(tx)
+ assert.NoError(err)
+ assert.Equal(uint64(1), available)
+
+ // pruning distance, first run
+ err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 5, Receipts: 15, CallTraces: 25}}, ctx, false)
+ assert.NoError(err)
+
+ available, err = changeset.AvailableFrom(tx)
+ assert.NoError(err)
+ assert.Equal(uint64(45), available)
+ available, err = changeset.AvailableStorageFrom(tx)
+ assert.NoError(err)
+ assert.Equal(uint64(45), available)
+
+ // pruning distance, second run
+ err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 5, Receipts: 15, CallTraces: 25}}, ctx, false)
+ assert.NoError(err)
+
+ available, err = changeset.AvailableFrom(tx)
+ assert.NoError(err)
+ assert.Equal(uint64(45), available)
+ available, err = changeset.AvailableStorageFrom(tx)
+ assert.NoError(err)
+ assert.Equal(uint64(45), available)
+}
diff --git a/eth/stagedsync/stage_indexes.go b/eth/stagedsync/stage_indexes.go
index 8cbb66a77b6fd857e5cbb209f75cd1d6094c1bfd..bc7d860a21a48b8da4f0dd0e58b07ecdd8afc1ee 100644
--- a/eth/stagedsync/stage_indexes.go
+++ b/eth/stagedsync/stage_indexes.go
@@ -18,19 +18,22 @@ import (
"github.com/ledgerwatch/erigon/common/etl"
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/ethdb/bitmapdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
)
type HistoryCfg struct {
db ethdb.RwKV
bufLimit datasize.ByteSize
+ prune prune.Mode
flushEvery time.Duration
tmpdir string
}
-func StageHistoryCfg(db ethdb.RwKV, tmpDir string) HistoryCfg {
+func StageHistoryCfg(db ethdb.RwKV, prune prune.Mode, tmpDir string) HistoryCfg {
return HistoryCfg{
db: db,
+ prune: prune,
bufLimit: bitmapsBufLimit,
flushEvery: bitmapsFlushEvery,
tmpdir: tmpDir,
@@ -49,26 +52,31 @@ func SpawnAccountHistoryIndex(s *StageState, tx ethdb.RwTx, cfg HistoryCfg, ctx
}
quitCh := ctx.Done()
- executionAt, err := s.ExecutionAt(tx)
+ endBlock, err := s.ExecutionAt(tx)
logPrefix := s.LogPrefix()
if err != nil {
return fmt.Errorf("%s: getting last executed block: %w", logPrefix, err)
}
- if executionAt <= s.BlockNumber {
+ if endBlock <= s.BlockNumber {
return nil
}
- var startChangeSetsLookupAt uint64
+ var startBlock uint64
if s.BlockNumber > 0 {
- startChangeSetsLookupAt = s.BlockNumber + 1
+ startBlock = s.BlockNumber + 1
+ }
+ stopChangeSetsLookupAt := endBlock + 1
+
+ pruneTo := cfg.prune.History.PruneTo(endBlock)
+ if startBlock < pruneTo {
+ startBlock = pruneTo
}
- stopChangeSetsLookupAt := executionAt + 1
- if err := promoteHistory(logPrefix, tx, dbutils.AccountChangeSetBucket, startChangeSetsLookupAt, stopChangeSetsLookupAt, cfg, quitCh); err != nil {
+ if err := promoteHistory(logPrefix, tx, dbutils.AccountChangeSetBucket, startBlock, stopChangeSetsLookupAt, cfg, quitCh); err != nil {
return fmt.Errorf("[%s] %w", logPrefix, err)
}
- if err := s.Update(tx, executionAt); err != nil {
+ if err := s.Update(tx, endBlock); err != nil {
return fmt.Errorf("[%s] %w", logPrefix, err)
}
@@ -114,7 +122,6 @@ func SpawnStorageHistoryIndex(s *StageState, tx ethdb.RwTx, cfg HistoryCfg, ctx
if err := s.Update(tx, executionAt); err != nil {
return fmt.Errorf("[%s] %w", logPrefix, err)
}
-
if !useExternalTx {
if err := tx.Commit(); err != nil {
return err
@@ -283,15 +290,14 @@ func unwindHistory(logPrefix string, db ethdb.RwTx, csBucket string, to uint64,
updates := map[string]struct{}{}
if err := changeset.Walk(db, csBucket, dbutils.EncodeBlockNumber(to), 0, func(blockN uint64, k, v []byte) (bool, error) {
- if err := common.Stopped(quitCh); err != nil {
- return false, err
- }
select {
- default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Progress", logPrefix), "number", blockN, "alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys))
+ case <-quitCh:
+ return false, common.ErrStopped
+ default:
}
k = dbutils.CompositeKeyWithoutIncarnation(k)
updates[string(k)] = struct{}{}
@@ -348,6 +354,11 @@ func truncateBitmaps64(tx ethdb.RwTx, bucket string, inMem map[string]struct{},
}
func PruneAccountHistoryIndex(s *PruneState, tx ethdb.RwTx, cfg HistoryCfg, ctx context.Context) (err error) {
+ if !cfg.prune.History.Enabled() {
+ return nil
+ }
+ logPrefix := s.LogPrefix()
+
useExternalTx := tx != nil
if !useExternalTx {
tx, err = cfg.db.BeginRw(ctx)
@@ -357,6 +368,14 @@ func PruneAccountHistoryIndex(s *PruneState, tx ethdb.RwTx, cfg HistoryCfg, ctx
defer tx.Rollback()
}
+ pruneTo := cfg.prune.History.PruneTo(s.ForwardProgress)
+ if err = pruneHistoryIndex(tx, dbutils.AccountChangeSetBucket, logPrefix, cfg.tmpdir, pruneTo, ctx); err != nil {
+ return err
+ }
+ if err = s.Done(tx); err != nil {
+ return err
+ }
+
if !useExternalTx {
if err = tx.Commit(); err != nil {
return err
@@ -366,6 +385,11 @@ func PruneAccountHistoryIndex(s *PruneState, tx ethdb.RwTx, cfg HistoryCfg, ctx
}
func PruneStorageHistoryIndex(s *PruneState, tx ethdb.RwTx, cfg HistoryCfg, ctx context.Context) (err error) {
+ if !cfg.prune.History.Enabled() {
+ return nil
+ }
+ logPrefix := s.LogPrefix()
+
useExternalTx := tx != nil
if !useExternalTx {
tx, err = cfg.db.BeginRw(ctx)
@@ -374,6 +398,13 @@ func PruneStorageHistoryIndex(s *PruneState, tx ethdb.RwTx, cfg HistoryCfg, ctx
}
defer tx.Rollback()
}
+ pruneTo := cfg.prune.History.PruneTo(s.ForwardProgress)
+ if err = pruneHistoryIndex(tx, dbutils.StorageChangeSetBucket, logPrefix, cfg.tmpdir, pruneTo, ctx); err != nil {
+ return err
+ }
+ if err = s.Done(tx); err != nil {
+ return err
+ }
if !useExternalTx {
if err = tx.Commit(); err != nil {
@@ -382,3 +413,59 @@ func PruneStorageHistoryIndex(s *PruneState, tx ethdb.RwTx, cfg HistoryCfg, ctx
}
return nil
}
+
+func pruneHistoryIndex(tx ethdb.RwTx, csTable, logPrefix, tmpDir string, pruneTo uint64, ctx context.Context) error {
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+
+ collector := etl.NewCollector(tmpDir, etl.NewOldestEntryBuffer(etl.BufferOptimalSize))
+ defer collector.Close(logPrefix)
+
+ if err := changeset.Walk(tx, csTable, nil, 0, func(blockNum uint64, k, _ []byte) (bool, error) {
+ if blockNum >= pruneTo {
+ return false, nil
+ }
+ if err := collector.Collect(k, nil); err != nil {
+ return false, err
+ }
+ return true, nil
+ }); err != nil {
+ return err
+ }
+
+ c, err := tx.RwCursor(changeset.Mapper[csTable].IndexBucket)
+ if err != nil {
+ return fmt.Errorf("failed to create cursor for pruning %w", err)
+ }
+ defer c.Close()
+ prefixLen := common.AddressLength
+ if csTable == dbutils.StorageChangeSetBucket {
+ prefixLen = common.HashLength
+ }
+ if err := collector.Load(logPrefix, tx, "", func(addr, _ []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error {
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", changeset.Mapper[csTable].IndexBucket, "key", fmt.Sprintf("%x", addr))
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ for k, _, err := c.Seek(addr); k != nil; k, _, err = c.Next() {
+ if err != nil {
+ return err
+ }
+ blockNum := binary.BigEndian.Uint64(k[prefixLen:])
+ if !bytes.HasPrefix(k, addr) || blockNum >= pruneTo {
+ break
+ }
+ if err = c.DeleteCurrent(); err != nil {
+ return fmt.Errorf("failed to remove for block %d: %w", blockNum, err)
+ }
+ }
+ return nil
+ }, etl.TransformArgs{}); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/eth/stagedsync/stage_indexes_test.go b/eth/stagedsync/stage_indexes_test.go
index e23e5ee1fb9feac1de3e10b205e1fa3bc33ed841..f0a46cdf54917adc123d05d9df79ddf730e22858 100644
--- a/eth/stagedsync/stage_indexes_test.go
+++ b/eth/stagedsync/stage_indexes_test.go
@@ -4,35 +4,32 @@ import (
"context"
"encoding/binary"
"fmt"
- "os"
"reflect"
"sort"
"strconv"
"testing"
"time"
- "github.com/ledgerwatch/erigon/common/changeset"
- "github.com/ledgerwatch/erigon/common/math"
- "github.com/ledgerwatch/erigon/ethdb/bitmapdb"
- kv2 "github.com/ledgerwatch/erigon/ethdb/kv"
-
"github.com/ledgerwatch/erigon/common"
+ "github.com/ledgerwatch/erigon/common/changeset"
"github.com/ledgerwatch/erigon/common/dbutils"
+ "github.com/ledgerwatch/erigon/common/math"
"github.com/ledgerwatch/erigon/crypto"
"github.com/ledgerwatch/erigon/ethdb"
- "github.com/ledgerwatch/erigon/log"
+ "github.com/ledgerwatch/erigon/ethdb/bitmapdb"
+ kv2 "github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestIndexGenerator_GenerateIndex_SimpleCase(t *testing.T) {
- log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
kv := kv2.NewTestKV(t)
- cfg := StageHistoryCfg(kv, t.TempDir())
+ cfg := StageHistoryCfg(kv, prune.DefaultMode, t.TempDir())
test := func(blocksNum int, csBucket string) func(t *testing.T) {
return func(t *testing.T) {
tx, err := kv.BeginRw(context.Background())
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
defer tx.Rollback()
csInfo, ok := changeset.Mapper[csBucket]
@@ -42,19 +39,16 @@ func TestIndexGenerator_GenerateIndex_SimpleCase(t *testing.T) {
addrs, expecedIndexes := generateTestData(t, tx, csBucket, blocksNum)
cfgCopy := cfg
cfgCopy.bufLimit = 10
- cfgCopy.flushEvery = time.Millisecond
+ cfgCopy.flushEvery = time.Microsecond
err = promoteHistory("logPrefix", tx, csBucket, 0, uint64(blocksNum/2), cfgCopy, nil)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
err = promoteHistory("logPrefix", tx, csBucket, uint64(blocksNum/2), uint64(blocksNum), cfgCopy, nil)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
checkIndex(t, tx, csInfo.IndexBucket, addrs[0], expecedIndexes[string(addrs[0])])
checkIndex(t, tx, csInfo.IndexBucket, addrs[1], expecedIndexes[string(addrs[1])])
checkIndex(t, tx, csInfo.IndexBucket, addrs[2], expecedIndexes[string(addrs[2])])
+
}
}
@@ -64,17 +58,15 @@ func TestIndexGenerator_GenerateIndex_SimpleCase(t *testing.T) {
}
func TestIndexGenerator_Truncate(t *testing.T) {
- log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
buckets := []string{dbutils.AccountChangeSetBucket, dbutils.StorageChangeSetBucket}
+ tmpDir, ctx := t.TempDir(), context.Background()
kv := kv2.NewTestKV(t)
- cfg := StageHistoryCfg(kv, t.TempDir())
+ cfg := StageHistoryCfg(kv, prune.DefaultMode, t.TempDir())
for i := range buckets {
csbucket := buckets[i]
tx, err := kv.BeginRw(context.Background())
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
defer tx.Rollback()
hashes, expected := generateTestData(t, tx, csbucket, 2100)
@@ -82,11 +74,9 @@ func TestIndexGenerator_Truncate(t *testing.T) {
indexBucket := mp.IndexBucket
cfgCopy := cfg
cfgCopy.bufLimit = 10
- cfgCopy.flushEvery = time.Millisecond
+ cfgCopy.flushEvery = time.Microsecond
err = promoteHistory("logPrefix", tx, csbucket, 0, uint64(2100), cfgCopy, nil)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
reduceSlice := func(arr []uint64, timestamtTo uint64) []uint64 {
pos := sort.Search(len(arr), func(i int) bool {
@@ -101,9 +91,7 @@ func TestIndexGenerator_Truncate(t *testing.T) {
expected[string(hashes[2])] = reduceSlice(expected[string(hashes[2])], 2050)
err = unwindHistory("logPrefix", tx, csbucket, 2050, cfg, nil)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
checkIndex(t, tx, indexBucket, hashes[0], expected[string(hashes[0])])
checkIndex(t, tx, indexBucket, hashes[1], expected[string(hashes[1])])
@@ -116,9 +104,7 @@ func TestIndexGenerator_Truncate(t *testing.T) {
expected[string(hashes[2])] = reduceSlice(expected[string(hashes[2])], 2000)
err = unwindHistory("logPrefix", tx, csbucket, 2000, cfg, nil)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
checkIndex(t, tx, indexBucket, hashes[0], expected[string(hashes[0])])
checkIndex(t, tx, indexBucket, hashes[1], expected[string(hashes[1])])
@@ -127,9 +113,7 @@ func TestIndexGenerator_Truncate(t *testing.T) {
//t.Run("truncate to 1999 "+csbucket, func(t *testing.T) {
err = unwindHistory("logPrefix", tx, csbucket, 1999, cfg, nil)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
expected[string(hashes[0])] = reduceSlice(expected[string(hashes[0])], 1999)
expected[string(hashes[1])] = reduceSlice(expected[string(hashes[1])], 1999)
expected[string(hashes[2])] = reduceSlice(expected[string(hashes[2])], 1999)
@@ -138,16 +122,12 @@ func TestIndexGenerator_Truncate(t *testing.T) {
checkIndex(t, tx, indexBucket, hashes[1], expected[string(hashes[1])])
checkIndex(t, tx, indexBucket, hashes[2], expected[string(hashes[2])])
bm, err := bitmapdb.Get64(tx, indexBucket, hashes[0], 1999, math.MaxUint32)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
if bm.GetCardinality() > 0 && bm.Maximum() > 1999 {
t.Fatal(bm.Maximum())
}
bm, err = bitmapdb.Get64(tx, indexBucket, hashes[1], 1999, math.MaxUint32)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
if bm.GetCardinality() > 0 && bm.Maximum() > 1999 {
t.Fatal()
}
@@ -163,16 +143,12 @@ func TestIndexGenerator_Truncate(t *testing.T) {
t.Fatal(err)
}
bm, err = bitmapdb.Get64(tx, indexBucket, hashes[0], 999, math.MaxUint32)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
if bm.GetCardinality() > 0 && bm.Maximum() > 999 {
t.Fatal()
}
bm, err = bitmapdb.Get64(tx, indexBucket, hashes[1], 999, math.MaxUint32)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
if bm.GetCardinality() > 0 && bm.Maximum() > 999 {
t.Fatal()
}
@@ -180,12 +156,37 @@ func TestIndexGenerator_Truncate(t *testing.T) {
checkIndex(t, tx, indexBucket, hashes[0], expected[string(hashes[0])])
checkIndex(t, tx, indexBucket, hashes[1], expected[string(hashes[1])])
checkIndex(t, tx, indexBucket, hashes[2], expected[string(hashes[2])])
+
//})
+ err = pruneHistoryIndex(tx, csbucket, "", tmpDir, 128, ctx)
+ assert.NoError(t, err)
+ expectNoHistoryBefore(t, tx, csbucket, 128)
+
+ // double prune is safe
+ err = pruneHistoryIndex(tx, csbucket, "", tmpDir, 128, ctx)
+ assert.NoError(t, err)
+ expectNoHistoryBefore(t, tx, csbucket, 128)
tx.Rollback()
}
}
-func generateTestData(t *testing.T, db ethdb.RwTx, csBucket string, numOfBlocks int) ([][]byte, map[string][]uint64) { //nolint
+func expectNoHistoryBefore(t *testing.T, tx ethdb.Tx, csbucket string, prunedTo uint64) {
+ prefixLen := common.AddressLength
+ if csbucket == dbutils.StorageChangeSetBucket {
+ prefixLen = common.HashLength
+ }
+ afterPrune := 0
+ err := tx.ForEach(changeset.Mapper[csbucket].IndexBucket, nil, func(k, _ []byte) error {
+ n := binary.BigEndian.Uint64(k[prefixLen:])
+ require.True(t, n >= prunedTo)
+ afterPrune++
+ return nil
+ })
+ require.True(t, afterPrune > 0)
+ assert.NoError(t, err)
+}
+
+func generateTestData(t *testing.T, tx ethdb.RwTx, csBucket string, numOfBlocks int) ([][]byte, map[string][]uint64) { //nolint
csInfo, ok := changeset.Mapper[csBucket]
if !ok {
t.Fatal("incorrect cs bucket")
@@ -195,14 +196,10 @@ func generateTestData(t *testing.T, db ethdb.RwTx, csBucket string, numOfBlocks
isPlain = true
}
addrs, err := generateAddrs(3, isPlain)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
if dbutils.StorageChangeSetBucket == csBucket {
keys, innerErr := generateAddrs(3, false)
- if innerErr != nil {
- t.Fatal(innerErr)
- }
+ require.NoError(t, innerErr)
defaultIncarnation := make([]byte, 8)
binary.BigEndian.PutUint64(defaultIncarnation, uint64(1))
@@ -219,32 +216,24 @@ func generateTestData(t *testing.T, db ethdb.RwTx, csBucket string, numOfBlocks
for i := 0; i < numOfBlocks; i++ {
cs := csInfo.New()
err = cs.Add(addrs[0], []byte(strconv.Itoa(i)))
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
res = append(res, uint64(i))
if i%2 == 0 {
err = cs.Add(addrs[1], []byte(strconv.Itoa(i)))
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
res2 = append(res2, uint64(i))
}
if i%3 == 0 {
err = cs.Add(addrs[2], []byte(strconv.Itoa(i)))
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
res3 = append(res3, uint64(i))
}
err = csInfo.Encode(uint64(i), cs, func(k, v []byte) error {
- return db.Put(csBucket, k, v)
+ return tx.Put(csBucket, k, v)
})
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
}
return addrs, map[string][]uint64{
diff --git a/eth/stagedsync/stage_interhashes.go b/eth/stagedsync/stage_interhashes.go
index 896ac6654eca40d852c62a04b794d7ee23601a50..196a4310a622c641f61735647aa296a10e8ad160 100644
--- a/eth/stagedsync/stage_interhashes.go
+++ b/eth/stagedsync/stage_interhashes.go
@@ -7,7 +7,6 @@ import (
"math/bits"
"os"
"sort"
- "time"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/changeset"
@@ -129,7 +128,6 @@ func RegenerateIntermediateHashes(logPrefix string, db ethdb.RwTx, cfg TrieCfg,
if err := loader.Reset(trie.NewRetainList(0), accTrieCollectorFunc, stTrieCollectorFunc, false); err != nil {
return trie.EmptyRoot, err
}
- calcStart := time.Now()
hash, err := loader.CalcTrieRoot(db, []byte{}, quit)
if err != nil {
return trie.EmptyRoot, err
@@ -138,8 +136,7 @@ func RegenerateIntermediateHashes(logPrefix string, db ethdb.RwTx, cfg TrieCfg,
if cfg.checkRoot && hash != expectedRootHash {
return hash, nil
}
- log.Info(fmt.Sprintf("[%s] Trie root", logPrefix), "hash", hash.Hex(),
- "in", time.Since(calcStart))
+ log.Info(fmt.Sprintf("[%s] Trie root", logPrefix), "hash", hash.Hex())
if err := accTrieCollector.Load(logPrefix, db, dbutils.TrieOfAccountsBucket, etl.IdentityLoadFunc, etl.TransformArgs{Quit: quit}); err != nil {
return trie.EmptyRoot, err
@@ -365,7 +362,6 @@ func incrementIntermediateHashes(logPrefix string, s *StageState, db ethdb.RwTx,
if err := loader.Reset(rl, accTrieCollectorFunc, stTrieCollectorFunc, false); err != nil {
return trie.EmptyRoot, err
}
- calcStart := time.Now()
hash, err := loader.CalcTrieRoot(db, []byte{}, quit)
if err != nil {
return trie.EmptyRoot, err
@@ -375,8 +371,7 @@ func incrementIntermediateHashes(logPrefix string, s *StageState, db ethdb.RwTx,
return hash, nil
}
log.Info(fmt.Sprintf("[%s] Trie root", logPrefix),
- " hash", hash.Hex(),
- "in", time.Since(calcStart))
+ " hash", hash.Hex())
if err := accTrieCollector.Load(logPrefix, db, dbutils.TrieOfAccountsBucket, etl.IdentityLoadFunc, etl.TransformArgs{Quit: quit}); err != nil {
return trie.EmptyRoot, err
@@ -454,7 +449,6 @@ func unwindIntermediateHashesStageImpl(logPrefix string, u *UnwindState, s *Stag
if err := loader.Reset(rl, accTrieCollectorFunc, stTrieCollectorFunc, false); err != nil {
return err
}
- calcStart := time.Now()
hash, err := loader.CalcTrieRoot(db, []byte{}, quit)
if err != nil {
return err
@@ -462,7 +456,7 @@ func unwindIntermediateHashesStageImpl(logPrefix string, u *UnwindState, s *Stag
if hash != expectedRootHash {
return fmt.Errorf("%s: wrong trie root: %x, expected (from header): %x", logPrefix, hash, expectedRootHash)
}
- log.Info(fmt.Sprintf("[%s] Trie root", logPrefix), "hash", hash.Hex(), "in", time.Since(calcStart))
+ log.Info(fmt.Sprintf("[%s] Trie root", logPrefix), "hash", hash.Hex())
if err := accTrieCollector.Load(logPrefix, db, dbutils.TrieOfAccountsBucket, etl.IdentityLoadFunc, etl.TransformArgs{Quit: quit}); err != nil {
return err
}
@@ -557,6 +551,7 @@ func PruneIntermediateHashesStage(s *PruneState, tx ethdb.RwTx, cfg TrieCfg, ctx
}
defer tx.Rollback()
}
+ s.Done(tx)
if !useExternalTx {
if err = tx.Commit(); err != nil {
diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go
index b05e20727d602e884d805d831c8e6122d6f91cda..8dae6287e28557f440f0d82fef5a4b4c9ba0a8bb 100644
--- a/eth/stagedsync/stage_log_index.go
+++ b/eth/stagedsync/stage_log_index.go
@@ -18,6 +18,7 @@ import (
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/ethdb/bitmapdb"
"github.com/ledgerwatch/erigon/ethdb/cbor"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
)
@@ -29,13 +30,15 @@ const (
type LogIndexCfg struct {
tmpdir string
db ethdb.RwKV
+ prune prune.Mode
bufLimit datasize.ByteSize
flushEvery time.Duration
}
-func StageLogIndexCfg(db ethdb.RwKV, tmpDir string) LogIndexCfg {
+func StageLogIndexCfg(db ethdb.RwKV, prune prune.Mode, tmpDir string) LogIndexCfg {
return LogIndexCfg{
db: db,
+ prune: prune,
bufLimit: bitmapsBufLimit,
flushEvery: bitmapsFlushEvery,
tmpdir: tmpDir,
@@ -62,20 +65,24 @@ func SpawnLogIndex(s *StageState, tx ethdb.RwTx, cfg LogIndexCfg, ctx context.Co
return nil
}
- start := s.BlockNumber
- if start > 0 {
- start++
+ startBlock := s.BlockNumber
+ pruneTo := cfg.prune.Receipts.PruneTo(endBlock)
+ if startBlock < pruneTo {
+ startBlock = pruneTo
+ }
+ if startBlock > 0 {
+ startBlock++
}
- if err := promoteLogIndex(logPrefix, tx, start, cfg, ctx); err != nil {
+ if err = promoteLogIndex(logPrefix, tx, startBlock, cfg, ctx); err != nil {
return err
}
-
- if err := s.Update(tx, endBlock); err != nil {
+ if err = s.Update(tx, endBlock); err != nil {
return err
}
+
if !useExternalTx {
- if err := tx.Commit(); err != nil {
+ if err = tx.Commit(); err != nil {
return err
}
}
@@ -247,6 +254,7 @@ func unwindLogIndex(logPrefix string, db ethdb.RwTx, to uint64, cfg LogIndexCfg,
topics := map[string]struct{}{}
addrs := map[string]struct{}{}
+ reader := bytes.NewReader(nil)
c, err := db.Cursor(dbutils.Log)
if err != nil {
return err
@@ -261,7 +269,8 @@ func unwindLogIndex(logPrefix string, db ethdb.RwTx, to uint64, cfg LogIndexCfg,
return err
}
var logs types.Logs
- if err := cbor.Unmarshal(&logs, bytes.NewReader(v)); err != nil {
+ reader.Reset(v)
+ if err := cbor.Unmarshal(&logs, reader); err != nil {
return fmt.Errorf("%s: receipt unmarshal failed: %w, block=%d", logPrefix, err, binary.BigEndian.Uint64(k))
}
@@ -323,7 +332,50 @@ func truncateBitmaps(tx ethdb.RwTx, bucket string, inMem map[string]struct{}, to
return nil
}
+func pruneOldLogChunks(tx ethdb.RwTx, bucket string, inMem map[string]struct{}, pruneTo uint64, logPrefix string, ctx context.Context) error {
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+ keys := make([]string, 0, len(inMem))
+ for k := range inMem {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ c, err := tx.RwCursor(bucket)
+ if err != nil {
+ return err
+ }
+ defer c.Close()
+ for _, kS := range keys {
+ seek := []byte(kS)
+ for k, _, err := c.Seek(seek); k != nil; k, _, err = c.Next() {
+ if err != nil {
+ return err
+ }
+ blockNum := uint64(binary.BigEndian.Uint32(k[len(seek):]))
+ if !bytes.HasPrefix(k, seek) || blockNum >= pruneTo {
+ break
+ }
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", dbutils.AccountsHistoryBucket, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+ if err = c.DeleteCurrent(); err != nil {
+ return fmt.Errorf("failed delete, block=%d: %w", blockNum, err)
+ }
+ }
+ }
+ return nil
+}
+
func PruneLogIndex(s *PruneState, tx ethdb.RwTx, cfg LogIndexCfg, ctx context.Context) (err error) {
+ if !cfg.prune.History.Enabled() {
+ return nil
+ }
+ logPrefix := s.LogPrefix()
+
useExternalTx := tx != nil
if !useExternalTx {
tx, err = cfg.db.BeginRw(ctx)
@@ -333,6 +385,14 @@ func PruneLogIndex(s *PruneState, tx ethdb.RwTx, cfg LogIndexCfg, ctx context.Co
defer tx.Rollback()
}
+ pruneTo := cfg.prune.History.PruneTo(s.ForwardProgress)
+ if err = pruneLogIndex(logPrefix, tx, cfg.tmpdir, pruneTo, ctx); err != nil {
+ return err
+ }
+ if err = s.Done(tx); err != nil {
+ return err
+ }
+
if !useExternalTx {
if err = tx.Commit(); err != nil {
return err
@@ -340,3 +400,58 @@ func PruneLogIndex(s *PruneState, tx ethdb.RwTx, cfg LogIndexCfg, ctx context.Co
}
return nil
}
+
+func pruneLogIndex(logPrefix string, tx ethdb.RwTx, tmpDir string, pruneTo uint64, ctx context.Context) error {
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+
+ topics := map[string]struct{}{}
+ addrs := map[string]struct{}{}
+
+ reader := bytes.NewReader(nil)
+ {
+ c, err := tx.Cursor(dbutils.Log)
+ if err != nil {
+ return err
+ }
+ defer c.Close()
+
+ for k, v, err := c.First(); k != nil; k, v, err = c.Next() {
+ if err != nil {
+ return err
+ }
+ blockNum := binary.BigEndian.Uint64(k)
+ if blockNum >= pruneTo {
+ break
+ }
+ select {
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Mode", logPrefix), "table", dbutils.Log, "block", blockNum)
+ case <-ctx.Done():
+ return common.ErrStopped
+ default:
+ }
+
+ var logs types.Logs
+ reader.Reset(v)
+ if err := cbor.Unmarshal(&logs, reader); err != nil {
+ return fmt.Errorf("%s: receipt unmarshal failed: %w, block=%d", logPrefix, err, binary.BigEndian.Uint64(k))
+ }
+
+ for _, l := range logs {
+ for _, topic := range l.Topics {
+ topics[string(topic.Bytes())] = struct{}{}
+ }
+ addrs[string(l.Address.Bytes())] = struct{}{}
+ }
+ }
+ }
+
+ if err := pruneOldLogChunks(tx, dbutils.LogTopicIndex, topics, pruneTo, logPrefix, ctx); err != nil {
+ return err
+ }
+ if err := pruneOldLogChunks(tx, dbutils.LogAddressIndex, addrs, pruneTo, logPrefix, ctx); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/eth/stagedsync/stage_log_index_test.go b/eth/stagedsync/stage_log_index_test.go
index 3dc1ca64ec28e985c81a88b9120f72900b838de7..c918b0479d06a5a564c399642f3ad7e86210e457 100644
--- a/eth/stagedsync/stage_log_index_test.go
+++ b/eth/stagedsync/stage_log_index_test.go
@@ -2,6 +2,7 @@ package stagedsync
import (
"context"
+ "encoding/binary"
"testing"
"time"
@@ -9,104 +10,148 @@ import (
"github.com/ledgerwatch/erigon/common/dbutils"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/core/types"
+ "github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/ethdb/bitmapdb"
"github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/stretchr/testify/require"
)
+func genReceipts(t *testing.T, tx ethdb.RwTx, blocks uint64) (map[common.Address]uint64, map[common.Hash]uint64) {
+ addrs := []common.Address{{1}, {2}, {3}}
+ topics := []common.Hash{{1}, {2}, {3}}
+
+ expectAddrs := map[common.Address]uint64{}
+ expectTopics := map[common.Hash]uint64{}
+ for i := range addrs {
+ expectAddrs[addrs[i]] = 0
+ }
+ for i := range topics {
+ expectTopics[topics[i]] = 0
+ }
+
+ var receipts types.Receipts
+ for i := uint64(0); i < blocks; i++ {
+ switch i % 3 {
+ case 0:
+ a, t1, t2 := addrs[i%3], topics[i%3], topics[(i+1)%3]
+ receipts = types.Receipts{{
+ Logs: []*types.Log{
+ {
+ Address: a,
+ Topics: []common.Hash{t1, t2},
+ },
+ {
+ Address: a,
+ Topics: []common.Hash{t2},
+ },
+ {
+ Address: a,
+ Topics: []common.Hash{},
+ },
+ },
+ }}
+ expectAddrs[a]++
+ expectTopics[t1]++
+ expectTopics[t2]++
+
+ case 1:
+ a1, a2, t1, t2 := addrs[i%3], addrs[(i+1)%3], topics[i%3], topics[(i+1)%3]
+ receipts = types.Receipts{{
+ Logs: []*types.Log{
+ {
+ Address: a1,
+ Topics: []common.Hash{t1, t2, t1, t2},
+ },
+ },
+ }, {
+ Logs: []*types.Log{
+ {
+ Address: a2,
+ Topics: []common.Hash{t1, t2, t1, t2},
+ },
+ {
+ Address: a1,
+ Topics: []common.Hash{t1},
+ },
+ },
+ }}
+ expectAddrs[a1]++
+ expectAddrs[a2]++
+ expectTopics[t1]++
+ expectTopics[t2]++
+ case 2:
+ receipts = types.Receipts{{}, {}, {}}
+ }
+ err := rawdb.AppendReceipts(tx, i, receipts)
+ require.NoError(t, err)
+ }
+ return expectAddrs, expectTopics
+}
+
func TestLogIndex(t *testing.T) {
- require, ctx := require.New(t), context.Background()
- db, tx := kv.NewTestTx(t)
+ require, tmpDir, ctx := require.New(t), t.TempDir(), context.Background()
+ _, tx := kv.NewTestTx(t)
- addr1, addr2 := common.HexToAddress("0x0"), common.HexToAddress("0x376c47978271565f56DEB45495afa69E59c16Ab2")
- topic1, topic2 := common.HexToHash("0x0"), common.HexToHash("0x1234")
- receipts1 := types.Receipts{{
- Logs: []*types.Log{
- {
- Address: addr1,
- Topics: []common.Hash{topic1},
- },
- {
- Address: addr1,
- Topics: []common.Hash{topic2},
- },
- },
- }}
- receipts2 := types.Receipts{{
- Logs: []*types.Log{
- {
- Address: addr2,
- Topics: []common.Hash{topic2},
- },
- },
- }}
- err := rawdb.AppendReceipts(tx, 1, receipts1)
- require.NoError(err)
+ expectAddrs, expectTopics := genReceipts(t, tx, 10000)
- err = rawdb.AppendReceipts(tx, 2, receipts2)
- require.NoError(err)
- cfg := StageLogIndexCfg(db, "")
+ cfg := StageLogIndexCfg(nil, prune.DefaultMode, "")
cfgCopy := cfg
cfgCopy.bufLimit = 10
- cfgCopy.flushEvery = time.Millisecond
- err = promoteLogIndex("logPrefix", tx, 0, cfgCopy, ctx)
+ cfgCopy.flushEvery = time.Nanosecond
+ err := promoteLogIndex("logPrefix", tx, 0, cfgCopy, ctx)
require.NoError(err)
// Check indices GetCardinality (in how many blocks they meet)
- m, err := bitmapdb.Get(tx, dbutils.LogAddressIndex, addr1[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(1, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogAddressIndex, addr2[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(1, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogTopicIndex, topic1[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(1, int(m.GetCardinality()), 0, 10_000_000)
-
- m, err = bitmapdb.Get(tx, dbutils.LogTopicIndex, topic2[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(2, int(m.GetCardinality()))
+ for addr, expect := range expectAddrs {
+ m, err := bitmapdb.Get(tx, dbutils.LogAddressIndex, addr[:], 0, 10_000_000)
+ require.NoError(err)
+ require.Equal(expect, m.GetCardinality())
+ }
+ for topic, expect := range expectTopics {
+ m, err := bitmapdb.Get(tx, dbutils.LogTopicIndex, topic[:], 0, 10_000_000)
+ require.NoError(err)
+ require.Equal(expect, m.GetCardinality())
+ }
+
+ // Mode test
+ err = pruneLogIndex("", tx, tmpDir, 500, ctx)
+ require.NoError(err)
+
+ {
+ total := 0
+ err = tx.ForEach(dbutils.LogAddressIndex, nil, func(k, v []byte) error {
+ require.True(binary.BigEndian.Uint32(k[common.AddressLength:]) >= 500)
+ total++
+ return nil
+ })
+ require.NoError(err)
+ require.True(total > 0)
+ }
+ {
+ total := 0
+ err = tx.ForEach(dbutils.LogTopicIndex, nil, func(k, v []byte) error {
+ require.True(binary.BigEndian.Uint32(k[common.HashLength:]) >= 500)
+ total++
+ return nil
+ })
+ require.NoError(err)
+ require.True(total > 0)
+ }
// Unwind test
- err = unwindLogIndex("logPrefix", tx, 1, cfg, nil)
- require.NoError(err)
-
- m, err = bitmapdb.Get(tx, dbutils.LogAddressIndex, addr1[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(1, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogAddressIndex, addr2[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(0, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogTopicIndex, topic1[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(1, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogTopicIndex, topic2[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(1, int(m.GetCardinality()))
-
- // Unwind test
- err = unwindLogIndex("logPrefix", tx, 0, cfg, nil)
- require.NoError(err)
-
- m, err = bitmapdb.Get(tx, dbutils.LogAddressIndex, addr1[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(0, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogAddressIndex, addr2[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(0, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogTopicIndex, topic1[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(0, int(m.GetCardinality()))
-
- m, err = bitmapdb.Get(tx, dbutils.LogTopicIndex, topic2[:], 0, 10_000_000)
- require.NoError(err)
- require.Equal(0, int(m.GetCardinality()))
+ err = unwindLogIndex("logPrefix", tx, 700, cfg, nil)
+ require.NoError(err)
+
+ for addr := range expectAddrs {
+ m, err := bitmapdb.Get(tx, dbutils.LogAddressIndex, addr[:], 0, 10_000_000)
+ require.NoError(err)
+ require.True(m.Maximum() <= 700)
+ }
+ for topic := range expectTopics {
+ m, err := bitmapdb.Get(tx, dbutils.LogTopicIndex, topic[:], 0, 10_000_000)
+ require.NoError(err)
+ require.True(m.Maximum() <= 700)
+ }
}
diff --git a/eth/stagedsync/stage_txlookup.go b/eth/stagedsync/stage_txlookup.go
index 1cdc32224b6318e5c364a651eb7683a90048672f..d7f3032c89b665108b71b65641a4fb1ae986523b 100644
--- a/eth/stagedsync/stage_txlookup.go
+++ b/eth/stagedsync/stage_txlookup.go
@@ -13,20 +13,24 @@ import (
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/core/types"
"github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/rlp"
)
type TxLookupCfg struct {
db ethdb.RwKV
+ prune prune.Mode
tmpdir string
}
func StageTxLookupCfg(
db ethdb.RwKV,
+ prune prune.Mode,
tmpdir string,
) TxLookupCfg {
return TxLookupCfg{
db: db,
+ prune: prune,
tmpdir: tmpdir,
}
}
@@ -41,25 +45,25 @@ func SpawnTxLookup(s *StageState, tx ethdb.RwTx, cfg TxLookupCfg, ctx context.Co
}
defer tx.Rollback()
}
-
- var blockNum uint64
- var startKey []byte
-
- lastProcessedBlockNumber := s.BlockNumber
- if lastProcessedBlockNumber > 0 {
- blockNum = lastProcessedBlockNumber + 1
- }
- syncHeadNumber, err := s.ExecutionAt(tx)
+ logPrefix := s.LogPrefix()
+ endBlock, err := s.ExecutionAt(tx)
if err != nil {
return err
}
- logPrefix := s.LogPrefix()
- startKey = dbutils.EncodeBlockNumber(blockNum)
- if err = TxLookupTransform(logPrefix, tx, startKey, dbutils.EncodeBlockNumber(syncHeadNumber), quitCh, cfg); err != nil {
+ startBlock := s.BlockNumber
+ pruneTo := cfg.prune.TxIndex.PruneTo(endBlock)
+ if startBlock < pruneTo {
+ startBlock = pruneTo
+ }
+ if startBlock > 0 {
+ startBlock++
+ }
+ startKey := dbutils.EncodeBlockNumber(startBlock)
+ if err = TxLookupTransform(logPrefix, tx, startKey, dbutils.EncodeBlockNumber(endBlock), quitCh, cfg); err != nil {
return err
}
- if err = s.Update(tx, syncHeadNumber); err != nil {
+ if err = s.Update(tx, endBlock); err != nil {
return err
}
@@ -72,6 +76,7 @@ func SpawnTxLookup(s *StageState, tx ethdb.RwTx, cfg TxLookupCfg, ctx context.Co
}
func TxLookupTransform(logPrefix string, tx ethdb.RwTx, startKey, endKey []byte, quitCh <-chan struct{}, cfg TxLookupCfg) error {
+ bigNum := new(big.Int)
return etl.Transform(logPrefix, tx, dbutils.HeaderCanonicalBucket, dbutils.TxLookupPrefix, cfg.tmpdir, func(k []byte, v []byte, next etl.ExtractNextFunc) error {
blocknum := binary.BigEndian.Uint64(k)
blockHash := common.BytesToHash(v)
@@ -80,9 +85,8 @@ func TxLookupTransform(logPrefix string, tx ethdb.RwTx, startKey, endKey []byte,
return fmt.Errorf("%s: tx lookup generation, empty block body %d, hash %x", logPrefix, blocknum, v)
}
- blockNumBytes := new(big.Int).SetUint64(blocknum).Bytes()
for _, tx := range body.Transactions {
- if err := next(k, tx.Hash().Bytes(), blockNumBytes); err != nil {
+ if err := next(k, tx.Hash().Bytes(), bigNum.SetUint64(blocknum).Bytes()); err != nil {
return err
}
}
@@ -126,53 +130,40 @@ func UnwindTxLookup(u *UnwindState, s *StageState, tx ethdb.RwTx, cfg TxLookupCf
}
func unwindTxLookup(u *UnwindState, s *StageState, tx ethdb.RwTx, cfg TxLookupCfg, quitCh <-chan struct{}) error {
- collector := etl.NewCollector(cfg.tmpdir, etl.NewSortableBuffer(etl.BufferOptimalSize))
- defer collector.Close("TxLookup")
-
+ reader := bytes.NewReader(nil)
logPrefix := s.LogPrefix()
- c, err := tx.Cursor(dbutils.BlockBodyPrefix)
- if err != nil {
- return err
- }
- defer c.Close()
- // Remove lookup entries for blocks between unwindPoint+1 and stage.BlockNumber
- if err := ethdb.Walk(c, dbutils.EncodeBlockNumber(u.UnwindPoint+1), 0, func(k, v []byte) (b bool, e error) {
- if err := common.Stopped(quitCh); err != nil {
- return false, err
- }
-
- blockNumber := binary.BigEndian.Uint64(k[:8])
- if blockNumber > s.BlockNumber {
- return false, nil
- }
-
- if err := common.Stopped(quitCh); err != nil {
- return false, err
- }
-
+ return etl.Transform(logPrefix, tx, dbutils.BlockBodyPrefix, dbutils.TxLookupPrefix, cfg.tmpdir, func(k, v []byte, next etl.ExtractNextFunc) error {
body := new(types.BodyForStorage)
- if err := rlp.Decode(bytes.NewReader(v), body); err != nil {
- return false, fmt.Errorf("%s, rlp decode err: %w", logPrefix, err)
+ reader.Reset(v)
+ if err := rlp.Decode(reader, body); err != nil {
+ return fmt.Errorf("%s, rlp decode err: %w", logPrefix, err)
}
- txs, _ := rawdb.ReadTransactions(tx, body.BaseTxId, body.TxAmount)
+ txs, err := rawdb.ReadTransactions(tx, body.BaseTxId, body.TxAmount)
+ if err != nil {
+ return err
+ }
for _, txn := range txs {
- if err := collector.Collect(txn.Hash().Bytes(), nil); err != nil {
- return false, err
+ if err = next(k, txn.Hash().Bytes(), nil); err != nil {
+ return err
}
}
-
- return true, nil
- }); err != nil {
- return err
- }
- if err := collector.Load(logPrefix, tx, dbutils.TxLookupPrefix, etl.IdentityLoadFunc, etl.TransformArgs{Quit: quitCh}); err != nil {
- return err
- }
- return nil
+ return nil
+ }, etl.IdentityLoadFunc, etl.TransformArgs{
+ Quit: quitCh,
+ ExtractStartKey: dbutils.EncodeBlockNumber(u.UnwindPoint + 1),
+ //ExtractEndKey: dbutils.EncodeBlockNumber(s.BlockNumber - 1),
+ LogDetailsExtract: func(k, v []byte) (additionalLogArguments []interface{}) {
+ return []interface{}{"block", binary.BigEndian.Uint64(k)}
+ },
+ })
}
func PruneTxLookup(s *PruneState, tx ethdb.RwTx, cfg TxLookupCfg, ctx context.Context) (err error) {
+ if !cfg.prune.TxIndex.Enabled() {
+ return nil
+ }
+ logPrefix := s.LogPrefix()
useExternalTx := tx != nil
if !useExternalTx {
tx, err = cfg.db.BeginRw(ctx)
@@ -182,6 +173,18 @@ func PruneTxLookup(s *PruneState, tx ethdb.RwTx, cfg TxLookupCfg, ctx context.Co
defer tx.Rollback()
}
+ to := cfg.prune.TxIndex.PruneTo(s.ForwardProgress)
+ // Forward stage doesn't write anything before PruneTo point
+ // TODO: maybe need do binary search of values in db in this case
+ if s.PruneProgress != 0 {
+ if err = pruneTxLookup(tx, logPrefix, cfg.tmpdir, s, to, ctx); err != nil {
+ return err
+ }
+ }
+ if err = s.Done(tx); err != nil {
+ return err
+ }
+
if !useExternalTx {
if err = tx.Commit(); err != nil {
return err
@@ -189,3 +192,32 @@ func PruneTxLookup(s *PruneState, tx ethdb.RwTx, cfg TxLookupCfg, ctx context.Co
}
return nil
}
+
+func pruneTxLookup(tx ethdb.RwTx, logPrefix, tmpDir string, s *PruneState, pruneTo uint64, ctx context.Context) error {
+ reader := bytes.NewReader(nil)
+ return etl.Transform(logPrefix, tx, dbutils.BlockBodyPrefix, dbutils.TxLookupPrefix, tmpDir, func(k, v []byte, next etl.ExtractNextFunc) error {
+ body := new(types.BodyForStorage)
+ reader.Reset(v)
+ if err := rlp.Decode(reader, body); err != nil {
+ return fmt.Errorf("%s, rlp decode err: %w", logPrefix, err)
+ }
+
+ txs, err := rawdb.ReadTransactions(tx, body.BaseTxId, body.TxAmount)
+ if err != nil {
+ return err
+ }
+ for _, txn := range txs {
+ if err := next(k, txn.Hash().Bytes(), nil); err != nil {
+ return err
+ }
+ }
+ return nil
+ }, etl.IdentityLoadFunc, etl.TransformArgs{
+ Quit: ctx.Done(),
+ ExtractStartKey: dbutils.EncodeBlockNumber(s.ForwardProgress),
+ ExtractEndKey: dbutils.EncodeBlockNumber(pruneTo),
+ LogDetailsExtract: func(k, v []byte) (additionalLogArguments []interface{}) {
+ return []interface{}{"block", binary.BigEndian.Uint64(k)}
+ },
+ })
+}
diff --git a/eth/stagedsync/stages/stages.go b/eth/stagedsync/stages/stages.go
index 1d255a3c2391c0979d7af8ca46c01d5452a1871e..158a53170e251f1cee2680b5d4730b3e5794fbcc 100644
--- a/eth/stagedsync/stages/stages.go
+++ b/eth/stagedsync/stages/stages.go
@@ -86,6 +86,19 @@ func SaveStageProgress(db ethdb.Putter, stage SyncStage, progress uint64) error
return db.Put(dbutils.SyncStageProgress, []byte(stage), marshalData(progress))
}
+// GetStagePruneProgress retrieves saved progress of given sync stage from the database
+func GetStagePruneProgress(db ethdb.KVGetter, stage SyncStage) (uint64, error) {
+ v, err := db.GetOne(dbutils.SyncStageProgress, []byte("prune_"+stage))
+ if err != nil {
+ return 0, err
+ }
+ return unmarshalData(v)
+}
+
+func SaveStagePruneProgress(db ethdb.Putter, stage SyncStage, progress uint64) error {
+ return db.Put(dbutils.SyncStageProgress, []byte("prune_"+stage), marshalData(progress))
+}
+
func marshalData(blockNumber uint64) []byte {
return encodeBigEndian(blockNumber)
}
diff --git a/eth/stagedsync/sync.go b/eth/stagedsync/sync.go
index 0f664afe9fb9dcf7fd6b89c299607af46daeb207..fb151b3043ab65e3197b7e3b7ffed9e045598ec0 100644
--- a/eth/stagedsync/sync.go
+++ b/eth/stagedsync/sync.go
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"os"
- "runtime"
"time"
"github.com/ledgerwatch/erigon/common"
@@ -13,7 +12,6 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/log"
- "github.com/ledgerwatch/erigon/params"
)
type Sync struct {
@@ -25,6 +23,13 @@ type Sync struct {
unwindOrder []*Stage
pruningOrder []*Stage
currentStage uint
+ timings []Timing
+}
+type Timing struct {
+ isUnwind bool
+ isPrune bool
+ stage stages.SyncStage
+ took time.Duration
}
func (s *Sync) Len() int { return len(s.stages) }
@@ -34,8 +39,28 @@ func (s *Sync) NewUnwindState(id stages.SyncStage, unwindPoint, currentProgress
return &UnwindState{id, unwindPoint, currentProgress, common.Hash{}, s}
}
-func (s *Sync) NewPruneState(id stages.SyncStage, prunePoint, currentProgress uint64) *PruneState {
- return &PruneState{id, prunePoint, currentProgress, s}
+func (s *Sync) PruneStageState(id stages.SyncStage, forwardProgress uint64, tx ethdb.Tx, db ethdb.RwKV) (*PruneState, error) {
+ var pruneProgress uint64
+ var err error
+ useExternalTx := tx != nil
+ if useExternalTx {
+ pruneProgress, err = stages.GetStagePruneProgress(tx, id)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ if err = db.View(context.Background(), func(tx ethdb.Tx) error {
+ pruneProgress, err = stages.GetStagePruneProgress(tx, id)
+ if err != nil {
+ return err
+ }
+ return nil
+ }); err != nil {
+ return nil, err
+ }
+ }
+
+ return &PruneState{id, forwardProgress, pruneProgress, s}, nil
}
func (s *Sync) NextStage() {
@@ -130,7 +155,7 @@ func New(stagesList []*Stage, unwindOrder UnwindOrder, pruneOrder PruneOrder) *S
stages: stagesList,
currentStage: 0,
unwindOrder: unwindStages,
- //pruningOrder: pruneStages,
+ pruningOrder: pruneStages,
}
}
@@ -160,18 +185,16 @@ func (s *Sync) StageState(stage stages.SyncStage, tx ethdb.Tx, db ethdb.RoKV) (*
func (s *Sync) Run(db ethdb.RwKV, tx ethdb.RwTx, firstCycle bool) error {
s.prevUnwindPoint = nil
- var timings []interface{}
+ s.timings = s.timings[:0]
for !s.IsDone() {
if s.unwindPoint != nil {
for j := 0; j < len(s.unwindOrder); j++ {
- if s.unwindOrder[j].Disabled || s.unwindOrder[j].Unwind == nil {
+ if s.unwindOrder[j] == nil || s.unwindOrder[j].Disabled || s.unwindOrder[j].Unwind == nil {
continue
}
- t := time.Now()
if err := s.unwindStage(firstCycle, s.unwindOrder[j], db, tx); err != nil {
return err
}
- timings = append(timings, "Unwind "+string(s.unwindOrder[j].ID), time.Since(t))
}
s.prevUnwindPoint = s.unwindPoint
s.unwindPoint = nil
@@ -196,44 +219,53 @@ func (s *Sync) Run(db ethdb.RwKV, tx ethdb.RwTx, firstCycle bool) error {
continue
}
- t := time.Now()
if err := s.runStage(stage, db, tx, firstCycle); err != nil {
return err
}
- timings = append(timings, string(stage.ID), time.Since(t))
s.NextStage()
}
for i := 0; i < len(s.pruningOrder); i++ {
- if s.pruningOrder[i].Disabled || s.pruningOrder[i].Prune == nil {
+ if s.pruningOrder[i] == nil || s.pruningOrder[i].Disabled || s.pruningOrder[i].Prune == nil {
continue
}
- t := time.Now()
if err := s.pruneStage(firstCycle, s.pruningOrder[i], db, tx); err != nil {
return err
}
- timings = append(timings, "Pruning "+string(s.pruningOrder[i].ID), time.Since(t))
}
if err := s.SetCurrentStage(s.stages[0].ID); err != nil {
return err
}
- if err := printLogs(tx, timings); err != nil {
+ if err := printLogs(tx, s.timings); err != nil {
return err
}
s.currentStage = 0
return nil
}
-func printLogs(tx ethdb.RwTx, timings []interface{}) error {
- var m runtime.MemStats
- runtime.ReadMemStats(&m)
- log.Info("Memory", "alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys))
- if len(timings) > 50 {
- log.Info("Timings (first 50)", timings[:50]...)
- } else {
- log.Info("Timings", timings...)
+func printLogs(tx ethdb.RwTx, timings []Timing) error {
+ var logCtx []interface{}
+ count := 0
+ for i := range timings {
+ if timings[i].took < 10*time.Millisecond {
+ continue
+ }
+ count++
+ if count == 50 {
+ break
+ }
+ if timings[i].isUnwind {
+ logCtx = append(logCtx, "Unwind "+string(timings[i].stage), timings[i].took.String())
+ } else if timings[i].isPrune {
+ logCtx = append(logCtx, "Prune "+string(timings[i].stage), timings[i].took.String())
+ } else {
+ logCtx = append(logCtx, string(timings[i].stage), timings[i].took.String())
+ }
+ }
+ if len(logCtx) > 0 {
+ log.Info("Timings (slower than 10ms)", logCtx...)
}
if tx == nil {
@@ -261,26 +293,27 @@ func printLogs(tx ethdb.RwTx, timings []interface{}) error {
}
func (s *Sync) runStage(stage *Stage, db ethdb.RwKV, tx ethdb.RwTx, firstCycle bool) (err error) {
+ start := time.Now()
stageState, err := s.StageState(stage.ID, tx, db)
if err != nil {
return err
}
- start := time.Now()
logPrefix := s.LogPrefix()
if err = stage.Forward(firstCycle, stageState, s, tx); err != nil {
return err
}
- if time.Since(start) > 30*time.Second {
- log.Info(fmt.Sprintf("[%s] DONE", logPrefix), "in", time.Since(start))
+ t := time.Since(start)
+ if t > 60*time.Second {
+ log.Info(fmt.Sprintf("[%s] DONE", logPrefix), "in", t)
}
return nil
}
func (s *Sync) unwindStage(firstCycle bool, stage *Stage, db ethdb.RwKV, tx ethdb.RwTx) error {
- start := time.Now()
- log.Info("Unwind...", "stage", stage.ID)
+ t := time.Now()
+ log.Debug("Unwind...", "stage", stage.ID)
stageState, err := s.StageState(stage.ID, tx, db)
if err != nil {
return err
@@ -302,25 +335,26 @@ func (s *Sync) unwindStage(firstCycle bool, stage *Stage, db ethdb.RwKV, tx ethd
return err
}
- if time.Since(start) > 30*time.Second {
- log.Info("Unwind... DONE!", "stage", string(unwind.ID))
+ took := time.Since(t)
+ if took > 60*time.Second {
+ log.Info("Unwind... DONE!", "stage", string(unwind.ID), "in", took)
}
+ s.timings = append(s.timings, Timing{isUnwind: true, stage: stage.ID, took: time.Since(t)})
return nil
}
func (s *Sync) pruneStage(firstCycle bool, stage *Stage, db ethdb.RwKV, tx ethdb.RwTx) error {
- start := time.Now()
- log.Info("Prune...", "stage", stage.ID)
+ t := time.Now()
+ log.Debug("Prune...", "stage", stage.ID)
stageState, err := s.StageState(stage.ID, tx, db)
if err != nil {
return err
}
- prunePoint := stageState.BlockNumber - params.FullImmutabilityThreshold // TODO: cli-customizable
- prune := s.NewPruneState(stage.ID, prunePoint, stageState.BlockNumber)
- if stageState.BlockNumber <= prune.PrunePoint {
- return nil
+ prune, err := s.PruneStageState(stage.ID, stageState.BlockNumber, tx, db)
+ if err != nil {
+ return err
}
if err = s.SetCurrentStage(stage.ID); err != nil {
return err
@@ -331,9 +365,11 @@ func (s *Sync) pruneStage(firstCycle bool, stage *Stage, db ethdb.RwKV, tx ethdb
return err
}
- if time.Since(start) > 30*time.Second {
- log.Info("Prune... DONE!", "stage", string(prune.ID))
+ took := time.Since(t)
+ if took > 60*time.Second {
+ log.Info("Prune... DONE!", "stage", string(prune.ID), "in", took)
}
+ s.timings = append(s.timings, Timing{isPrune: true, stage: stage.ID, took: time.Since(t)})
return nil
}
diff --git a/ethdb/kv_interface.go b/ethdb/kv_interface.go
index 07f08df95876cd303106d1505b6fd24a8fb4b4c1..b6a1bd9958a1fb59686ced96f41c29b2632b50dd 100644
--- a/ethdb/kv_interface.go
+++ b/ethdb/kv_interface.go
@@ -89,7 +89,10 @@ type Has interface {
// Has indicates whether a key exists in the database.
Has(bucket string, key []byte) (bool, error)
}
-
+type GetPut interface {
+ KVGetter
+ Putter
+}
type KVGetter interface {
Has
diff --git a/ethdb/prune/storage_mode.go b/ethdb/prune/storage_mode.go
new file mode 100644
index 0000000000000000000000000000000000000000..a289c398270241838b1be6782e6c77a7230de29a
--- /dev/null
+++ b/ethdb/prune/storage_mode.go
@@ -0,0 +1,306 @@
+package prune
+
+import (
+ "encoding/binary"
+ "fmt"
+ "math"
+
+ "github.com/ledgerwatch/erigon/common/dbutils"
+ "github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/params"
+)
+
+var DefaultMode = Mode{
+ Initialised: true,
+ History: math.MaxUint64, // all off
+ Receipts: math.MaxUint64,
+ TxIndex: math.MaxUint64,
+ CallTraces: math.MaxUint64,
+ Experiments: Experiments{}, // all off
+}
+
+type Experiments struct {
+ TEVM bool
+}
+
+func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallTraces uint64, experiments []string) (Mode, error) {
+ mode := DefaultMode
+ if flags == "default" || flags == "disabled" {
+ return DefaultMode, nil
+ }
+ mode.Initialised = true
+ for _, flag := range flags {
+ switch flag {
+ case 'h':
+ mode.History = params.FullImmutabilityThreshold
+ case 'r':
+ mode.Receipts = params.FullImmutabilityThreshold
+ case 't':
+ mode.TxIndex = params.FullImmutabilityThreshold
+ case 'c':
+ mode.CallTraces = params.FullImmutabilityThreshold
+ default:
+ return DefaultMode, fmt.Errorf("unexpected flag found: %c", flag)
+ }
+ }
+
+ if exactHistory > 0 {
+ mode.History = Distance(exactHistory)
+ }
+ if exactReceipts > 0 {
+ mode.Receipts = Distance(exactReceipts)
+ }
+ if exactTxIndex > 0 {
+ mode.TxIndex = Distance(exactTxIndex)
+ }
+ if exactCallTraces > 0 {
+ mode.CallTraces = Distance(exactCallTraces)
+ }
+
+ for _, ex := range experiments {
+ switch ex {
+ case "tevm":
+ mode.Experiments.TEVM = true
+ case "":
+ // skip
+ default:
+ return DefaultMode, fmt.Errorf("unexpected experiment found: %s", ex)
+ }
+ }
+
+ return mode, nil
+}
+
+func Get(db ethdb.KVGetter) (Mode, error) {
+ prune := DefaultMode
+ prune.Initialised = true
+
+ v, err := db.GetOne(dbutils.DatabaseInfoBucket, dbutils.PruneDistanceHistory)
+ if err != nil {
+ return prune, err
+ }
+ if v != nil {
+ prune.History = Distance(binary.BigEndian.Uint64(v))
+ } else {
+ prune.History = math.MaxUint64
+ }
+ v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.PruneDistanceReceipts)
+ if err != nil {
+ return prune, err
+ }
+ if v != nil {
+ prune.Receipts = Distance(binary.BigEndian.Uint64(v))
+ } else {
+ prune.Receipts = math.MaxUint64
+ }
+ v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.PruneDistanceTxIndex)
+ if err != nil {
+ return prune, err
+ }
+ if v != nil {
+ prune.TxIndex = Distance(binary.BigEndian.Uint64(v))
+ } else {
+ prune.TxIndex = math.MaxUint64
+ }
+
+ v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.PruneDistanceCallTraces)
+ if err != nil {
+ return prune, err
+ }
+ if v != nil {
+ prune.CallTraces = Distance(binary.BigEndian.Uint64(v))
+ } else {
+ prune.CallTraces = math.MaxUint64
+ }
+
+ v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeTEVM)
+ if err != nil {
+ return prune, err
+ }
+ prune.Experiments.TEVM = len(v) == 1 && v[0] == 1
+ return prune, nil
+}
+
+type Mode struct {
+ Initialised bool // Set when the values are initialised (not default)
+ History Distance
+ Receipts Distance
+ TxIndex Distance
+ CallTraces Distance
+ Experiments Experiments
+}
+
+// Distance amount of blocks to keep in DB
+// but manual manipulation with such distance is very unsafe
+// for example:
+// deleteUntil := currentStageProgress - pruningDistance
+// may delete whole db - because of uint64 underflow when pruningDistance > currentStageProgress
+type Distance uint64
+
+func (p Distance) Enabled() bool { return p != math.MaxUint64 }
+
+func (p Distance) PruneTo(stageHead uint64) uint64 {
+ if p == 0 {
+ panic("pruning distance were not set")
+ }
+ if uint64(p) > stageHead {
+ return 0
+ }
+ return stageHead - uint64(p)
+}
+
+func (m Mode) ToString() string {
+ if !m.Initialised {
+ return "default"
+ }
+ modeString := ""
+ if m.History.Enabled() {
+ modeString += fmt.Sprintf("history=%d,", m.History)
+ } else {
+ modeString += "history=no,"
+ }
+ if m.Receipts.Enabled() {
+ modeString += fmt.Sprintf("receipt=%d,", m.Receipts)
+ } else {
+ modeString += "receipt=no,"
+ }
+ if m.TxIndex.Enabled() {
+ modeString += fmt.Sprintf("txindex=%d,", m.TxIndex)
+ } else {
+ modeString += "txindex=no,"
+ }
+ if m.CallTraces.Enabled() {
+ modeString += fmt.Sprintf("calltrace=%d,", m.CallTraces)
+ } else {
+ modeString += "calltrace=no,"
+ }
+ if m.Experiments.TEVM {
+ modeString += "experiments.tevm=enabled"
+ } else {
+ modeString += "experiments.tevm=disabled"
+ }
+ return modeString
+}
+
+func Override(db ethdb.RwTx, sm Mode) error {
+ var (
+ err error
+ )
+
+ err = setDistance(db, dbutils.PruneDistanceHistory, sm.History)
+ if err != nil {
+ return err
+ }
+
+ err = setDistance(db, dbutils.PruneDistanceReceipts, sm.Receipts)
+ if err != nil {
+ return err
+ }
+
+ err = setDistance(db, dbutils.PruneDistanceTxIndex, sm.TxIndex)
+ if err != nil {
+ return err
+ }
+
+ err = setDistance(db, dbutils.PruneDistanceCallTraces, sm.CallTraces)
+ if err != nil {
+ return err
+ }
+
+ err = setMode(db, dbutils.StorageModeTEVM, sm.Experiments.TEVM)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func SetIfNotExist(db ethdb.GetPut, pm Mode) error {
+ var (
+ err error
+ )
+ if !pm.Initialised {
+ pm = DefaultMode
+ }
+
+ err = setDistanceOnEmpty(db, dbutils.PruneDistanceHistory, pm.History)
+ if err != nil {
+ return err
+ }
+
+ err = setDistanceOnEmpty(db, dbutils.PruneDistanceReceipts, pm.Receipts)
+ if err != nil {
+ return err
+ }
+
+ err = setDistanceOnEmpty(db, dbutils.PruneDistanceTxIndex, pm.TxIndex)
+ if err != nil {
+ return err
+ }
+
+ err = setDistanceOnEmpty(db, dbutils.PruneDistanceCallTraces, pm.CallTraces)
+ if err != nil {
+ return err
+ }
+
+ err = setModeOnEmpty(db, dbutils.StorageModeTEVM, pm.Experiments.TEVM)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func setDistance(db ethdb.Putter, key []byte, distance Distance) error {
+ v := make([]byte, 8)
+ binary.BigEndian.PutUint64(v, uint64(distance))
+ if err := db.Put(dbutils.DatabaseInfoBucket, key, v); err != nil {
+ return err
+ }
+ return nil
+}
+
+func setDistanceOnEmpty(db ethdb.GetPut, key []byte, distance Distance) error {
+ mode, err := db.GetOne(dbutils.DatabaseInfoBucket, key)
+ if err != nil {
+ return err
+ }
+ if len(mode) == 0 || binary.BigEndian.Uint64(mode) == math.MaxUint64 {
+ v := make([]byte, 8)
+ binary.BigEndian.PutUint64(v, uint64(distance))
+ if err = db.Put(dbutils.DatabaseInfoBucket, key, v); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func setMode(db ethdb.RwTx, key []byte, currentValue bool) error {
+ val := []byte{2}
+ if currentValue {
+ val = []byte{1}
+ }
+ if err := db.Put(dbutils.DatabaseInfoBucket, key, val); err != nil {
+ return err
+ }
+ return nil
+}
+
+func setModeOnEmpty(db ethdb.GetPut, key []byte, currentValue bool) error {
+ mode, err := db.GetOne(dbutils.DatabaseInfoBucket, key)
+ if err != nil {
+ return err
+ }
+ if len(mode) == 0 {
+ val := []byte{2}
+ if currentValue {
+ val = []byte{1}
+ }
+ if err = db.Put(dbutils.DatabaseInfoBucket, key, val); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/ethdb/prune/storage_mode_test.go b/ethdb/prune/storage_mode_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..d121ab5642b79d11269926d1137fe12709ce0513
--- /dev/null
+++ b/ethdb/prune/storage_mode_test.go
@@ -0,0 +1,23 @@
+package prune
+
+import (
+ "testing"
+
+ "github.com/ledgerwatch/erigon/common/math"
+ "github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestSetStorageModeIfNotExist(t *testing.T) {
+ _, tx := kv.NewTestTx(t)
+ prune, err := Get(tx)
+ assert.NoError(t, err)
+ assert.Equal(t, Mode{true, math.MaxUint64, math.MaxUint64, math.MaxUint64, math.MaxUint64, Experiments{TEVM: false}}, prune)
+
+ err = SetIfNotExist(tx, Mode{true, 1, 2, 3, 4, Experiments{TEVM: false}})
+ assert.NoError(t, err)
+
+ prune, err = Get(tx)
+ assert.NoError(t, err)
+ assert.Equal(t, Mode{true, 1, 2, 3, 4, Experiments{TEVM: false}}, prune)
+}
diff --git a/ethdb/storage_mode.go b/ethdb/storage_mode.go
deleted file mode 100644
index 050c04c257ba15ded35676deb0441d93558f01a4..0000000000000000000000000000000000000000
--- a/ethdb/storage_mode.go
+++ /dev/null
@@ -1,212 +0,0 @@
-package ethdb
-
-import (
- "fmt"
-
- "github.com/ledgerwatch/erigon/common/dbutils"
-)
-
-type StorageMode struct {
- Initialised bool // Set when the values are initialised (not default)
- History bool
- Receipts bool
- TxIndex bool
- CallTraces bool
- TEVM bool
-}
-
-var DefaultStorageMode = StorageMode{
- Initialised: true,
- History: true,
- Receipts: true,
- TxIndex: true,
- CallTraces: true,
- TEVM: false,
-}
-
-func (m StorageMode) ToString() string {
- if !m.Initialised {
- return "default"
- }
- modeString := ""
- if m.History {
- modeString += "h"
- }
- if m.Receipts {
- modeString += "r"
- }
- if m.TxIndex {
- modeString += "t"
- }
- if m.CallTraces {
- modeString += "c"
- }
- if m.TEVM {
- modeString += "e"
- }
- return modeString
-}
-
-func StorageModeFromString(flags string) (StorageMode, error) {
- mode := StorageMode{}
- if flags == "default" {
- return mode, nil
- }
- mode.Initialised = true
- for _, flag := range flags {
- switch flag {
- case 'h':
- mode.History = true
- case 'r':
- mode.Receipts = true
- case 't':
- mode.TxIndex = true
- case 'c':
- mode.CallTraces = true
- case 'e':
- mode.TEVM = true
- default:
- return mode, fmt.Errorf("unexpected flag found: %c", flag)
- }
- }
-
- return mode, nil
-}
-
-func GetStorageModeFromDB(db KVGetter) (StorageMode, error) {
- var (
- sm StorageMode
- v []byte
- err error
- )
- sm.Initialised = true
-
- v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeHistory)
- if err != nil {
- return StorageMode{}, err
- }
- sm.History = len(v) == 1 && v[0] == 1
-
- v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeReceipts)
- if err != nil {
- return StorageMode{}, err
- }
- sm.Receipts = len(v) == 1 && v[0] == 1
-
- v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeTxIndex)
- if err != nil {
- return StorageMode{}, err
- }
- sm.TxIndex = len(v) == 1 && v[0] == 1
-
- v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeCallTraces)
- if err != nil {
- return StorageMode{}, err
- }
- sm.CallTraces = len(v) == 1 && v[0] == 1
-
- v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeTEVM)
- if err != nil {
- return StorageMode{}, err
- }
- sm.TEVM = len(v) == 1 && v[0] == 1
- return sm, nil
-}
-
-func OverrideStorageMode(db RwTx, sm StorageMode) error {
- var (
- err error
- )
-
- err = setMode(db, dbutils.StorageModeHistory, sm.History)
- if err != nil {
- return err
- }
-
- err = setMode(db, dbutils.StorageModeReceipts, sm.Receipts)
- if err != nil {
- return err
- }
-
- err = setMode(db, dbutils.StorageModeTxIndex, sm.TxIndex)
- if err != nil {
- return err
- }
-
- err = setMode(db, dbutils.StorageModeCallTraces, sm.CallTraces)
- if err != nil {
- return err
- }
-
- err = setMode(db, dbutils.StorageModeTEVM, sm.TEVM)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func SetStorageModeIfNotExist(db RwTx, sm StorageMode) error {
- var (
- err error
- )
- if !sm.Initialised {
- sm = DefaultStorageMode
- }
-
- err = setModeOnEmpty(db, dbutils.StorageModeHistory, sm.History)
- if err != nil {
- return err
- }
-
- err = setModeOnEmpty(db, dbutils.StorageModeReceipts, sm.Receipts)
- if err != nil {
- return err
- }
-
- err = setModeOnEmpty(db, dbutils.StorageModeTxIndex, sm.TxIndex)
- if err != nil {
- return err
- }
-
- err = setModeOnEmpty(db, dbutils.StorageModeCallTraces, sm.CallTraces)
- if err != nil {
- return err
- }
-
- err = setModeOnEmpty(db, dbutils.StorageModeTEVM, sm.TEVM)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func setMode(db RwTx, key []byte, currentValue bool) error {
- val := []byte{2}
- if currentValue {
- val = []byte{1}
- }
- if err := db.Put(dbutils.DatabaseInfoBucket, key, val); err != nil {
- return err
- }
- return nil
-}
-
-func setModeOnEmpty(db RwTx, key []byte, currentValue bool) error {
- mode, err := db.GetOne(dbutils.DatabaseInfoBucket, key)
- if err != nil {
- return err
- }
- if len(mode) == 0 {
- val := []byte{2}
- if currentValue {
- val = []byte{1}
- }
- if err = db.Put(dbutils.DatabaseInfoBucket, key, val); err != nil {
- return err
- }
- }
-
- return nil
-}
diff --git a/ethdb/storage_mode_test.go b/ethdb/storage_mode_test.go
deleted file mode 100644
index 7d452b522ef2b2eef25f2d0ff2858be21d449ada..0000000000000000000000000000000000000000
--- a/ethdb/storage_mode_test.go
+++ /dev/null
@@ -1,51 +0,0 @@
-package ethdb_test
-
-import (
- "reflect"
- "testing"
-
- "github.com/davecgh/go-spew/spew"
- "github.com/ledgerwatch/erigon/ethdb"
- "github.com/ledgerwatch/erigon/ethdb/kv"
-)
-
-func TestSetStorageModeIfNotExist(t *testing.T) {
- _, tx := kv.NewTestTx(t)
- sm, err := ethdb.GetStorageModeFromDB(tx)
- if err != nil {
- t.Fatal(err)
- }
-
- if !reflect.DeepEqual(sm, ethdb.StorageMode{Initialised: true}) {
- t.Fatal()
- }
-
- err = ethdb.SetStorageModeIfNotExist(tx, ethdb.StorageMode{
- true,
- true,
- true,
- true,
- true,
- false,
- })
- if err != nil {
- t.Fatal(err)
- }
-
- sm, err = ethdb.GetStorageModeFromDB(tx)
- if err != nil {
- t.Fatal(err)
- }
-
- if !reflect.DeepEqual(sm, ethdb.StorageMode{
- true,
- true,
- true,
- true,
- true,
- false,
- }) {
- spew.Dump(sm)
- t.Fatal("not equal")
- }
-}
diff --git a/migrations/call_trace_index.go b/migrations/call_trace_index.go
index 0772538ee1b72ce5357f6b7cd7df13b4af95a077..0fff9faf85bfde2d7a6bf16e7615bd03b9981699 100644
--- a/migrations/call_trace_index.go
+++ b/migrations/call_trace_index.go
@@ -9,20 +9,13 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
)
var rebuilCallTraceIndex = Migration{
Name: "rebuild_call_trace_index",
Up: func(db ethdb.Database, tmpdir string, progress []byte, CommitProgress etl.LoadCommitHandler) (err error) {
- sm, err := ethdb.GetStorageModeFromDB(db)
- if err != nil {
- return err
- }
- if !sm.CallTraces {
- // Call traces are not on, nothing to migrate
- return CommitProgress(db, nil, true)
- }
// Find the lowest key in the TraceCallSet table
tx := db.(ethdb.HasTx).Tx().(ethdb.RwTx)
c, err := tx.CursorDupSort(dbutils.CallTraceSet)
@@ -45,7 +38,12 @@ var rebuilCallTraceIndex = Migration{
return CommitProgress(db, nil, true)
}
logPrefix := "db migration rebuild_call_trace_index"
- if err = stagedsync.DoUnwindCallTraces(logPrefix, tx, 999_999_999, blockNum-1, context.Background().Done(), stagedsync.StageCallTracesCfg(nil, 0, tmpdir)); err != nil {
+
+ pm, err := prune.Get(tx)
+ if err != nil {
+ return err
+ }
+ if err = stagedsync.DoUnwindCallTraces(logPrefix, tx, 999_999_999, blockNum-1, context.Background(), stagedsync.StageCallTracesCfg(nil, pm, 0, tmpdir)); err != nil {
return err
}
diff --git a/migrations/migrations.go b/migrations/migrations.go
index 90f4594ecc0f1f29ad1e022c71edd3165408d943..2e149cfeff61b0feb29fead88bddae0c2f151336 100644
--- a/migrations/migrations.go
+++ b/migrations/migrations.go
@@ -62,6 +62,7 @@ var migrations = map[ethdb.Label][]Migration{
dbSchemaVersion,
rebuilCallTraceIndex,
fixSequences,
+ storageMode,
},
ethdb.TxPool: {},
ethdb.Sentry: {},
diff --git a/migrations/prune.go b/migrations/prune.go
new file mode 100644
index 0000000000000000000000000000000000000000..d12c1a40ed2884944817c082d974c255ab4cf3d7
--- /dev/null
+++ b/migrations/prune.go
@@ -0,0 +1,76 @@
+package migrations
+
+import (
+ "github.com/ledgerwatch/erigon/common/dbutils"
+ "github.com/ledgerwatch/erigon/common/etl"
+ "github.com/ledgerwatch/erigon/common/math"
+ "github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
+ "github.com/ledgerwatch/erigon/params"
+)
+
+var storageMode = Migration{
+ Name: "storage_mode",
+ Up: func(db ethdb.Database, tmpdir string, progress []byte, CommitProgress etl.LoadCommitHandler) (err error) {
+ var ( // old db keys
+ //StorageModeHistory - does node save history.
+ StorageModeHistory = []byte("smHistory")
+ //StorageModeReceipts - does node save receipts.
+ StorageModeReceipts = []byte("smReceipts")
+ //StorageModeTxIndex - does node save transactions index.
+ StorageModeTxIndex = []byte("smTxIndex")
+ //StorageModeCallTraces - does not build index of call traces
+ StorageModeCallTraces = []byte("smCallTraces")
+ )
+ pm := prune.Mode{Initialised: true}
+ castToPruneDistance := func(v []byte) prune.Distance {
+ if len(v) == 1 && v[0] == 2 {
+ return params.FullImmutabilityThreshold // means, prune enabled
+ }
+ return math.MaxUint64 // means, prune disabled
+ }
+ {
+ v, err := db.GetOne(dbutils.DatabaseInfoBucket, StorageModeHistory)
+ if err != nil {
+ return err
+ }
+ pm.History = castToPruneDistance(v)
+
+ }
+ {
+ v, err := db.GetOne(dbutils.DatabaseInfoBucket, StorageModeReceipts)
+ if err != nil {
+ return err
+ }
+ pm.Receipts = castToPruneDistance(v)
+ }
+ {
+ v, err := db.GetOne(dbutils.DatabaseInfoBucket, StorageModeTxIndex)
+ if err != nil {
+ return err
+ }
+ pm.TxIndex = castToPruneDistance(v)
+ }
+ {
+ v, err := db.GetOne(dbutils.DatabaseInfoBucket, StorageModeCallTraces)
+ if err != nil {
+ return err
+ }
+ pm.CallTraces = castToPruneDistance(v)
+ }
+ {
+ v, err := db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeTEVM)
+ if err != nil {
+ return err
+ }
+ pm.Experiments.TEVM = len(v) == 1 && v[0] == 1
+ }
+
+ err = prune.SetIfNotExist(db, pm)
+ if err != nil {
+ return err
+ }
+
+ return CommitProgress(db, nil, true)
+ },
+}
diff --git a/migrations/receipt_cbor.go b/migrations/receipt_cbor.go
index cc308a5950209196fc4400638ced9e32863ee6cd..b4aec90bae7a61e6dab2902067a2d22ccb5d140c 100644
--- a/migrations/receipt_cbor.go
+++ b/migrations/receipt_cbor.go
@@ -69,7 +69,7 @@ var ReceiptCbor = Migration{
}
for blockNum := uint64(1); blockNum <= to; blockNum++ {
binary.BigEndian.PutUint64(key[:], blockNum)
- if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
+ if v, err = tx.GetOne(dbutils.Receipts, key[:]); err != nil {
return err
}
if v == nil {
@@ -108,7 +108,7 @@ var ReceiptCbor = Migration{
if err = cbor.Marshal(&buf, receipts); err != nil {
return err
}
- if err = tx.Put(dbutils.BlockReceiptsPrefix, common.CopyBytes(key[:]), common.CopyBytes(buf.Bytes())); err != nil {
+ if err = tx.Put(dbutils.Receipts, common.CopyBytes(key[:]), common.CopyBytes(buf.Bytes())); err != nil {
return err
}
}
diff --git a/migrations/receipt_repair.go b/migrations/receipt_repair.go
index 59eec8c88d53a62cf05e8c959e14f3a6940c4d0d..d667ad4490ab1c45c173cc13b19e9c44fcb50687 100644
--- a/migrations/receipt_repair.go
+++ b/migrations/receipt_repair.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/ledgerwatch/erigon/common"
+ "github.com/ledgerwatch/erigon/common/changeset"
"github.com/ledgerwatch/erigon/common/dbutils"
"github.com/ledgerwatch/erigon/common/etl"
"github.com/ledgerwatch/erigon/consensus/ethash"
@@ -22,6 +23,22 @@ import (
"github.com/ledgerwatch/erigon/params"
)
+func availableReceiptFrom(tx ethdb.Tx) (uint64, error) {
+ c, err := tx.Cursor(dbutils.Receipts)
+ if err != nil {
+ return 0, err
+ }
+ defer c.Close()
+ k, _, err := c.First()
+ if err != nil {
+ return 0, err
+ }
+ if len(k) == 0 {
+ return 0, nil
+ }
+ return binary.BigEndian.Uint64(k), nil
+}
+
var ReceiptRepair = Migration{
Name: "receipt_repair",
Up: func(db ethdb.Database, tmpdir string, progress []byte, CommitProgress etl.LoadCommitHandler) (err error) {
@@ -31,17 +48,17 @@ var ReceiptRepair = Migration{
} else {
return fmt.Errorf("no transaction")
}
- if sm, smErr := ethdb.GetStorageModeFromDB(tx); smErr != nil {
- return smErr
- } else {
- if !sm.History {
- log.Warn("Could not perform this migration because history is not in storage mode")
- return CommitProgress(db, nil, true)
- }
- if !sm.Receipts {
- log.Info("Migration is only relevant for storage mode with receipts, skipping")
- return CommitProgress(db, nil, true)
- }
+
+ blockNum, err := changeset.AvailableFrom(tx)
+ if err != nil {
+ return err
+ }
+ receiptsFrom, err := availableReceiptFrom(tx)
+ if err != nil {
+ return err
+ }
+ if receiptsFrom > blockNum {
+ blockNum = receiptsFrom
}
genesisBlock, err := rawdb.ReadBlockByNumber(tx, 0)
@@ -60,7 +77,7 @@ var ReceiptRepair = Migration{
logEvery := time.NewTicker(logInterval)
var key [8]byte
var v []byte
- for blockNum := uint64(1); true; blockNum++ {
+ for ; true; blockNum++ {
select {
default:
case <-logEvery.C:
@@ -74,7 +91,7 @@ var ReceiptRepair = Migration{
break
}
binary.BigEndian.PutUint64(key[:], blockNum)
- if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
+ if v, err = tx.GetOne(dbutils.Receipts, key[:]); err != nil {
return err
}
var receipts types.Receipts
@@ -118,7 +135,7 @@ var ReceiptRepair = Migration{
if err != nil {
return fmt.Errorf("encode block receipts for block %d: %v", blockNum, err)
}
- if err = tx.Put(dbutils.BlockReceiptsPrefix, key[:], buf.Bytes()); err != nil {
+ if err = tx.Put(dbutils.Receipts, key[:], buf.Bytes()); err != nil {
return fmt.Errorf("writing receipts for block %d: %v", blockNum, err)
}
fixedCount++
diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go
index 7da1abaadb00437a894e1ded8a4885a232a65b3b..a92cd705846698ab43f0f591d1c2ce6506d392a8 100644
--- a/turbo/cli/default_flags.go
+++ b/turbo/cli/default_flags.go
@@ -21,7 +21,11 @@ var DefaultFlags = []cli.Flag{
utils.TxPoolAccountQueueFlag,
utils.TxPoolGlobalQueueFlag,
utils.TxPoolLifetimeFlag,
- StorageModeFlag,
+ PruneFlag,
+ PruneHistoryFlag,
+ PruneReceiptFlag,
+ PruneTxIndexFlag,
+ PruneCallTracesFlag,
SnapshotModeFlag,
SeedSnapshotsFlag,
SnapshotDatabaseLayoutFlag,
diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go
index 6b5fbfb20c2471e6f7777d9eb2500463ccc066f7..2fcbd56f04dcd23e9d3abd4f64253062bcc2d047 100644
--- a/turbo/cli/flags.go
+++ b/turbo/cli/flags.go
@@ -2,6 +2,7 @@ package cli
import (
"fmt"
+ "strings"
"time"
"github.com/c2h5oh/datasize"
@@ -9,6 +10,7 @@ import (
"github.com/ledgerwatch/erigon/common/etl"
"github.com/ledgerwatch/erigon/eth/ethconfig"
"github.com/ledgerwatch/erigon/ethdb"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/log"
"github.com/ledgerwatch/erigon/node"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
@@ -56,16 +58,41 @@ var (
Value: node.DefaultConfig.P2P.MaxPeers,
}
- StorageModeFlag = cli.StringFlag{
- Name: "storage-mode",
- Usage: `Configures the storage mode of the app:
-* h - write history to the DB
-* r - write receipts to the DB
-* t - write tx lookup index to the DB
-* c - write call traces index to the DB,
-* e - write TEVM translated code to the DB`,
+ PruneFlag = cli.StringFlag{
+ Name: "prune",
+ Usage: `Choose which ancient data delete from DB:
+ h - prune history (ChangeSets, HistoryIndices - used by historical state access)
+ r - prune receipts (Receipts, Logs, LogTopicIndex, LogAddressIndex - used by eth_getLogs and similar RPC methods)
+ t - prune transaction by it's hash index
+ c - prune call traces (used by trace_* methods)
+ Does delete data older than 90K block (can set another value by '--prune.*.older' flags).
+ If item is NOT in the list - means NO pruning for this data.s
+ Example: --prune=hrtc`,
+ Value: "disabled",
+ }
+ PruneHistoryFlag = cli.Uint64Flag{
+ Name: "prune.history.older",
+ Usage: `Prune data after this amount of blocks (if --prune flag has 'h', then default is 90K)`,
+ }
+ PruneReceiptFlag = cli.Uint64Flag{
+ Name: "prune.receipt.older",
+ Usage: `Prune data after this amount of blocks (if --prune flag has 'r', then default is 90K)`,
+ }
+ PruneTxIndexFlag = cli.Uint64Flag{
+ Name: "prune.txindex.older",
+ Usage: `Prune data after this amount of blocks (if --prune flag has 't', then default is 90K)`,
+ }
+ PruneCallTracesFlag = cli.Uint64Flag{
+ Name: "prune.calltrace.older",
+ Usage: `Prune data after this amount of blocks (if --prune flag has 'c', then default is 90K)`,
+ }
+ ExperimentsFlag = cli.StringFlag{
+ Name: "experiments",
+ Usage: `Enable some experimental stages:
+* tevm - write TEVM translated code to the DB`,
Value: "default",
}
+
SnapshotModeFlag = cli.StringFlag{
Name: "snapshot.mode",
Usage: `Configures the snapshot mode of the app:
@@ -125,11 +152,19 @@ var (
)
func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config) {
- mode, err := ethdb.StorageModeFromString(ctx.GlobalString(StorageModeFlag.Name))
+ mode, err := prune.FromCli(
+ ctx.GlobalString(PruneFlag.Name),
+ ctx.GlobalUint64(PruneHistoryFlag.Name),
+ ctx.GlobalUint64(PruneReceiptFlag.Name),
+ ctx.GlobalUint64(PruneTxIndexFlag.Name),
+ ctx.GlobalUint64(PruneCallTracesFlag.Name),
+ strings.Split(ctx.GlobalString(ExperimentsFlag.Name), ","),
+ )
if err != nil {
utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err))
}
- cfg.StorageMode = mode
+ cfg.Prune = mode
+
snMode, err := snapshotsync.SnapshotModeFromString(ctx.GlobalString(SnapshotModeFlag.Name))
if err != nil {
utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err))
@@ -169,12 +204,29 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config) {
}
func ApplyFlagsForEthConfigCobra(f *pflag.FlagSet, cfg *ethconfig.Config) {
- if v := f.String(StorageModeFlag.Name, StorageModeFlag.Value, StorageModeFlag.Usage); v != nil {
- mode, err := ethdb.StorageModeFromString(*v)
+ if v := f.String(PruneFlag.Name, PruneFlag.Value, PruneFlag.Usage); v != nil {
+ var experiments []string
+ if exp := f.StringSlice(ExperimentsFlag.Name, nil, ExperimentsFlag.Usage); exp != nil {
+ experiments = *exp
+ }
+ var exactH, exactR, exactT, exactC uint64
+ if v := f.Uint64(PruneHistoryFlag.Name, PruneHistoryFlag.Value, PruneHistoryFlag.Usage); v != nil {
+ exactH = *v
+ }
+ if v := f.Uint64(PruneReceiptFlag.Name, PruneReceiptFlag.Value, PruneReceiptFlag.Usage); v != nil {
+ exactR = *v
+ }
+ if v := f.Uint64(PruneTxIndexFlag.Name, PruneTxIndexFlag.Value, PruneTxIndexFlag.Usage); v != nil {
+ exactT = *v
+ }
+ if v := f.Uint64(PruneCallTracesFlag.Name, PruneCallTracesFlag.Value, PruneCallTracesFlag.Usage); v != nil {
+ exactC = *v
+ }
+ mode, err := prune.FromCli(*v, exactH, exactR, exactT, exactC, experiments)
if err != nil {
utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err))
}
- cfg.StorageMode = mode
+ cfg.Prune = mode
}
if v := f.String(SnapshotModeFlag.Name, SnapshotModeFlag.Value, SnapshotModeFlag.Usage); v != nil {
snMode, err := snapshotsync.SnapshotModeFromString(*v)
diff --git a/turbo/snapshotsync/snapshot_builder_test.go b/turbo/snapshotsync/snapshot_builder_test.go
index cf8275ea1482d07005397c7218d6c15a789fa76c..07f2921b2ae6743891f83da562416d8042f48312 100644
--- a/turbo/snapshotsync/snapshot_builder_test.go
+++ b/turbo/snapshotsync/snapshot_builder_test.go
@@ -621,6 +621,7 @@ func TestSnapshotMigratorStageSyncMode(t *testing.T) {
t.Error(err)
}
+ defer rotx.Rollback()
//just start snapshot transaction
// it can't be empty slice but shouldn't be in main db
_, err = roTX.GetOne(dbutils.HeadersBucket, []byte{1})
diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go
index b90609a0662deb4d98c728a3532a5986604c3bd4..08713553d63d0b39c55b933320c178e45f9fa763 100644
--- a/turbo/stages/blockchain_test.go
+++ b/turbo/stages/blockchain_test.go
@@ -18,19 +18,23 @@ package stages_test
import (
"context"
+ "encoding/binary"
"errors"
"fmt"
+ "math"
"math/big"
"os"
"testing"
"github.com/holiman/uint256"
+ "github.com/ledgerwatch/erigon/common/dbutils"
+ "github.com/ledgerwatch/erigon/ethdb/bitmapdb"
"github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/ledgerwatch/erigon/common"
- "github.com/ledgerwatch/erigon/common/dbutils"
"github.com/ledgerwatch/erigon/common/hexutil"
"github.com/ledgerwatch/erigon/common/u256"
"github.com/ledgerwatch/erigon/consensus/ethash"
@@ -663,8 +667,9 @@ func TestModes(t *testing.T) {
)
}
-func doModesTest(t *testing.T, history, receipts, txlookup bool) error {
- fmt.Printf("h=%v, r=%v, t=%v\n", history, receipts, txlookup)
+func doModesTest(t *testing.T, pm prune.Mode) error {
+ fmt.Printf("h=%v, r=%v, t=%v\n", pm.History.Enabled(), pm.Receipts.Enabled(), pm.TxIndex.Enabled())
+ require := require.New(t)
// Configure and generate a sample block chain
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
@@ -676,9 +681,10 @@ func doModesTest(t *testing.T, history, receipts, txlookup bool) error {
Alloc: core.GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}},
}
)
- m := stages.MockWithGenesisStorageMode(t, gspec, key, ethdb.StorageMode{Initialised: true, History: history, Receipts: receipts, TxIndex: txlookup})
+ m := stages.MockWithGenesisStorageMode(t, gspec, key, pm)
- chain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 4, func(i int, block *core.BlockGen) {
+ head := uint64(4)
+ chain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, int(head), func(i int, block *core.BlockGen) {
var (
tx types.Transaction
err error
@@ -728,69 +734,129 @@ func doModesTest(t *testing.T, history, receipts, txlookup bool) error {
}
tx, err := m.DB.BeginRo(context.Background())
- require.NoError(t, err)
+ require.NoError(err)
defer tx.Rollback()
- for bucketName, shouldBeEmpty := range map[string]bool{
- dbutils.AccountsHistoryBucket: !history,
- dbutils.BlockReceiptsPrefix: !receipts,
- dbutils.TxLookupPrefix: !txlookup,
- } {
- numberOfEntries := 0
-
- err := tx.ForEach(bucketName, nil, func(k, v []byte) error {
- // we ignore empty account history
- //nolint:scopelint
- if bucketName == dbutils.AccountsHistoryBucket && len(v) == 0 {
- return nil
- }
-
- numberOfEntries++
+ if pm.Receipts.Enabled() {
+ receiptsAvailable, err := rawdb.ReceiptsAvailableFrom(tx)
+ require.NoError(err)
+ found := uint64(0)
+ err = tx.ForEach(dbutils.Receipts, nil, func(k, v []byte) error {
+ found++
return nil
})
- if err != nil {
- return err
- }
+ require.NoError(err)
+ require.GreaterOrEqual(receiptsAvailable, pm.Receipts.PruneTo(head))
+ require.Greater(found, uint64(0))
+ } else {
+ receiptsAvailable, err := rawdb.ReceiptsAvailableFrom(tx)
+ require.NoError(err)
+ require.Equal(uint64(0), receiptsAvailable)
+ }
+
+ if pm.History.Enabled() {
+ afterPrune := uint64(0)
+ err := tx.ForEach(dbutils.AccountsHistoryBucket, nil, func(k, _ []byte) error {
+ n := binary.BigEndian.Uint64(k[common.AddressLength:])
+ require.Greater(n, pm.History.PruneTo(head))
+ afterPrune++
+ return nil
+ })
+ require.Greater(afterPrune, uint64(0))
+ assert.NoError(t, err)
+ } else {
+ found, err := bitmapdb.Get64(tx, dbutils.AccountsHistoryBucket, address[:], 0, 1024)
+ require.NoError(err)
+ require.Equal(uint64(0), found.Minimum())
+ }
+
+ if pm.TxIndex.Enabled() {
+ b, err := rawdb.ReadBlockByNumber(tx, 1)
+ require.NoError(err)
+ for _, txn := range b.Transactions() {
+ found, err := rawdb.ReadTxLookupEntry(tx, txn.Hash())
+ require.NoError(err)
+ require.Nil(found)
+ }
+ } else {
+ b, err := rawdb.ReadBlockByNumber(tx, 1)
+ require.NoError(err)
+ for _, txn := range b.Transactions() {
+ found, err := rawdb.ReadTxLookupEntry(tx, txn.Hash())
+ require.NoError(err)
+ if found == nil {
+ require.NotNil(found)
+ }
+ require.Equal(uint64(1), *found)
+ }
+ }
+ /*
+ for bucketName, shouldBeEmpty := range map[string]bool{
+ //dbutils.AccountsHistoryBucket: pm.History.Enabled(),
+ dbutils.Receipts: pm.Receipts.Enabled(),
+ //dbutils.TxLookupPrefix: pm.TxIndex.Enabled(),
+ } {
+ numberOfEntries := 0
+
+ err := tx.ForEach(bucketName, nil, func(k, v []byte) error {
+ // we ignore empty account history
+ //nolint:scopelint
+ if bucketName == dbutils.AccountsHistoryBucket && len(v) == 0 {
+ return nil
+ }
- if bucketName == dbutils.BlockReceiptsPrefix {
- // we will always have a receipt for genesis
- numberOfEntries--
- }
+ numberOfEntries++
+ return nil
+ })
+ if err != nil {
+ return err
+ }
- if (shouldBeEmpty && numberOfEntries > 0) || (!shouldBeEmpty && numberOfEntries == 0) {
- return fmt.Errorf("bucket '%s' should be empty? %v (actually %d entries)", bucketName, shouldBeEmpty, numberOfEntries)
- }
- }
+ if bucketName == dbutils.Receipts {
+ // we will always have a receipt for genesis
+ numberOfEntries--
+ }
+ if (shouldBeEmpty && numberOfEntries > 0) || (!shouldBeEmpty && numberOfEntries == 0) {
+ return fmt.Errorf("bucket '%s' should be empty? %v (actually %d entries)", bucketName, shouldBeEmpty, numberOfEntries)
+ }
+ }
+ */
return nil
}
-func runWithModesPermuations(t *testing.T, testFunc func(*testing.T, bool, bool, bool) error) {
- err := runPermutation(t, testFunc, 0, true, true, true)
+func runWithModesPermuations(t *testing.T, testFunc func(*testing.T, prune.Mode) error) {
+ err := runPermutation(t, testFunc, 0, prune.DefaultMode)
if err != nil {
t.Errorf("error while testing stuff: %v", err)
}
}
-func runPermutation(t *testing.T, testFunc func(*testing.T, bool, bool, bool) error, current int, history, receipts, txlookup bool) error {
+func runPermutation(t *testing.T, testFunc func(*testing.T, prune.Mode) error, current int, pm prune.Mode) error {
if current == 3 {
- return testFunc(t, history, receipts, txlookup)
+ return testFunc(t, pm)
}
- if err := runPermutation(t, testFunc, current+1, history, receipts, txlookup); err != nil {
+ if err := runPermutation(t, testFunc, current+1, pm); err != nil {
return err
}
+ invert := func(a prune.Distance) prune.Distance {
+ if a.Enabled() {
+ return math.MaxUint64
+ }
+ return 2
+ }
switch current {
case 0:
- history = !history
+ pm.History = invert(pm.History)
case 1:
- receipts = !receipts
+ pm.Receipts = invert(pm.Receipts)
case 2:
- txlookup = !txlookup
+ pm.TxIndex = invert(pm.TxIndex)
default:
panic("unexpected current item")
}
- return runPermutation(t, testFunc, current+1, history, receipts, txlookup)
+ return runPermutation(t, testFunc, current+1, pm)
}
func TestEIP161AccountRemoval(t *testing.T) {
diff --git a/turbo/stages/genesis_test.go b/turbo/stages/genesis_test.go
index 5745ee092df62358b5d749f9451ff818a6b4d990..61d742644073bf81a0db20cb87d9dd93c82de949 100644
--- a/turbo/stages/genesis_test.go
+++ b/turbo/stages/genesis_test.go
@@ -144,7 +144,7 @@ func TestSetupGenesis(t *testing.T) {
{
name: "genesis without ChainConfig",
fn: func(db ethdb.RwKV) (*params.ChainConfig, *types.Block, error) {
- return core.CommitGenesisBlock(db, new(core.Genesis), true)
+ return core.CommitGenesisBlock(db, new(core.Genesis))
},
wantErr: core.ErrGenesisNoConfig,
wantConfig: params.AllEthashProtocolChanges,
@@ -152,7 +152,7 @@ func TestSetupGenesis(t *testing.T) {
{
name: "no block in DB, genesis == nil",
fn: func(db ethdb.RwKV) (*params.ChainConfig, *types.Block, error) {
- return core.CommitGenesisBlock(db, nil, true)
+ return core.CommitGenesisBlock(db, nil)
},
wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig,
@@ -160,7 +160,7 @@ func TestSetupGenesis(t *testing.T) {
{
name: "mainnet block in DB, genesis == nil",
fn: func(db ethdb.RwKV) (*params.ChainConfig, *types.Block, error) {
- return core.CommitGenesisBlock(db, nil, true)
+ return core.CommitGenesisBlock(db, nil)
},
wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig,
@@ -169,7 +169,7 @@ func TestSetupGenesis(t *testing.T) {
name: "custom block in DB, genesis == nil",
fn: func(db ethdb.RwKV) (*params.ChainConfig, *types.Block, error) {
customg.MustCommit(db)
- return core.CommitGenesisBlock(db, nil, true)
+ return core.CommitGenesisBlock(db, nil)
},
wantHash: customghash,
wantConfig: customg.Config,
@@ -178,7 +178,7 @@ func TestSetupGenesis(t *testing.T) {
name: "custom block in DB, genesis == ropsten",
fn: func(db ethdb.RwKV) (*params.ChainConfig, *types.Block, error) {
customg.MustCommit(db)
- return core.CommitGenesisBlock(db, core.DefaultRopstenGenesisBlock(), true)
+ return core.CommitGenesisBlock(db, core.DefaultRopstenGenesisBlock())
},
wantErr: &core.GenesisMismatchError{Stored: customghash, New: params.RopstenGenesisHash},
wantHash: params.RopstenGenesisHash,
@@ -188,7 +188,7 @@ func TestSetupGenesis(t *testing.T) {
name: "compatible config in DB",
fn: func(db ethdb.RwKV) (*params.ChainConfig, *types.Block, error) {
oldcustomg.MustCommit(db)
- return core.CommitGenesisBlock(db, &customg, true)
+ return core.CommitGenesisBlock(db, &customg)
},
wantHash: customghash,
wantConfig: customg.Config,
@@ -209,7 +209,7 @@ func TestSetupGenesis(t *testing.T) {
return nil, nil, err
}
// This should return a compatibility error.
- return core.CommitGenesisBlock(m.DB, &customg, true)
+ return core.CommitGenesisBlock(m.DB, &customg)
},
wantHash: customghash,
wantConfig: customg.Config,
diff --git a/turbo/stages/mock_sentry.go b/turbo/stages/mock_sentry.go
index b1bf864e079a102596ac14631d59a0df7d9b3b43..fad4347eb5d64586304ce4c2463a278022f0acd7 100644
--- a/turbo/stages/mock_sentry.go
+++ b/turbo/stages/mock_sentry.go
@@ -30,6 +30,7 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/ethdb/kv"
+ "github.com/ledgerwatch/erigon/ethdb/prune"
"github.com/ledgerwatch/erigon/ethdb/remote/remotedbserver"
"github.com/ledgerwatch/erigon/log"
"github.com/ledgerwatch/erigon/params"
@@ -129,19 +130,19 @@ func (ms *MockSentry) Messages(req *proto_sentry.MessagesRequest, stream proto_s
}
func MockWithGenesis(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey) *MockSentry {
- return MockWithGenesisStorageMode(t, gspec, key, ethdb.DefaultStorageMode)
+ return MockWithGenesisStorageMode(t, gspec, key, prune.DefaultMode)
}
func MockWithGenesisEngine(t *testing.T, gspec *core.Genesis, engine consensus.Engine) *MockSentry {
key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
- return MockWithEverything(t, gspec, key, ethdb.DefaultStorageMode, engine)
+ return MockWithEverything(t, gspec, key, prune.DefaultMode, engine)
}
-func MockWithGenesisStorageMode(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey, sm ethdb.StorageMode) *MockSentry {
- return MockWithEverything(t, gspec, key, sm, ethash.NewFaker())
+func MockWithGenesisStorageMode(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey, prune prune.Mode) *MockSentry {
+ return MockWithEverything(t, gspec, key, prune, ethash.NewFaker())
}
-func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey, sm ethdb.StorageMode, engine consensus.Engine) *MockSentry {
+func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey, prune prune.Mode, engine consensus.Engine) *MockSentry {
var tmpdir string
if t != nil {
tmpdir = t.TempDir()
@@ -206,7 +207,7 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
mock.TxPoolP2PServer.TxFetcher = fetcher.NewTxFetcher(txPool.Has, txPool.AddRemotes, fetchTx)
// Committed genesis will be shared between download and mock sentry
- _, mock.Genesis, err = core.CommitGenesisBlock(mock.DB, gspec, sm.History)
+ _, mock.Genesis, err = core.CommitGenesisBlock(mock.DB, gspec)
if _, ok := err.(*params.ConfigCompatError); err != nil && !ok {
if t != nil {
t.Fatal(err)
@@ -229,7 +230,7 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
}
mock.Sync = stagedsync.New(
stagedsync.DefaultStages(
- mock.Ctx, sm,
+ mock.Ctx, prune,
stagedsync.StageHeadersCfg(
mock.DB,
mock.downloader.Hd,
@@ -260,15 +261,12 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, mock.tmpdir),
stagedsync.StageExecuteBlocksCfg(
mock.DB,
- sm.Receipts,
- sm.CallTraces,
- sm.TEVM,
- 0,
+ prune,
cfg.BatchSize,
nil,
mock.ChainConfig,
mock.Engine,
- &vm.Config{NoReceipts: !sm.Receipts},
+ &vm.Config{},
nil,
cfg.StateStream,
mock.tmpdir,
@@ -286,10 +284,10 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
),
stagedsync.StageHashStateCfg(mock.DB, mock.tmpdir),
stagedsync.StageTrieCfg(mock.DB, true, true, mock.tmpdir),
- stagedsync.StageHistoryCfg(mock.DB, mock.tmpdir),
- stagedsync.StageLogIndexCfg(mock.DB, mock.tmpdir),
- stagedsync.StageCallTracesCfg(mock.DB, 0, mock.tmpdir),
- stagedsync.StageTxLookupCfg(mock.DB, mock.tmpdir),
+ stagedsync.StageHistoryCfg(mock.DB, prune, mock.tmpdir),
+ stagedsync.StageLogIndexCfg(mock.DB, prune, mock.tmpdir),
+ stagedsync.StageCallTracesCfg(mock.DB, prune, 0, mock.tmpdir),
+ stagedsync.StageTxLookupCfg(mock.DB, prune, mock.tmpdir),
stagedsync.StageTxPoolCfg(mock.DB, txPool, func() {
mock.StreamWg.Add(1)
go txpool.RecvTxMessageLoop(mock.Ctx, mock.SentryClient, mock.downloader, mock.TxPoolP2PServer.HandleInboundMessage, &mock.ReceiveWg)
@@ -413,7 +411,7 @@ func (ms *MockSentry) InsertChain(chain *core.ChainPack) error {
}
ms.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed
initialCycle := false
- highestSeenHeader := uint64(chain.TopBlock.NumberU64())
+ highestSeenHeader := chain.TopBlock.NumberU64()
if err := StageLoopStep(ms.Ctx, ms.DB, ms.Sync, highestSeenHeader, ms.Notifications, initialCycle, ms.UpdateHead, nil); err != nil {
return err
}
diff --git a/turbo/stages/sentry_mock_test.go b/turbo/stages/sentry_mock_test.go
index cf6b3b5683bee68ca3e6f139a84e3ff8a1fcb608..9784ddc53ee01e9b9b7e66c7a966de94379556e1 100644
--- a/turbo/stages/sentry_mock_test.go
+++ b/turbo/stages/sentry_mock_test.go
@@ -64,6 +64,7 @@ func TestHeaderStep(t *testing.T) {
}
func TestMineBlockWith1Tx(t *testing.T) {
+ t.Skip("revive me")
defer log.Root().SetHandler(log.Root().GetHandler())
log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
require, m := require.New(t), stages.Mock(t)
diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go
index d273d0b415fc2df0682bdd68066ee3648d6d40b6..afebc3a702719beb7f92171d618e9d075acda163 100644
--- a/turbo/stages/stageloop.go
+++ b/turbo/stages/stageloop.go
@@ -19,7 +19,6 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/log"
- "github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/turbo/shards"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
"github.com/ledgerwatch/erigon/turbo/stages/headerdownload"
@@ -206,15 +205,10 @@ func NewStagedSync2(
snapshotMigrator *snapshotsync.SnapshotMigrator,
accumulator *shards.Accumulator,
) (*stagedsync.Sync, error) {
- var pruningDistance uint64
- if !cfg.StorageMode.History {
- pruningDistance = params.FullImmutabilityThreshold
- }
-
return stagedsync.New(
stagedsync.DefaultStages(
ctx,
- cfg.StorageMode,
+ cfg.Prune,
stagedsync.StageHeadersCfg(
db,
controlServer.Hd,
@@ -240,15 +234,12 @@ func NewStagedSync2(
stagedsync.StageSendersCfg(db, controlServer.ChainConfig, tmpdir),
stagedsync.StageExecuteBlocksCfg(
db,
- cfg.StorageMode.Receipts,
- cfg.StorageMode.CallTraces,
- cfg.StorageMode.TEVM,
- pruningDistance,
+ cfg.Prune,
cfg.BatchSize,
nil,
controlServer.ChainConfig,
controlServer.Engine,
- &vm.Config{NoReceipts: !cfg.StorageMode.Receipts, EnableTEMV: cfg.StorageMode.TEVM},
+ &vm.Config{EnableTEMV: cfg.Prune.Experiments.TEVM},
accumulator,
cfg.StateStream,
tmpdir,
@@ -261,10 +252,10 @@ func NewStagedSync2(
stagedsync.StageSnapshotStateCfg(db, cfg.Snapshot, tmpdir, client, snapshotMigrator),
stagedsync.StageHashStateCfg(db, tmpdir),
stagedsync.StageTrieCfg(db, true, true, tmpdir),
- stagedsync.StageHistoryCfg(db, tmpdir),
- stagedsync.StageLogIndexCfg(db, tmpdir),
- stagedsync.StageCallTracesCfg(db, 0, tmpdir),
- stagedsync.StageTxLookupCfg(db, tmpdir),
+ stagedsync.StageHistoryCfg(db, cfg.Prune, tmpdir),
+ stagedsync.StageLogIndexCfg(db, cfg.Prune, tmpdir),
+ stagedsync.StageCallTracesCfg(db, cfg.Prune, 0, tmpdir),
+ stagedsync.StageTxLookupCfg(db, cfg.Prune, tmpdir),
stagedsync.StageTxPoolCfg(db, txPool, func() {
for i := range txPoolServer.Sentries {
go func(i int) {
diff --git a/turbo/trie/intermediate_trie_hashes.go b/turbo/trie/intermediate_trie_hashes.go
deleted file mode 100644
index 6536c0b015599d545eb21a4643432715e5642c0d..0000000000000000000000000000000000000000
--- a/turbo/trie/intermediate_trie_hashes.go
+++ /dev/null
@@ -1 +0,0 @@
-package trie