From eb6a16860325891bbcdf9315840bdcd75746fa00 Mon Sep 17 00:00:00 2001
From: ledgerwatch <akhounov@gmail.com>
Date: Sat, 12 Feb 2022 00:06:57 +0000
Subject: [PATCH] [erigon2] Chain history and bitmap indices (#3437)
* First update
* Update to latest erigon-lib
* Fix lint
* Update
* Update
* Update
* Add history2 command
* Updates
* Update erigon-lib
Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>
Co-authored-by: Alex Sharp <alexsharp@Alexs-MacBook-Pro.local>
---
cmd/state/commands/erigon2.go | 17 ++-
cmd/state/commands/history2.go | 220 +++++++++++++++++++++++++++++++++
go.mod | 2 +-
go.sum | 4 +-
4 files changed, 235 insertions(+), 8 deletions(-)
create mode 100644 cmd/state/commands/history2.go
diff --git a/cmd/state/commands/erigon2.go b/cmd/state/commands/erigon2.go
index 9440169553..6d637f3a74 100644
--- a/cmd/state/commands/erigon2.go
+++ b/cmd/state/commands/erigon2.go
@@ -38,6 +38,7 @@ const (
var (
commitmentFrequency int // How many blocks to skip between calculating commitment
changesets bool
+ commitments bool
)
func init() {
@@ -45,6 +46,7 @@ func init() {
withDatadir(erigon2Cmd)
erigon2Cmd.Flags().BoolVar(&changesets, "changesets", false, "set to true to generate changesets")
erigon2Cmd.Flags().IntVar(&commitmentFrequency, "commfreq", 625, "how many blocks to skip between calculating commitment")
+ erigon2Cmd.Flags().BoolVar(&commitments, "commitments", false, "set to true to calculate commitments")
rootCmd.AddCommand(erigon2Cmd)
}
@@ -103,6 +105,7 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
}
defer agg.Close()
agg.GenerateChangesets(changesets)
+ agg.Commitments(commitments)
chainConfig := genesis.Config
vmConfig := vm.Config{}
@@ -124,14 +127,18 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
if err = w.FinishTx(0, false); err != nil {
return err
}
- if rootHash, err = w.ComputeCommitment(false); err != nil {
- return err
+ if commitments {
+ if rootHash, err = w.ComputeCommitment(false); err != nil {
+ return err
+ }
}
if err = w.Aggregate(false); err != nil {
return err
}
- if !bytes.Equal(rootHash, genBlock.Header().Root[:]) {
- return fmt.Errorf("root hash mismatch for genesis block, expected [%x], was [%x]", genBlock.Header().Root[:], rootHash)
+ if commitments {
+ if !bytes.Equal(rootHash, genBlock.Header().Root[:]) {
+ return fmt.Errorf("root hash mismatch for genesis block, expected [%x], was [%x]", genBlock.Header().Root[:], rootHash)
+ }
}
}
var txNum uint64 = 1
@@ -197,7 +204,7 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
log.Info(fmt.Sprintf("interrupted, please wait for cleanup, next time start with --block %d", blockNum))
default:
}
- if interrupt || blockNum%uint64(commitmentFrequency) == 0 {
+ if commitments && (interrupt || blockNum%uint64(commitmentFrequency) == 0) {
if rootHash, err = w.ComputeCommitment(trace /* trace */); err != nil {
return err
}
diff --git a/cmd/state/commands/history2.go b/cmd/state/commands/history2.go
new file mode 100644
index 0000000000..dae5b5f4f5
--- /dev/null
+++ b/cmd/state/commands/history2.go
@@ -0,0 +1,220 @@
+package commands
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/signal"
+ "path"
+ "syscall"
+ "time"
+
+ "github.com/ledgerwatch/erigon-lib/aggregator"
+ kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx"
+ "github.com/ledgerwatch/erigon/common"
+ "github.com/ledgerwatch/erigon/consensus/ethash"
+ "github.com/ledgerwatch/erigon/consensus/misc"
+ "github.com/ledgerwatch/erigon/core"
+ "github.com/ledgerwatch/erigon/core/rawdb"
+ "github.com/ledgerwatch/erigon/core/state"
+ "github.com/ledgerwatch/erigon/core/types"
+ "github.com/ledgerwatch/erigon/core/types/accounts"
+ "github.com/ledgerwatch/erigon/core/vm"
+ "github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/log/v3"
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ withBlock(history2Cmd)
+ withDatadir(history2Cmd)
+ rootCmd.AddCommand(history2Cmd)
+}
+
+var history2Cmd = &cobra.Command{
+ Use: "history2",
+ Short: "Exerimental command to re-execute historical transactions in erigon2 format",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ logger := log.New()
+ return History2(genesis, logger)
+ },
+}
+
+func History2(genesis *core.Genesis, logger log.Logger) error {
+ sigs := make(chan os.Signal, 1)
+ interruptCh := make(chan bool, 1)
+ signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+ go func() {
+ <-sigs
+ interruptCh <- true
+ }()
+ historyDb, err := kv2.NewMDBX(logger).Path(path.Join(datadir, "chaindata")).Open()
+ if err != nil {
+ return fmt.Errorf("opening chaindata as read only: %v", err)
+ }
+ defer historyDb.Close()
+ ctx := context.Background()
+ historyTx, err1 := historyDb.BeginRo(ctx)
+ if err1 != nil {
+ return err1
+ }
+ defer historyTx.Rollback()
+ aggPath := path.Join(datadir, "aggregator")
+ h, err3 := aggregator.NewHistory(aggPath, 499_999, aggregationStep)
+ if err3 != nil {
+ return fmt.Errorf("create history: %w", err3)
+ }
+ defer h.Close()
+ chainConfig := genesis.Config
+ vmConfig := vm.Config{}
+
+ interrupt := false
+ blockNum := block
+ var txNum uint64 = 1
+ trace := false
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+ prevBlock := blockNum
+ prevTime := time.Now()
+ for !interrupt {
+ select {
+ default:
+ case <-logEvery.C:
+ currentTime := time.Now()
+ interval := currentTime.Sub(prevTime)
+ speed := float64(blockNum-prevBlock) / (float64(interval) / float64(time.Second))
+ prevBlock = blockNum
+ prevTime = currentTime
+ log.Info("Progress", "block", blockNum, "blk/s", speed)
+ }
+ blockNum++
+ blockHash, err := rawdb.ReadCanonicalHash(historyTx, blockNum)
+ if err != nil {
+ return err
+ }
+ var b *types.Block
+ b, _, err = rawdb.ReadBlockWithSenders(historyTx, blockHash, blockNum)
+ if err != nil {
+ return err
+ }
+ if b == nil {
+ break
+ }
+ r := h.MakeHistoryReader()
+ readWrapper := &HistoryWrapper{r: r}
+ //readWrapper.trace = blockNum == 999_999
+ writeWrapper := state.NewNoopWriter()
+ getHeader := func(hash common.Hash, number uint64) *types.Header { return rawdb.ReadHeader(historyTx, hash, number) }
+ if txNum, _, err = runHistory2(trace, blockNum, txNum, readWrapper, writeWrapper, chainConfig, getHeader, b, vmConfig); err != nil {
+ return fmt.Errorf("block %d: %w", blockNum, err)
+ }
+ txNum++
+ // Check for interrupts
+ select {
+ case interrupt = <-interruptCh:
+ log.Info(fmt.Sprintf("interrupted, please wait for cleanup, next time start with --block %d", blockNum))
+ default:
+ }
+ }
+ return nil
+}
+
+func runHistory2(trace bool, blockNum, txNumStart uint64, hw *HistoryWrapper, ww state.StateWriter, chainConfig *params.ChainConfig, getHeader func(hash common.Hash, number uint64) *types.Header, block *types.Block, vmConfig vm.Config) (uint64, types.Receipts, error) {
+ header := block.Header()
+ vmConfig.TraceJumpDest = true
+ engine := ethash.NewFullFaker()
+ gp := new(core.GasPool).AddGas(block.GasLimit())
+ usedGas := new(uint64)
+ var receipts types.Receipts
+ daoBlock := chainConfig.DAOForkSupport && chainConfig.DAOForkBlock != nil && chainConfig.DAOForkBlock.Cmp(block.Number()) == 0
+ txNum := txNumStart
+ for i, tx := range block.Transactions() {
+ hw.r.SetNums(blockNum, txNum, false)
+ ibs := state.New(hw)
+ if daoBlock {
+ misc.ApplyDAOHardFork(ibs)
+ daoBlock = false
+ }
+ ibs.Prepare(tx.Hash(), block.Hash(), i)
+
+ receipt, _, err := core.ApplyTransaction(chainConfig, getHeader, engine, nil, gp, ibs, ww, header, tx, usedGas, vmConfig, nil)
+ if err != nil {
+ return 0, nil, fmt.Errorf("could not apply tx %d [%x] failed: %w", i, tx.Hash(), err)
+ }
+ receipts = append(receipts, receipt)
+ txNum++
+ }
+
+ return txNum, receipts, nil
+}
+
+// Implements StateReader and StateWriter
+type HistoryWrapper struct {
+ r *aggregator.HistoryReader
+ trace bool
+}
+
+func (hw *HistoryWrapper) ReadAccountData(address common.Address) (*accounts.Account, error) {
+ enc, err := hw.r.ReadAccountData(address.Bytes(), hw.trace)
+ if err != nil {
+ return nil, err
+ }
+ if len(enc) == 0 {
+ return nil, nil
+ }
+ var a accounts.Account
+ a.Reset()
+ pos := 0
+ nonceBytes := int(enc[pos])
+ pos++
+ if nonceBytes > 0 {
+ a.Nonce = bytesToUint64(enc[pos : pos+nonceBytes])
+ pos += nonceBytes
+ }
+ balanceBytes := int(enc[pos])
+ pos++
+ if balanceBytes > 0 {
+ a.Balance.SetBytes(enc[pos : pos+balanceBytes])
+ pos += balanceBytes
+ }
+ codeHashBytes := int(enc[pos])
+ pos++
+ if codeHashBytes > 0 {
+ copy(a.CodeHash[:], enc[pos:pos+codeHashBytes])
+ pos += codeHashBytes
+ }
+ incBytes := int(enc[pos])
+ pos++
+ if incBytes > 0 {
+ a.Incarnation = bytesToUint64(enc[pos : pos+incBytes])
+ }
+ return &a, nil
+}
+
+func (hw *HistoryWrapper) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) {
+ enc, err := hw.r.ReadAccountStorage(address.Bytes(), key.Bytes(), hw.trace)
+ if hw.trace {
+ fmt.Printf("ReadAccountStorage [%x] [%x] => [%x]\n", address, key.Bytes(), enc)
+ }
+ if err != nil {
+ fmt.Printf("%v\n", err)
+ return nil, err
+ }
+ if enc == nil {
+ return nil, nil
+ }
+ return enc.Bytes(), nil
+}
+
+func (hw *HistoryWrapper) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) {
+ return hw.r.ReadAccountCode(address.Bytes(), false /* trace */)
+}
+
+func (hw *HistoryWrapper) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) {
+ return hw.r.ReadAccountCodeSize(address.Bytes(), false /* trace */)
+}
+
+func (hw *HistoryWrapper) ReadAccountIncarnation(address common.Address) (uint64, error) {
+ return 0, nil
+}
diff --git a/go.mod b/go.mod
index 1a87a45689..b1ff62ea3b 100644
--- a/go.mod
+++ b/go.mod
@@ -40,7 +40,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
- github.com/ledgerwatch/erigon-lib v0.0.0-20220211090027-9689e7046626
+ github.com/ledgerwatch/erigon-lib v0.0.0-20220211234430-5f7283d0f1f1
github.com/ledgerwatch/log/v3 v3.4.0
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
diff --git a/go.sum b/go.sum
index 3237beaf4b..dcc4a3b9ec 100644
--- a/go.sum
+++ b/go.sum
@@ -632,8 +632,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220211090027-9689e7046626 h1:cJGqT2RxriqihCXjpLfUc98vBEoqy7HzLxKI4hD7gKA=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220211090027-9689e7046626/go.mod h1:NzHpr5GGDbLM/hYSU4BWB0INU6ZALzMzaqX4i03R6i0=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220211234430-5f7283d0f1f1 h1:VMkjKgWg5eDcB+lntInPIi8c9zdSkCu0g6ZDQBRT4FI=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220211234430-5f7283d0f1f1/go.mod h1:NzHpr5GGDbLM/hYSU4BWB0INU6ZALzMzaqX4i03R6i0=
github.com/ledgerwatch/log/v3 v3.4.0 h1:SEIOcv5a2zkG3PmoT5jeTU9m/0nEUv0BJS5bzsjwKCI=
github.com/ledgerwatch/log/v3 v3.4.0/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
--
GitLab