From dffcc57ecc7fa64ea207cc9075a29e8874669327 Mon Sep 17 00:00:00 2001
From: ledgerwatch <akhounov@gmail.com>
Date: Sun, 13 Feb 2022 20:02:18 +0000
Subject: [PATCH] [erigon2] Fix history bug, optimise bitmap search (#3510)
* Fix history bug, optimise bitmap search
* Optimise SeekInBitmap64
* Update erigon-lib
Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>
---
cmd/state/commands/history2.go | 14 ++++++++++++--
ethdb/bitmapdb/dbutils.go | 11 +++++++++--
go.mod | 2 +-
go.sum | 4 ++--
4 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/cmd/state/commands/history2.go b/cmd/state/commands/history2.go
index 7ded3435d7..1a4fd8516f 100644
--- a/cmd/state/commands/history2.go
+++ b/cmd/state/commands/history2.go
@@ -27,6 +27,7 @@ import (
)
var (
+ blockTo int
traceBlock int
)
@@ -34,6 +35,7 @@ func init() {
withBlock(history2Cmd)
withDatadir(history2Cmd)
history2Cmd.Flags().IntVar(&traceBlock, "traceblock", 0, "block number at which to turn on tracing")
+ history2Cmd.Flags().IntVar(&blockTo, "blockto", 0, "block number to stop replay of history at")
rootCmd.AddCommand(history2Cmd)
}
@@ -67,7 +69,7 @@ func History2(genesis *core.Genesis, logger log.Logger) error {
}
defer historyTx.Rollback()
aggPath := filepath.Join(datadir, "aggregator")
- h, err3 := aggregator.NewHistory(aggPath, block, aggregationStep)
+ h, err3 := aggregator.NewHistory(aggPath, uint64(blockTo), aggregationStep)
if err3 != nil {
return fmt.Errorf("create history: %w", err3)
}
@@ -95,6 +97,9 @@ func History2(genesis *core.Genesis, logger log.Logger) error {
log.Info("Progress", "block", blockNum, "blk/s", speed)
}
blockNum++
+ if blockNum > uint64(blockTo) {
+ break
+ }
blockHash, err := rawdb.ReadCanonicalHash(historyTx, blockNum)
if err != nil {
return err
@@ -107,6 +112,11 @@ func History2(genesis *core.Genesis, logger log.Logger) error {
if b == nil {
break
}
+ if blockNum < block {
+ // Skip that block, but increase txNum
+ txNum += uint64(len(b.Transactions())) + 1
+ continue
+ }
r := h.MakeHistoryReader()
readWrapper := &HistoryWrapper{r: r}
if traceBlock != 0 {
@@ -205,7 +215,7 @@ func (hw *HistoryWrapper) ReadAccountData(address common.Address) (*accounts.Acc
func (hw *HistoryWrapper) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) {
enc, err := hw.r.ReadAccountStorage(address.Bytes(), key.Bytes(), hw.trace)
if hw.trace {
- fmt.Printf("ReadAccountStorage [%x] [%x] => [%x]\n", address, key.Bytes(), enc)
+ fmt.Printf("ReadAccountStorage [%x] [%x] => [%x]\n", address, key.Bytes(), enc.Bytes())
}
if err != nil {
fmt.Printf("%v\n", err)
diff --git a/ethdb/bitmapdb/dbutils.go b/ethdb/bitmapdb/dbutils.go
index 7ecdd84688..b8810d8ebe 100644
--- a/ethdb/bitmapdb/dbutils.go
+++ b/ethdb/bitmapdb/dbutils.go
@@ -311,9 +311,16 @@ func Get64(db kv.Tx, bucket string, key []byte, from, to uint64) (*roaring64.Bit
// SeekInBitmap - returns value in bitmap which is >= n
func SeekInBitmap64(m *roaring64.Bitmap, n uint64) (found uint64, ok bool) {
- m.RemoveRange(0, n)
if m.IsEmpty() {
return 0, false
}
- return m.Minimum(), true
+ if n == 0 {
+ return m.Minimum(), true
+ }
+ searchRank := m.Rank(n - 1)
+ if searchRank >= m.GetCardinality() {
+ return 0, false
+ }
+ found, _ = m.Select(searchRank)
+ return found, true
}
diff --git a/go.mod b/go.mod
index 8df7a5b478..e64528b18a 100644
--- a/go.mod
+++ b/go.mod
@@ -40,7 +40,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
- github.com/ledgerwatch/erigon-lib v0.0.0-20220213022242-61c78273b7d0
+ github.com/ledgerwatch/erigon-lib v0.0.0-20220213185636-136385995894
github.com/ledgerwatch/log/v3 v3.4.0
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
diff --git a/go.sum b/go.sum
index 525c342aa3..e5ba9019cf 100644
--- a/go.sum
+++ b/go.sum
@@ -632,8 +632,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220213022242-61c78273b7d0 h1:oYSSMbr3qAU+b5R3yJiD47gVJ+Gl23oO46aRGVg7w8M=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220213022242-61c78273b7d0/go.mod h1:phuzMr8tLvqjo5cQVA9jj8odAso6eLyS4LFmUJrDFGw=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220213185636-136385995894 h1:YLRLN/rmjQ3fLMtgna1iVqAQbZVIz+G07G3jXHc+lpU=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220213185636-136385995894/go.mod h1:phuzMr8tLvqjo5cQVA9jj8odAso6eLyS4LFmUJrDFGw=
github.com/ledgerwatch/log/v3 v3.4.0 h1:SEIOcv5a2zkG3PmoT5jeTU9m/0nEUv0BJS5bzsjwKCI=
github.com/ledgerwatch/log/v3 v3.4.0/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
--
GitLab