diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
index 9a797d816218b173c15f7a87bb6ea0b92df7ebea..010a2419cece75817de4625bfea6992aa3dadbdb 100644
--- a/cmd/hack/hack.go
+++ b/cmd/hack/hack.go
@@ -15,9 +15,11 @@ import (
_ "net/http/pprof" //nolint:gosec
"os"
"os/signal"
+ "regexp"
"runtime"
"runtime/pprof"
"sort"
+ "strconv"
"strings"
"syscall"
"time"
@@ -2380,6 +2382,79 @@ func junkdb() error {
return nil
}
+func histStats() error {
+ files, err := os.ReadDir(".")
+ if err != nil {
+ return err
+ }
+ endBlockMap := map[uint64]struct{}{}
+ pageMap := map[string]map[uint64]uint64{}
+ keys := []string{"ahistory", "shistory", "abitmap", "sbitmap"}
+ for _, k := range keys {
+ pageMap[k] = map[uint64]uint64{}
+ }
+ re := regexp.MustCompile(`(ahistory|shistory|abitmap|sbitmap).([0-9]+).txt`)
+ for _, f := range files {
+ name := f.Name()
+ subs := re.FindStringSubmatch(name)
+ if len(subs) != 3 {
+ if len(subs) != 0 {
+ log.Warn("File ignored by changes scan, more than 3 submatches", "name", name, "submatches", len(subs))
+ }
+ continue
+ }
+ var endBlock uint64
+ if endBlock, err = strconv.ParseUint(subs[2], 10, 64); err != nil {
+ return err
+ }
+ endBlockMap[endBlock] = struct{}{}
+ var ff *os.File
+ if ff, err = os.Open(name); err != nil {
+ return err
+ }
+ scanner := bufio.NewScanner(ff)
+ // Skip 5 lines
+ for i := 0; i < 5; i++ {
+ scanner.Scan()
+ }
+ var totalPages uint64
+ for i := 0; i < 3; i++ {
+ scanner.Scan()
+ line := scanner.Text()
+ p := strings.Index(line, ": ")
+ var pages uint64
+ if pages, err = strconv.ParseUint(line[p+2:], 10, 64); err != nil {
+ return err
+ }
+ totalPages += pages
+ }
+ pageMap[subs[1]][endBlock] = totalPages
+ ff.Close()
+ }
+ var endBlocks []uint64
+ for endBlock := range endBlockMap {
+ endBlocks = append(endBlocks, endBlock)
+ }
+ sort.Slice(endBlocks, func(i, j int) bool {
+ return endBlocks[i] < endBlocks[j]
+ })
+ var lastEndBlock uint64
+ fmt.Printf("endBlock,%s\n", strings.Join(keys, ","))
+ for _, endBlock := range endBlocks {
+ fmt.Printf("%d", endBlock)
+ for _, k := range keys {
+ if lastEndBlock == 0 {
+ fmt.Printf(",%.3f", float64(pageMap[k][endBlock])/256.0/1024.0)
+ } else {
+ fmt.Printf(",%.3f", float64(pageMap[k][endBlock]-pageMap[k][lastEndBlock])/256.0/1024.0)
+ }
+ }
+ fmt.Printf("\n")
+ lastEndBlock = endBlock
+ }
+ return nil
+}
+
func main() {
debug.RaiseFdLimit()
flag.Parse()
@@ -2553,6 +2628,8 @@ func main() {
err = mainnetGenesis()
case "junkdb":
err = junkdb()
+ case "histStats":
+ err = histStats()
}
if err != nil {
diff --git a/cmd/state/commands/check_change_sets.go b/cmd/state/commands/check_change_sets.go
index 38713577e6a5e6f9c28ac5bde91c72914d6aae7e..199e7e778ef7da811abfa22fb94c6cdd4f35a54f 100644
--- a/cmd/state/commands/check_change_sets.go
+++ b/cmd/state/commands/check_change_sets.go
@@ -120,17 +120,19 @@ func CheckChangeSets(genesis *core.Genesis, logger log.Logger, blockNum uint64,
if err != nil {
return err
}
- var block *types.Block
- block, _, err = rawdb.ReadBlockWithSenders(rwtx, blockHash, blockNum)
+ var b *types.Block
+ b, _, err = rawdb.ReadBlockWithSenders(rwtx, blockHash, blockNum)
if err != nil {
return err
}
- if block == nil {
+ if b == nil {
break
}
- intraBlockState := state.New(state.NewPlainState(historyTx, block.NumberU64()-1))
- csw := state.NewChangeSetWriterPlain(nil /* db */, block.NumberU64()-1)
+ reader := state.NewPlainState(historyTx, blockNum-1)
+ reader.SetTrace(blockNum == uint64(block))
+ intraBlockState := state.New(reader)
+ csw := state.NewChangeSetWriterPlain(nil /* db */, blockNum-1)
var blockWriter state.StateWriter
if nocheck {
blockWriter = noOpWriter
@@ -140,19 +142,19 @@ func CheckChangeSets(genesis *core.Genesis, logger log.Logger, blockNum uint64,
getHeader := func(hash common.Hash, number uint64) *types.Header { return rawdb.ReadHeader(rwtx, hash, number) }
contractHasTEVM := ethdb.GetHasTEVM(rwtx)
- receipts, err1 := runBlock(intraBlockState, noOpWriter, blockWriter, chainConfig, getHeader, contractHasTEVM, block, vmConfig)
+ receipts, err1 := runBlock(intraBlockState, noOpWriter, blockWriter, chainConfig, getHeader, contractHasTEVM, b, vmConfig)
if err1 != nil {
return err1
}
if writeReceipts {
- if chainConfig.IsByzantium(block.NumberU64()) {
+ if chainConfig.IsByzantium(blockNum) {
receiptSha := types.DeriveSha(receipts)
- if receiptSha != block.ReceiptHash() {
- return fmt.Errorf("mismatched receipt headers for block %d", block.NumberU64())
+ if receiptSha != b.ReceiptHash() {
+ return fmt.Errorf("mismatched receipt headers for block %d", blockNum)
}
}
- if err := rawdb.AppendReceipts(rwtx, block.NumberU64(), receipts); err != nil {
+ if err := rawdb.AppendReceipts(rwtx, blockNum, receipts); err != nil {
return err
}
}
@@ -226,7 +228,7 @@ func CheckChangeSets(genesis *core.Genesis, logger log.Logger, blockNum uint64,
fmt.Println("interrupted, please wait for cleanup...")
case <-commitEvery.C:
if writeReceipts {
- log.Info("Committing receipts", "up to block", block.NumberU64())
+ log.Info("Committing receipts", "up to block", b.NumberU64())
if err = rwtx.Commit(); err != nil {
return err
}
diff --git a/cmd/state/commands/history2.go b/cmd/state/commands/history2.go
index 88e93db01f022a7c63bace089810b822b9311308..7ded3435d73b704bf66850d7f4cbf0fcb3cc8018 100644
--- a/cmd/state/commands/history2.go
+++ b/cmd/state/commands/history2.go
@@ -26,9 +26,14 @@ import (
"github.com/spf13/cobra"
)
+var (
+ traceBlock int
+)
+
func init() {
withBlock(history2Cmd)
withDatadir(history2Cmd)
+ history2Cmd.Flags().IntVar(&traceBlock, "traceblock", 0, "block number at which to turn on tracing")
rootCmd.AddCommand(history2Cmd)
}
@@ -62,7 +67,7 @@ func History2(genesis *core.Genesis, logger log.Logger) error {
}
defer historyTx.Rollback()
aggPath := filepath.Join(datadir, "aggregator")
- h, err3 := aggregator.NewHistory(aggPath, 499_999, aggregationStep)
+ h, err3 := aggregator.NewHistory(aggPath, block, aggregationStep)
if err3 != nil {
return fmt.Errorf("create history: %w", err3)
}
@@ -71,7 +76,7 @@ func History2(genesis *core.Genesis, logger log.Logger) error {
vmConfig := vm.Config{}
interrupt := false
- blockNum := block
+ blockNum := uint64(0)
var txNum uint64 = 1
trace := false
logEvery := time.NewTicker(logInterval)
@@ -104,7 +109,9 @@ func History2(genesis *core.Genesis, logger log.Logger) error {
}
r := h.MakeHistoryReader()
readWrapper := &HistoryWrapper{r: r}
- //readWrapper.trace = blockNum == 999_999
+ if traceBlock != 0 {
+ readWrapper.trace = blockNum == uint64(traceBlock)
+ }
writeWrapper := state.NewNoopWriter()
getHeader := func(hash common.Hash, number uint64) *types.Header { return rawdb.ReadHeader(historyTx, hash, number) }
if txNum, _, err = runHistory2(trace, blockNum, txNum, readWrapper, writeWrapper, chainConfig, getHeader, b, vmConfig); err != nil {
@@ -138,11 +145,13 @@ func runHistory2(trace bool, blockNum, txNumStart uint64, hw *HistoryWrapper, ww
daoBlock = false
}
ibs.Prepare(tx.Hash(), block.Hash(), i)
-
receipt, _, err := core.ApplyTransaction(chainConfig, getHeader, engine, nil, gp, ibs, ww, header, tx, usedGas, vmConfig, nil)
if err != nil {
return 0, nil, fmt.Errorf("could not apply tx %d [%x] failed: %w", i, tx.Hash(), err)
}
+ if traceBlock != 0 && blockNum == uint64(traceBlock) {
+ fmt.Printf("tx idx %d, num %d, gas used %d\n", i, txNum, receipt.GasUsed)
+ }
receipts = append(receipts, receipt)
txNum++
}
diff --git a/core/state/plain_readonly.go b/core/state/plain_readonly.go
index a75c79503c3b0208cc7c3433bafb7900a45683cf..dc076b8867454b3ffe41422b0eb6424d4eb8923e 100644
--- a/core/state/plain_readonly.go
+++ b/core/state/plain_readonly.go
@@ -19,6 +19,7 @@ package state
import (
"bytes"
"encoding/binary"
+ "fmt"
"github.com/google/btree"
"github.com/holiman/uint256"
@@ -45,6 +46,7 @@ type PlainState struct {
tx kv.Tx
blockNr uint64
storage map[common.Address]*btree.BTree
+ trace bool
}
func NewPlainState(tx kv.Tx, blockNr uint64) *PlainState {
@@ -61,6 +63,10 @@ func NewPlainState(tx kv.Tx, blockNr uint64) *PlainState {
}
}
+func (s *PlainState) SetTrace(trace bool) {
+ s.trace = trace
+}
+
func (s *PlainState) SetBlockNr(blockNr uint64) {
s.blockNr = blockNr
}
@@ -175,6 +181,9 @@ func (s *PlainState) ReadAccountStorage(address common.Address, incarnation uint
if err != nil {
return nil, err
}
+ if s.trace {
+ fmt.Printf("ReadAccountStorage [%x] [%x] => [%x]\n", address, *key, enc)
+ }
if len(enc) == 0 {
return nil, nil
}
diff --git a/go.mod b/go.mod
index adb4374c622cd16b991376b208632a0f82103277..20f570ad526b4e43852dcc863962a58296d87ee8 100644
--- a/go.mod
+++ b/go.mod
@@ -40,7 +40,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
- github.com/ledgerwatch/erigon-lib v0.0.0-20220212125515-3442f42457cb
+ github.com/ledgerwatch/erigon-lib v0.0.0-20220212145123-d9367b582a21
github.com/ledgerwatch/log/v3 v3.4.0
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
diff --git a/go.sum b/go.sum
index 1025344a9e6c60d9607db5709ed9790a022eab09..c130a49d8eb4d587255f1e3d6fc08723ccc9e39a 100644
--- a/go.sum
+++ b/go.sum
@@ -632,8 +632,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220212125515-3442f42457cb h1:kzS/0e5EHNBeFGybHt2zboOlDeoPydFfjQqBvGhCt8c=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220212125515-3442f42457cb/go.mod h1:phuzMr8tLvqjo5cQVA9jj8odAso6eLyS4LFmUJrDFGw=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220212145123-d9367b582a21 h1:W0u7HzNxrwtSNISHac902xrJlT2O66gkuaBQdgNfHZI=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220212145123-d9367b582a21/go.mod h1:phuzMr8tLvqjo5cQVA9jj8odAso6eLyS4LFmUJrDFGw=
github.com/ledgerwatch/log/v3 v3.4.0 h1:SEIOcv5a2zkG3PmoT5jeTU9m/0nEUv0BJS5bzsjwKCI=
github.com/ledgerwatch/log/v3 v3.4.0/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=