From fb08514b9b30828d969d75bb4fe70b1cc60d4088 Mon Sep 17 00:00:00 2001
From: Igor Mandrigin <mandrigin@users.noreply.github.com>
Date: Sun, 12 Apr 2020 18:41:06 +0300
Subject: [PATCH] Use `geth export` generated files in the stateless prototype
(#447)
---
cmd/state/commands/stateless.go | 5 +-
cmd/state/stateless/stateless.go | 39 +++--
.../stateless/stateless_block_providers.go | 134 ++++++++++++++++++
3 files changed, 162 insertions(+), 16 deletions(-)
create mode 100644 cmd/state/stateless/stateless_block_providers.go
diff --git a/cmd/state/commands/stateless.go b/cmd/state/commands/stateless.go
index 5b9d73a1cb..55b8feb53f 100644
--- a/cmd/state/commands/stateless.go
+++ b/cmd/state/commands/stateless.go
@@ -20,13 +20,14 @@ var (
statelessResolver bool
witnessDatabase string
writeHistory bool
+ blockSource string
)
func init() {
- withChaindata(statelessCmd)
withStatsfile(statelessCmd)
withBlock(statelessCmd)
+ statelessCmd.Flags().StringVar(&blockSource, "blockSource", "", "Path to the block source: `db:///path/to/chaindata` or `exportfile:///path/to/my/exportfile`")
statelessCmd.Flags().StringVar(&statefile, "statefile", "state", "path to the file where the state will be periodically written during the analysis")
statelessCmd.Flags().Uint32Var(&triesize, "triesize", 4*1024*1024, "maximum size of a trie in bytes")
statelessCmd.Flags().BoolVar(&preroot, "preroot", false, "Attempt to compute hash of the trie without modifying it")
@@ -61,7 +62,7 @@ var statelessCmd = &cobra.Command{
stateless.Stateless(
ctx,
block,
- chaindata,
+ blockSource,
statefile,
triesize,
preroot,
diff --git a/cmd/state/stateless/stateless.go b/cmd/state/stateless/stateless.go
index 82315938ee..bb6260312f 100644
--- a/cmd/state/stateless/stateless.go
+++ b/cmd/state/stateless/stateless.go
@@ -133,7 +133,7 @@ type CreateDbFunc func(string) (ethdb.Database, error)
func Stateless(
ctx context.Context,
blockNum uint64,
- chaindata string,
+ blockSourceURI string,
statefile string,
triesize uint32,
tryPreRoot bool,
@@ -168,19 +168,14 @@ func Stateless(
defer timeF.Close()
fmt.Fprintf(timeF, "blockNr,exec,resolve,stateless_exec,calc_root,mod_root\n")
- ethDb, err := createDb(chaindata)
- check(err)
- defer ethDb.Close()
- chainConfig := params.MainnetChainConfig
-
stats, err := NewStatsFile(statsfile)
check(err)
defer stats.Close()
- vmConfig := vm.Config{}
- engine := ethash.NewFullFaker()
- bcb, err := core.NewBlockChain(ethDb, nil, chainConfig, engine, vm.Config{}, nil)
+ blockProvider, err := BlockProviderForURI(blockSourceURI, createDb)
check(err)
+ defer blockProvider.Close()
+
stateDb, err := createDb(statefile)
check(err)
defer stateDb.Close()
@@ -196,8 +191,16 @@ func Stateless(
genesisBlock, _, _, err1 := core.DefaultGenesisBlock().ToBlock(nil, writeHistory)
check(err1)
preRoot = genesisBlock.Header().Root
- } else {
- block := bcb.GetBlockByNumber(blockNum - 1)
+ }
+
+ chainConfig := params.MainnetChainConfig
+ vmConfig := vm.Config{}
+ engine := ethash.NewFullFaker()
+ bcb2, err := core.NewBlockChain(stateDb, nil, chainConfig, engine, vm.Config{}, nil)
+ check(err)
+
+ if blockNum > 1 {
+ block := bcb2.GetBlockByNumber(blockNum - 1)
fmt.Printf("Block number: %d\n", blockNum-1)
fmt.Printf("Block root hash: %x\n", block.Root())
preRoot = block.Root()
@@ -260,6 +263,9 @@ func Stateless(
}
+ err = blockProvider.FastFwd(blockNum)
+ check(err)
+
for !interrupt {
select {
case <-ctx.Done():
@@ -269,10 +275,14 @@ func Stateless(
trace := blockNum == 50492 // false // blockNum == 545080
tds.SetResolveReads(blockNum >= witnessThreshold)
- block := bcb.GetBlockByNumber(blockNum)
+ block, err := blockProvider.NextBlock()
+ check(err)
if block == nil {
break
}
+ if block.NumberU64() != blockNum {
+ check(fmt.Errorf("block number mismatch (want=%v got=%v)", blockNum, block.NumberU64()))
+ }
execStart := time.Now()
statedb := state.New(tds)
gp := new(core.GasPool).AddGas(block.GasLimit())
@@ -285,7 +295,8 @@ func Stateless(
}
for i, tx := range block.Transactions() {
statedb.Prepare(tx.Hash(), block.Hash(), i)
- receipt, err := core.ApplyTransaction(chainConfig, bcb, nil, gp, statedb, tds.TrieStateWriter(), header, tx, usedGas, vmConfig)
+ var receipt *types.Receipt
+ receipt, err = core.ApplyTransaction(chainConfig, bcb2, nil, gp, statedb, tds.TrieStateWriter(), header, tx, usedGas, vmConfig)
if err != nil {
fmt.Printf("tx %x failed: %v\n", tx.Hash(), err)
return
@@ -382,7 +393,7 @@ func Stateless(
ibs := state.New(s)
ibs.SetTrace(trace)
s.SetBlockNr(blockNum)
- if err = runBlock(ibs, s, s, chainConfig, bcb, block); err != nil {
+ if err = runBlock(ibs, s, s, chainConfig, bcb2, block); err != nil {
fmt.Printf("Error running block %d through stateless2: %v\n", blockNum, err)
finalRootFail = true
} else if !binary {
diff --git a/cmd/state/stateless/stateless_block_providers.go b/cmd/state/stateless/stateless_block_providers.go
new file mode 100644
index 0000000000..6d60f37803
--- /dev/null
+++ b/cmd/state/stateless/stateless_block_providers.go
@@ -0,0 +1,134 @@
+package stateless
+
+import (
+ "compress/gzip"
+ "fmt"
+ "io"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/ledgerwatch/turbo-geth/consensus/ethash"
+ "github.com/ledgerwatch/turbo-geth/core"
+ "github.com/ledgerwatch/turbo-geth/core/types"
+ "github.com/ledgerwatch/turbo-geth/core/vm"
+ "github.com/ledgerwatch/turbo-geth/ethdb"
+ "github.com/ledgerwatch/turbo-geth/params"
+ "github.com/ledgerwatch/turbo-geth/rlp"
+)
+
+const (
+ fileSchemeExportfile = "exportfile"
+ fileSchemeDb = "db"
+)
+
+type BlockProvider interface {
+ io.Closer
+ FastFwd(uint64) error
+ NextBlock() (*types.Block, error)
+}
+
+func BlockProviderForURI(uri string, createDbFunc CreateDbFunc) (BlockProvider, error) {
+ url, err := url.Parse(uri)
+ if err != nil {
+ return nil, err
+ }
+ switch url.Scheme {
+ case fileSchemeExportfile:
+ fmt.Println("Source of blocks: export file @", url.Path)
+ return NewBlockProviderFromExportFile(url.Path)
+ case fileSchemeDb:
+ fallthrough
+ default:
+ fmt.Println("Source of blocks: db @", url.Path)
+ return NewBlockProviderFromDb(url.Path, createDbFunc)
+ }
+}
+
+type BlockChainBlockProvider struct {
+ currentBlock uint64
+ bc *core.BlockChain
+ db ethdb.Database
+}
+
+func NewBlockProviderFromDb(path string, createDbFunc CreateDbFunc) (BlockProvider, error) {
+ ethDb, err := createDbFunc(path)
+ if err != nil {
+ return nil, err
+ }
+ chainConfig := params.MainnetChainConfig
+ engine := ethash.NewFullFaker()
+ chain, err := core.NewBlockChain(ethDb, nil, chainConfig, engine, vm.Config{}, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ return &BlockChainBlockProvider{
+ bc: chain,
+ }, nil
+}
+
+func (p *BlockChainBlockProvider) Close() error {
+ p.db.Close()
+ return nil
+}
+
+func (p *BlockChainBlockProvider) FastFwd(to uint64) error {
+ p.currentBlock = to
+ return nil
+}
+
+func (p *BlockChainBlockProvider) NextBlock() (*types.Block, error) {
+ block := p.bc.GetBlockByNumber(p.currentBlock)
+ p.currentBlock++
+ return block, nil
+}
+
+type ExportFileBlockProvider struct {
+ stream *rlp.Stream
+ fh io.Closer
+}
+
+func NewBlockProviderFromExportFile(fn string) (BlockProvider, error) {
+ // Open the file handle and potentially unwrap the gzip stream
+ fh, err := os.Open(fn)
+ if err != nil {
+ return nil, err
+ }
+
+ var reader io.Reader = fh
+ if strings.HasSuffix(fn, ".gz") {
+ if reader, err = gzip.NewReader(reader); err != nil {
+ return nil, err
+ }
+ }
+ stream := rlp.NewStream(reader, 0)
+ return &ExportFileBlockProvider{stream, fh}, nil
+}
+
+func (p *ExportFileBlockProvider) Close() error {
+ return p.fh.Close()
+}
+
+func (p *ExportFileBlockProvider) FastFwd(to uint64) error {
+ var b types.Block
+ for {
+ if err := p.stream.Decode(&b); err == io.EOF {
+ return nil
+ } else if err != nil {
+ return fmt.Errorf("error fast fwd: %v", err)
+ } else if b.NumberU64() >= to-1 {
+ return nil
+ }
+ }
+}
+
+func (p *ExportFileBlockProvider) NextBlock() (*types.Block, error) {
+ var b types.Block
+ if err := p.stream.Decode(&b); err == io.EOF {
+ return nil, nil
+ } else if err != nil {
+ return nil, fmt.Errorf("error fast fwd: %v", err)
+ }
+ return &b, nil
+}
--
GitLab