From ba55611f29bbdc0b1b3912a59afca6b55d4fa093 Mon Sep 17 00:00:00 2001
From: ledgerwatch <akhounov@gmail.com>
Date: Fri, 2 Jul 2021 14:34:20 +0100
Subject: [PATCH] rpctest for eth_getTransactionReceipt (#2271)
* rpctest bench for eth_getTransationReceipt
* Add blockHash to re-generated receipts
* Modify scanReceipts
Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>
---
cmd/hack/hack.go | 69 +---------
cmd/rpcdaemon/commands/eth_receipts.go | 4 +-
cmd/rpctest/main.go | 24 +---
cmd/rpctest/rpctest/bench_txreceipts.go | 121 +++++++++++++++++
cmd/rpctest/rpctest/fixState.go | 135 -------------------
cmd/rpctest/rpctest/proofs.go | 167 ------------------------
6 files changed, 134 insertions(+), 386 deletions(-)
create mode 100644 cmd/rpctest/rpctest/bench_txreceipts.go
delete mode 100644 cmd/rpctest/rpctest/fixState.go
delete mode 100644 cmd/rpctest/rpctest/proofs.go
diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
index 397e509129..b4bc603154 100644
--- a/cmd/hack/hack.go
+++ b/cmd/hack/hack.go
@@ -21,9 +21,7 @@ import (
"github.com/RoaringBitmap/roaring/roaring64"
"github.com/holiman/uint256"
- "github.com/ledgerwatch/erigon/ethdb/cbor"
kv2 "github.com/ledgerwatch/erigon/ethdb/kv"
- "github.com/ledgerwatch/erigon/migrations"
"github.com/wcharczuk/go-chart"
"github.com/wcharczuk/go-chart/util"
@@ -2070,7 +2068,7 @@ func scanTxs(chaindata string) error {
return nil
}
-func scanReceipts(chaindata string) error {
+func scanReceipts(chaindata string, block uint64) error {
dbdb := kv2.MustOpen(chaindata).RwKV()
defer dbdb.Close()
txtx, err := dbdb.BeginRw(context.Background())
@@ -2085,70 +2083,13 @@ func scanReceipts(chaindata string) error {
} else {
return fmt.Errorf("no transaction")
}
- genesisBlock, err := rawdb.ReadBlockByNumber(tx, 0)
- if err != nil {
- return err
- }
- chainConfig, cerr := rawdb.ReadChainConfig(tx, genesisBlock.Hash())
- if cerr != nil {
- return cerr
- }
- logInterval := 30 * time.Second
- logEvery := time.NewTicker(logInterval)
- defer logEvery.Stop()
- var buf bytes.Buffer
var key [8]byte
var v []byte
- var to uint64
- if to, err = stages.GetStageProgress(tx, stages.Execution); err != nil {
+ binary.BigEndian.PutUint64(key[:], block)
+ if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
return err
}
- for blockNum := uint64(1); blockNum <= to; blockNum++ {
- binary.BigEndian.PutUint64(key[:], blockNum)
- if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
- return err
- }
- if v == nil {
- continue
- }
- //fmt.Printf("blockNum = %d\n", blockNum)
- select {
- default:
- case <-logEvery.C:
- log.Info("Scanned receipts up to", "block", blockNum)
- }
- var receipts types.Receipts
- var oldReceipts migrations.OldReceipts
- if err = cbor.Unmarshal(&oldReceipts, bytes.NewReader(v)); err != nil {
- continue
- }
-
- var blockHash common.Hash
- if blockHash, err = rawdb.ReadCanonicalHash(tx, blockNum); err != nil {
- return err
- }
- var body *types.Body
- if chainConfig.IsBerlin(blockNum) {
- body = rawdb.ReadBody(tx, blockHash, blockNum)
- }
- receipts = make(types.Receipts, len(oldReceipts))
- for i, oldReceipt := range oldReceipts {
- receipts[i] = new(types.Receipt)
- receipts[i].PostState = oldReceipt.PostState
- receipts[i].Status = oldReceipt.Status
- receipts[i].CumulativeGasUsed = oldReceipt.CumulativeGasUsed
- if body != nil {
- receipts[i].Type = body.Transactions[i].Type()
- }
- }
- buf.Reset()
- if err = cbor.Marshal(&buf, receipts); err != nil {
- return err
- }
- //if err = tx.Put(dbutils.BlockReceiptsPrefix, common.CopyBytes(key[:]), common.CopyBytes(buf.Bytes())); err != nil {
- // return err
- //}
- }
+ fmt.Printf("blockNum = %d, receipt %x\n", block, v)
return nil
}
@@ -2356,7 +2297,7 @@ func main() {
err = scanTxs(*chaindata)
case "scanReceipts":
- err = scanReceipts(*chaindata)
+ err = scanReceipts(*chaindata, uint64(*block))
case "testTxPool":
err = testTxPool()
diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go
index fd116abbe0..30ee9f9189 100644
--- a/cmd/rpcdaemon/commands/eth_receipts.go
+++ b/cmd/rpcdaemon/commands/eth_receipts.go
@@ -5,9 +5,10 @@ import (
"context"
"encoding/binary"
"fmt"
- "github.com/holiman/uint256"
"math/big"
+ "github.com/holiman/uint256"
+
"github.com/RoaringBitmap/roaring"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/dbutils"
@@ -53,6 +54,7 @@ func getReceipts(ctx context.Context, tx ethdb.Tx, chainConfig *params.ChainConf
if err != nil {
return nil, err
}
+ receipt.BlockHash = block.Hash()
receipts = append(receipts, receipt)
}
diff --git a/cmd/rpctest/main.go b/cmd/rpctest/main.go
index e9e46bb0dc..dcc16a57f3 100644
--- a/cmd/rpctest/main.go
+++ b/cmd/rpctest/main.go
@@ -22,7 +22,6 @@ func main() {
erigonURL string
blockFrom uint64
blockTo uint64
- chaindata string
recordFile string
)
withErigonUrl := func(cmd *cobra.Command) {
@@ -198,27 +197,15 @@ func main() {
}
with(benchTraceFilterCmd, withGethUrl, withErigonUrl, withNeedCompare, withBlockNum, withRecord)
- var proofsCmd = &cobra.Command{
- Use: "proofs",
+ var benchTxReceiptCmd = &cobra.Command{
+ Use: "benchTxReceipt",
Short: "",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
- rpctest.Proofs(chaindata, gethURL, blockFrom)
+ rpctest.BenchTxReceipt(erigonURL, gethURL, needCompare, blockFrom, blockTo, recordFile)
},
}
- proofsCmd.Flags().StringVar(&chaindata, "chaindata", "", "")
- with(proofsCmd, withGethUrl, withBlockNum)
-
- var fixStateCmd = &cobra.Command{
- Use: "fixstate",
- Short: "",
- Long: ``,
- Run: func(cmd *cobra.Command, args []string) {
- rpctest.FixState(chaindata, gethURL)
- },
- }
- fixStateCmd.Flags().StringVar(&chaindata, "chaindata", "", "")
- with(fixStateCmd, withGethUrl)
+ with(benchTxReceiptCmd, withGethUrl, withErigonUrl, withNeedCompare, withBlockNum, withRecord)
var replayCmd = &cobra.Command{
Use: "replay",
@@ -267,8 +254,7 @@ func main() {
bench13Cmd,
benchTraceBlockCmd,
benchTraceFilterCmd,
- proofsCmd,
- fixStateCmd,
+ benchTxReceiptCmd,
compareAccountRange,
replayCmd,
)
diff --git a/cmd/rpctest/rpctest/bench_txreceipts.go b/cmd/rpctest/rpctest/bench_txreceipts.go
new file mode 100644
index 0000000000..da47a90fd7
--- /dev/null
+++ b/cmd/rpctest/rpctest/bench_txreceipts.go
@@ -0,0 +1,121 @@
+package rpctest
+
+import (
+ "bufio"
+ "fmt"
+ "net/http"
+ "os"
+ "time"
+)
+
+// benchTxReceipt compares response of Erigon with Geth
+// but also can be used for comparing RPCDaemon with Geth
+// parameters:
+// needCompare - if false - doesn't call Erigon and doesn't compare responses
+// use false value - to generate vegeta files, it's faster but we can generate vegeta files for Geth and Erigon
+func BenchTxReceipt(erigonURL, gethURL string, needCompare bool, blockFrom uint64, blockTo uint64, recordFile string) {
+ setRoutes(erigonURL, gethURL)
+ var client = &http.Client{
+ Timeout: time.Second * 600,
+ }
+
+ var rec *bufio.Writer
+ if recordFile != "" {
+ f, err := os.Create(recordFile)
+ if err != nil {
+ fmt.Printf("Cannot create file %s for recording: %v\n", recordFile, err)
+ return
+ }
+ defer f.Close()
+ rec = bufio.NewWriter(f)
+ defer rec.Flush()
+ }
+
+ var res CallResult
+ reqGen := &RequestGenerator{
+ client: client,
+ }
+
+ reqGen.reqID++
+ var blockNumber EthBlockNumber
+ res = reqGen.Erigon("eth_blockNumber", reqGen.blockNumber(), &blockNumber)
+ if res.Err != nil {
+ fmt.Printf("Could not get block number: %v\n", res.Err)
+ return
+ }
+ if blockNumber.Error != nil {
+ fmt.Printf("Error getting block number: %d %s\n", blockNumber.Error.Code, blockNumber.Error.Message)
+ return
+ }
+ fmt.Printf("Last block: %d\n", blockNumber.Number)
+ for bn := blockFrom; bn <= blockTo; bn++ {
+ reqGen.reqID++
+ var b EthBlockByNumber
+ res = reqGen.Erigon("eth_getBlockByNumber", reqGen.getBlockByNumber(bn), &b)
+ if res.Err != nil {
+ fmt.Printf("Could not retrieve block (Erigon) %d: %v\n", bn, res.Err)
+ return
+ }
+
+ if b.Error != nil {
+ fmt.Printf("Error retrieving block (Erigon): %d %s\n", b.Error.Code, b.Error.Message)
+ return
+ }
+
+ if needCompare {
+ var bg EthBlockByNumber
+ res = reqGen.Geth("eth_getBlockByNumber", reqGen.getBlockByNumber(bn), &bg)
+ if res.Err != nil {
+ fmt.Printf("Could not retrieve block (geth) %d: %v\n", bn, res.Err)
+ return
+ }
+ if bg.Error != nil {
+ fmt.Printf("Error retrieving block (geth): %d %s\n", bg.Error.Code, bg.Error.Message)
+ return
+ }
+ if !compareBlocks(&b, &bg) {
+ fmt.Printf("Block difference for %d\n", bn)
+ return
+ }
+ }
+
+ for _, tx := range b.Result.Transactions {
+ reqGen.reqID++
+
+ request := reqGen.getTransactionReceipt(tx.Hash)
+ recording := rec != nil // This flag will be set to false if recording is not to be performed
+ res = reqGen.Erigon2("eth_getTransactionReceipt", request)
+ if res.Err != nil {
+ fmt.Printf("Could not eth getTransactionReceipt (Erigon) %d: %v\n", bn, res.Err)
+ return
+ }
+ if errVal := res.Result.Get("error"); errVal != nil {
+ fmt.Printf("Error eth getTransactionReceipt (Erigon): %d %s\n", errVal.GetInt("code"), errVal.GetStringBytes("message"))
+ return
+ }
+
+ if needCompare {
+ resg := reqGen.Geth2("eth_getTransactionReceipt", request)
+ if resg.Err != nil {
+ fmt.Printf("Could not eth getTransactionReceipt (geth) %d: %v\n", bn, resg.Err)
+ return
+ }
+ if errVal := resg.Result.Get("error"); errVal != nil {
+ fmt.Printf("Error eth getTransactionReceipt (geth): %d %s\n", errVal.GetInt("code"), errVal.GetStringBytes("message"))
+ return
+ }
+ if resg.Err == nil && resg.Result.Get("error") == nil {
+ if err := compareResults(res.Result, resg.Result); err != nil {
+ fmt.Printf("Different getTransactionReceipt block %d, tx %s: %v\n", bn, tx.Hash, err)
+ fmt.Printf("\n\nTG response=================================\n%s\n", res.Response)
+ fmt.Printf("\n\nG response=================================\n%s\n", resg.Response)
+ return
+ }
+ }
+ }
+ if recording {
+ fmt.Fprintf(rec, "%s\n%s\n\n", request, res.Response)
+ }
+ }
+ }
+}
diff --git a/cmd/rpctest/rpctest/fixState.go b/cmd/rpctest/rpctest/fixState.go
deleted file mode 100644
index 3000ecdb8b..0000000000
--- a/cmd/rpctest/rpctest/fixState.go
+++ /dev/null
@@ -1,135 +0,0 @@
-package rpctest
-
-import (
- "bytes"
- "context"
- "encoding/binary"
- "fmt"
- "net/http"
- "time"
-
- "github.com/ledgerwatch/erigon/common"
- "github.com/ledgerwatch/erigon/common/dbutils"
- "github.com/ledgerwatch/erigon/core/rawdb"
- "github.com/ledgerwatch/erigon/core/state"
- "github.com/ledgerwatch/erigon/core/types/accounts"
- "github.com/ledgerwatch/erigon/ethdb"
- "github.com/ledgerwatch/erigon/ethdb/kv"
- "github.com/ledgerwatch/erigon/turbo/trie"
-)
-
-func FixState(chaindata string, url string) {
- db := kv.MustOpen(chaindata).RwKV()
- defer db.Close()
- tx, err1 := db.BeginRw(context.Background())
- if err1 != nil {
- panic(err1)
- }
- defer tx.Rollback()
- currentHeader := rawdb.ReadCurrentHeader(tx)
- blockNum := currentHeader.Number.Uint64()
- blockHash := currentHeader.Hash()
- fmt.Printf("Block number: %d\n", blockNum)
- fmt.Printf("Block root hash: %x\n", currentHeader.Root)
- reqID := 0
- roots := make(map[common.Address]*accounts.Account)
- var client = &http.Client{
- Timeout: time.Second * 600,
- }
-
- c, err := tx.Cursor(dbutils.PlainStateBucket)
- if err != nil {
- panic(err)
- }
- defer c.Close()
- if err := ethdb.ForEach(c, func(k, v []byte) (bool, error) {
- if len(k) == common.AddressLength {
- var address common.Address
- address.SetBytes(k)
- if _, ok := roots[address]; !ok {
- if account, err2 := state.NewPlainStateReader(tx).ReadAccountData(address); err2 != nil {
- return false, err2
- } else {
- roots[address] = account
- }
- }
- }
-
- return true, nil
- }); err != nil {
- panic(err)
- }
- for address, account := range roots {
- if account != nil && account.Root != trie.EmptyRoot {
- contractPrefix := make([]byte, common.HashLength+common.IncarnationLength)
- addrHash, _ := common.HashData(address.Bytes())
- copy(contractPrefix, addrHash[:])
- binary.BigEndian.PutUint64(contractPrefix[common.HashLength:], account.Incarnation)
- rl := trie.NewRetainList(0)
- loader := trie.NewFlatDBTrieLoader("checkRoots")
- if err := loader.Reset(rl, nil, nil, false); err != nil {
- panic(err)
- }
- root, err1 := loader.CalcTrieRoot(tx, contractPrefix, nil)
- if err1 != nil || root != account.Root {
- fmt.Printf("%x: error %v, got hash %x, expected hash %x\n", addrHash, err1, root, account.Root)
- template := `{"jsonrpc":"2.0","method":"debug_storageRangeAt","params":["0x%x", %d,"0x%x","0x%x",%d],"id":%d}`
- sm := make(map[common.Hash]storageEntry)
- nextKey := &common.Hash{}
- for nextKey != nil {
- reqID++
- var sr DebugStorageRange
- if err := post(client, url, fmt.Sprintf(template, blockHash, 0, address, *nextKey, 1024, reqID), &sr); err != nil {
- fmt.Printf("Could not get storageRange: %v\n", err)
- return
- }
- if sr.Error != nil {
- fmt.Printf("Error getting storageRange: %d %s\n", sr.Error.Code, sr.Error.Message)
- break
- } else {
- nextKey = sr.Result.NextKey
- for k, v := range sr.Result.Storage {
- sm[k] = v
- }
- }
- }
- fmt.Printf("Retrieved %d storage items from geth archive node\n", len(sm))
- for key, entry := range sm {
- var cKey [common.HashLength + common.IncarnationLength + common.HashLength]byte
- copy(cKey[:], addrHash[:])
- binary.BigEndian.PutUint64(cKey[common.HashLength:], account.Incarnation)
- copy(cKey[common.HashLength+common.IncarnationLength:], key[:])
- dbValue, _ := tx.GetOne(dbutils.HashedStorageBucket, cKey[:])
- value := bytes.TrimLeft(entry.Value[:], "\x00")
- if !bytes.Equal(dbValue, value) {
- fmt.Printf("Key: %x, value: %x, dbValue: %x\n", key, value, dbValue)
- if err := tx.Put(dbutils.HashedStorageBucket, cKey[:], value); err != nil {
- fmt.Printf("%v\n", err)
- }
- }
- }
- var cKey [common.HashLength + common.IncarnationLength + common.HashLength]byte
- copy(cKey[:], addrHash[:])
- binary.BigEndian.PutUint64(cKey[common.HashLength:], account.Incarnation)
- c2, err := tx.Cursor(dbutils.HashedStorageBucket)
- if err != nil {
- panic(err)
- }
- if err := ethdb.Walk(c, cKey[:], 8*(common.HashLength+common.IncarnationLength), func(k, v []byte) (bool, error) {
- var kh common.Hash
- copy(kh[:], k[common.HashLength+common.IncarnationLength:])
- if _, ok := sm[kh]; !ok {
- fmt.Printf("Key: %x, dbValue: %x\n", kh, v)
- if err := tx.Delete(dbutils.HashedStorageBucket, k, nil); err != nil {
- fmt.Printf("%v\n", err)
- }
- }
- return true, nil
- }); err != nil {
- panic(err)
- }
- c2.Close()
- }
- }
- }
-}
diff --git a/cmd/rpctest/rpctest/proofs.go b/cmd/rpctest/rpctest/proofs.go
deleted file mode 100644
index dd7eeb0f00..0000000000
--- a/cmd/rpctest/rpctest/proofs.go
+++ /dev/null
@@ -1,167 +0,0 @@
-package rpctest
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/ledgerwatch/erigon/ethdb"
- "github.com/ledgerwatch/erigon/ethdb/kv"
- "github.com/ledgerwatch/erigon/turbo/trie"
-)
-
-func Proofs(chaindata string, url string, block uint64) {
- fileName := "trie.txt"
- db := kv.MustOpen(chaindata)
- defer db.Close()
- tx, err1 := db.Begin(context.Background(), ethdb.RW)
- if err1 != nil {
- panic(err1)
- }
- defer tx.Rollback()
-
- var t *trie.Trie
- if _, errf := os.Stat(fileName); errf != nil {
- if os.IsNotExist(errf) {
- // Resolve 6 top levels of the accounts trie
- rl := trie.NewRetainList(6)
- loader := trie.NewFlatDBTrieLoader("checkRoots")
- if err := loader.Reset(rl, nil, nil, false); err != nil {
- panic(err)
- }
- root, err := loader.CalcTrieRoot(tx.(ethdb.HasTx).Tx().(ethdb.RwTx), []byte{}, nil)
- if err != nil {
- panic(err)
- }
- fmt.Printf("Resolved with hash: %x\n", root)
- f, err1 := os.Create(fileName)
- if err1 == nil {
- defer f.Close()
- t.Print(f)
- } else {
- panic(err1)
- }
- fmt.Printf("Saved trie to file\n")
- } else {
- panic(errf)
- }
- } else {
- f, err1 := os.Open(fileName)
- if err1 == nil {
- defer f.Close()
- var err2 error
- t, err2 = trie.Load(f)
- if err2 != nil {
- panic(err2)
- }
- fmt.Printf("Restored from file with hash: %x\n", t.Hash())
- } else {
- panic(err1)
- }
- }
- /* TODO: migrate to usage cursors api - and to not use preimages
- var client = &http.Client{
- Timeout: time.Second * 600,
- }
- reqID := 0
-
- level := 0
- diffKeys := [][]byte{{}}
- var newDiffKeys [][]byte
- for len(diffKeys) > 0 && level < 6 {
-
- fmt.Printf("================================================\n")
- fmt.Printf("LEVEL %d, diffKeys: %d\n", level, len(diffKeys))
- fmt.Printf("================================================\n")
- for _, diffKey := range diffKeys {
- // Find account with the suitable hash
- var startKey common.Hash
- for i, d := range diffKey {
- if i%2 == 0 {
- startKey[i/2] |= (d << 4)
- } else {
- startKey[i/2] |= d
- }
- }
- var account common.Address
- var found bool
- err := tx.Walk(dbutils.PreimagePrefix, startKey[:], 4*len(diffKey), func(k, v []byte) (bool, error) {
- if len(v) == common.AddressLength {
- copy(account[:], v)
- found = true
- return false, nil
- }
- return true, nil
- })
- if err != nil {
- fmt.Printf("Error when looking for suitable account for diffKey %x\n", diffKey)
- return
- }
- if !found {
- fmt.Printf("Could not find suitable account for diffKey %x\n", diffKey)
- return
- }
- reqID++
- template := `{"jsonrpc":"2.0","method":"eth_getProof","params":["0x%x",[],"0x%x"],"id":%d}`
- var proof EthGetProof
- if err = post(client, url, fmt.Sprintf(template, account, block, reqID), &proof); err != nil {
- fmt.Printf("Could not get block number: %v\n", err)
- return
- }
- if proof.Error != nil {
- fmt.Printf("Error retrieving proof: %d %s\n", proof.Error.Code, proof.Error.Message)
- return
- }
- if len(proof.Result.AccountProof) <= len(diffKey) {
- fmt.Printf("RPC result needs to be at least %d levels deep\n", len(diffKey)+1)
- return
- }
- p := proof.Result.AccountProof[len(diffKey)]
- b := common.FromHex(p)
- h := common.BytesToHash(crypto.Keccak256(b))
- hE, err := t.HashOfHexKey(diffKey)
- if err != nil {
- fmt.Printf("Error computing partial hash for %x: %v\n", diffKey, err)
- return
- }
- if h != hE {
- fmt.Printf("key %x: %x %x adding nibbles: ", diffKey, h, hE)
- var branch [17][]byte
- err = rlp.DecodeBytes(b, &branch)
- if err != nil {
- fmt.Printf("Error decoding: %v\n", err)
- fmt.Printf("%s\n", p)
- }
- // Expand keys further
- for nibble := byte(0); nibble < 16; nibble++ {
- newDiff := make([]byte, len(diffKey)+1)
- copy(newDiff, diffKey)
- newDiff[len(diffKey)] = nibble
- var proofHash common.Hash
- copy(proofHash[:], branch[nibble])
- if proofHash != (common.Hash{}) || err != nil {
- newHash, err := t.HashOfHexKey(newDiff)
- if err != nil {
- fmt.Printf("Error computing partial hash for %x: %v\n", newDiff, err)
- }
- if proofHash != newHash {
- newDiffKeys = append(newDiffKeys, newDiff)
- fmt.Printf("%x", nibble)
- }
- }
- }
- fmt.Printf("\n")
- } else {
- fmt.Printf("MATCH key %x: %x %x\n", diffKey, h, hE)
- }
- }
- diffKeys = newDiffKeys
- newDiffKeys = nil
- level++
- }
- fmt.Printf("\n\nRESULT:\n")
- for _, diffKey := range diffKeys {
- fmt.Printf("%x\n", diffKey)
- }
- */
-}
--
GitLab