From 686721b6c6e3bf8a3df7533c6451335054115efe Mon Sep 17 00:00:00 2001
From: Alex Sharov <AskAlexSharov@gmail.com>
Date: Thu, 4 Nov 2021 13:25:34 +0700
Subject: [PATCH] EliasFano: fix jump calculation, fuzzing to trigger jump
logic (#2912)
---
cmd/hack/hack.go | 23 +++++++++++++++++++++--
go.mod | 4 ++--
go.sum | 8 ++++----
3 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
index f7c4b8c162..3579cd2e1e 100644
--- a/cmd/hack/hack.go
+++ b/cmd/hack/hack.go
@@ -63,6 +63,8 @@ import (
"github.com/wcharczuk/go-chart/v2"
)
+const ASSERT = false
+
var (
verbosity = flag.Uint("verbosity", 3, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default 3)")
action = flag.String("action", "", "action to execute")
@@ -2731,9 +2733,10 @@ func recsplitLookup(chaindata, name string) error {
idx := recsplit.MustOpen(name + ".idx")
defer idx.Close()
- var word = make([]byte, 0, 256)
+ var word, word2 = make([]byte, 0, 4096), make([]byte, 0, 4096)
wc := 0
g := d.MakeGetter()
+ dataGetter := d.MakeGetter()
parseCtx := txpool.NewTxParseContext(*chainID)
parseCtx.WithSender(false)
@@ -2741,6 +2744,8 @@ func recsplitLookup(chaindata, name string) error {
var sender [20]byte
var l1, l2, total time.Duration
start := time.Now()
+ var prev []byte
+ var prevOffset uint64
for g.HasNext() {
word, _ = g.Next(word[:0])
if _, err := parseCtx.ParseTransaction(word[1:], 0, &slot, sender[:]); err != nil {
@@ -2752,8 +2757,22 @@ func recsplitLookup(chaindata, name string) error {
recID := idx.Lookup(slot.IdHash[:])
l1 += time.Since(t)
t = time.Now()
- _ = idx.Lookup2(recID)
+ offset := idx.Lookup2(recID)
l2 += time.Since(t)
+ if ASSERT {
+ var dataP uint64
+ if prev != nil {
+ dataGetter.Reset(prevOffset)
+ word2, dataP = dataGetter.Next(word2[:0])
+ if !bytes.Equal(word, word2) {
+ fmt.Printf("wc=%d, %d,%d\n", wc, offset, dataP-uint64(len(word2)))
+ fmt.Printf("word: %x,%x\n\n", word, word2)
+ panic(fmt.Errorf("getter returned wrong data. IdHash=%x, offset=%x", slot.IdHash[:], offset))
+ }
+ }
+ prev = common.CopyBytes(word)
+ prevOffset = offset
+ }
select {
default:
diff --git a/go.mod b/go.mod
index 0409c5628a..3d9ed0e4cb 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.16
require (
github.com/RoaringBitmap/roaring v0.9.4
github.com/VictoriaMetrics/fastcache v1.7.0
- github.com/VictoriaMetrics/metrics v1.18.0
+ github.com/VictoriaMetrics/metrics v1.18.1
github.com/anacrolix/log v0.8.0
github.com/anacrolix/torrent v1.25.1
github.com/btcsuite/btcd v0.21.0-beta
@@ -36,7 +36,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
- github.com/ledgerwatch/erigon-lib v0.0.0-20211102145739-607ab3a85bf7
+ github.com/ledgerwatch/erigon-lib v0.0.0-20211104042003-182de9f4806b
github.com/ledgerwatch/log/v3 v3.4.0
github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d
github.com/logrusorgru/aurora/v3 v3.0.0
diff --git a/go.sum b/go.sum
index 3032f8322a..c71f0491c0 100644
--- a/go.sum
+++ b/go.sum
@@ -67,8 +67,8 @@ github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDO
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.7.0 h1:E6GibaGI685TafrI7E/QqZPkMsOzRw+3gpICQx08ISg=
github.com/VictoriaMetrics/fastcache v1.7.0/go.mod h1:n7Sl+ioh/HlWeYHLSIBIE8TcZFHg/+xgvomWSS5xuEE=
-github.com/VictoriaMetrics/metrics v1.18.0 h1:vov5NxDHRSXFbdiH4dYLYEjKLoAXXSQ7hcnG8TSD9JQ=
-github.com/VictoriaMetrics/metrics v1.18.0/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
+github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0=
+github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -497,8 +497,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20211102145739-607ab3a85bf7 h1:9ivRlLmbUWAKNjoMfz3iBjCfKmT0GBRL1cu1NpaEekg=
-github.com/ledgerwatch/erigon-lib v0.0.0-20211102145739-607ab3a85bf7/go.mod h1:tmLML7kM8BV92RBOUJ4FXmLY/gSqZ8AHW4kkoDQud8k=
+github.com/ledgerwatch/erigon-lib v0.0.0-20211104042003-182de9f4806b h1:pDZXCHYa0XyYAgk+/TFJuzpeoU9YlOZBzZYCEne+sKc=
+github.com/ledgerwatch/erigon-lib v0.0.0-20211104042003-182de9f4806b/go.mod h1:yqHdCAcVTYLxBeWQoqEd0r9znvew3MMHPeDgv2wgAxA=
github.com/ledgerwatch/log/v3 v3.4.0 h1:SEIOcv5a2zkG3PmoT5jeTU9m/0nEUv0BJS5bzsjwKCI=
github.com/ledgerwatch/log/v3 v3.4.0/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d h1:/IKMrJdfRsoYNc36PXqP4xMH3vhW/8IQyBKGQbKZUno=
--
GitLab