From ea8a347ece85e2445ae64717ac94ac8bbadb0370 Mon Sep 17 00:00:00 2001
From: Alex Sharov <AskAlexSharov@gmail.com>
Date: Thu, 28 Apr 2022 09:10:00 +0700
Subject: [PATCH] Downloader: calc stat inside, add --torrent.download.slots
and limit downloads inside (#3986)
* save
* save
* save
* save
* save
* save
* save
* save
* save
---
cmd/downloader/downloader/downloader.go | 387 +++++++++---------
cmd/downloader/downloader/grpc_server.go | 118 ++----
.../downloader/torrentcfg/torrentcfg.go | 27 +-
cmd/downloader/downloader/util.go | 78 +++-
cmd/downloader/main.go | 12 +-
cmd/observer/observer/crawler.go | 5 +-
cmd/utils/flags.go | 6 +
eth/backend.go | 10 +-
eth/stagedsync/stage_headers.go | 95 ++---
go.mod | 22 +-
go.sum | 99 +++--
turbo/cli/default_flags.go | 1 +
12 files changed, 457 insertions(+), 403 deletions(-)
diff --git a/cmd/downloader/downloader/downloader.go b/cmd/downloader/downloader/downloader.go
index ac417a7d3f..a28d58f603 100644
--- a/cmd/downloader/downloader/downloader.go
+++ b/cmd/downloader/downloader/downloader.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"runtime"
+ "sync"
"time"
"github.com/anacrolix/torrent"
@@ -13,14 +14,17 @@ import (
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/cmd/downloader/downloader/torrentcfg"
"github.com/ledgerwatch/log/v3"
+ "golang.org/x/sync/semaphore"
)
-const ASSERT = false
-
type Protocols struct {
TorrentClient *torrent.Client
DB kv.RwDB
cfg *torrentcfg.Cfg
+
+ statsLock *sync.RWMutex
+ stats AggStats
+ snapshotDir *dir.Rw
}
func New(cfg *torrentcfg.Cfg, snapshotDir *dir.Rw) (*Protocols, error) {
@@ -43,6 +47,8 @@ func New(cfg *torrentcfg.Cfg, snapshotDir *dir.Rw) (*Protocols, error) {
cfg: cfg,
TorrentClient: torrentClient,
DB: cfg.DB,
+ statsLock: &sync.RWMutex{},
+ snapshotDir: snapshotDir,
}, nil
}
@@ -66,245 +72,218 @@ func readPeerID(db kv.RoDB) (peerID []byte, err error) {
return peerID, nil
}
-func (cli *Protocols) Close() {
- for _, tr := range cli.TorrentClient.Torrents() {
- tr.Drop()
- }
- cli.TorrentClient.Close()
- cli.DB.Close()
- if cli.cfg.CompletionCloser != nil {
- cli.cfg.CompletionCloser.Close() //nolint
+func (cli *Protocols) Start(ctx context.Context, silent bool) error {
+ if err := BuildTorrentsAndAdd(ctx, cli.snapshotDir, cli.TorrentClient); err != nil {
+ return err
}
-}
-func (cli *Protocols) PeerID() []byte {
- peerID := cli.TorrentClient.PeerID()
- return peerID[:]
-}
+ var sem = semaphore.NewWeighted(int64(cli.cfg.DownloadSlots))
-func LoggingLoop(ctx context.Context, torrentClient *torrent.Client) {
- interval := time.Second * 20
- logEvery := time.NewTicker(interval)
- defer logEvery.Stop()
- var m runtime.MemStats
- var stats AggStats
-
- for {
- select {
- case <-ctx.Done():
- return
- case <-logEvery.C:
- torrents := torrentClient.Torrents()
- allComplete := true
- gotInfo := 0
+ go func() {
+ for {
+ torrents := cli.TorrentClient.Torrents()
for _, t := range torrents {
- select {
- case <-t.GotInfo(): // all good
- gotInfo++
- default:
+ <-t.GotInfo()
+ if t.Complete.Bool() {
+ continue
}
- allComplete = allComplete && t.Complete.Bool()
- }
- if gotInfo < len(torrents) {
- log.Info(fmt.Sprintf("[torrent] Waiting for torrents metadata: %d/%d", gotInfo, len(torrents)))
- continue
- }
+ if err := sem.Acquire(ctx, 1); err != nil {
+ return
+ }
+ t.AllowDataDownload()
+ t.DownloadAll()
+ go func(t *torrent.Torrent) {
+ //r := t.NewReader()
+ //_, _ = io.Copy(io.Discard, r) // enable streaming - it will prioritize sequential download
- runtime.ReadMemStats(&m)
- stats = CalcStats(stats, interval, torrentClient)
- if allComplete {
- log.Info("[torrent] Seeding",
- "download", common2.ByteCount(uint64(stats.readBytesPerSec))+"/s",
- "upload", common2.ByteCount(uint64(stats.writeBytesPerSec))+"/s",
- "unique_peers", stats.peersCount,
- "files", stats.torrentsCount,
- "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
- continue
+ <-t.Complete.On()
+ sem.Release(1)
+ }(t)
}
+ time.Sleep(330 * time.Second)
+ }
+ }()
- log.Info("[torrent] Downloading",
- "Progress", fmt.Sprintf("%.2f%%", stats.Progress),
- "download", common2.ByteCount(uint64(stats.readBytesPerSec))+"/s",
- "upload", common2.ByteCount(uint64(stats.writeBytesPerSec))+"/s",
- "unique_peers", stats.peersCount,
- "files", stats.torrentsCount,
- "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
- if stats.peersCount == 0 {
- ips := torrentClient.BadPeerIPs()
- if len(ips) > 0 {
- log.Info("[torrent] Stats", "banned", ips)
+ go func() {
+ var m runtime.MemStats
+ logEvery := time.NewTicker(20 * time.Second)
+ defer logEvery.Stop()
+
+ interval := 5 * time.Second
+ statEvery := time.NewTicker(interval)
+ defer statEvery.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-statEvery.C:
+ cli.ReCalcStats(interval)
+
+ case <-logEvery.C:
+ if silent {
+ continue
}
+ stats := cli.Stats()
+
+ if stats.MetadataReady < stats.FilesTotal {
+ log.Info(fmt.Sprintf("[Snapshots] Waiting for torrents metadata: %d/%d", stats.MetadataReady, stats.FilesTotal))
+ continue
+ }
+
+ runtime.ReadMemStats(&m)
+ if stats.Completed {
+ log.Info("[Snapshots] Seeding",
+ "up", common2.ByteCount(stats.UploadRate)+"/s",
+ "peers", stats.PeersUnique,
+ "connections", stats.ConnectionsTotal,
+ "files", stats.FilesTotal,
+ "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
+ continue
+ }
+
+ log.Info("[Snapshots] Downloading",
+ "progress", fmt.Sprintf("%.2f%% %s/%s", stats.Progress, common2.ByteCount(stats.BytesCompleted), common2.ByteCount(stats.BytesTotal)),
+ "download", common2.ByteCount(stats.DownloadRate)+"/s",
+ "upload", common2.ByteCount(stats.UploadRate)+"/s",
+ "peers", stats.PeersUnique,
+ "connections", stats.ConnectionsTotal,
+ "files", stats.FilesTotal,
+ "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
+ if stats.PeersUnique == 0 {
+ ips := cli.TorrentClient.BadPeerIPs()
+ if len(ips) > 0 {
+ log.Info("[Snapshots] Stats", "banned", ips)
+ }
+ }
}
}
- }
-}
-
-func (cli *Protocols) StopSeeding(hash metainfo.Hash) error {
- t, ok := cli.TorrentClient.Torrent(hash)
- if !ok {
- return nil
- }
- ch := t.Closed()
- t.Drop()
- <-ch
+ }()
return nil
}
-type AggStats struct {
- readBytesPerSec int64
- writeBytesPerSec int64
- peersCount int64
-
- Progress float32
- torrentsCount int
-
- bytesRead int64
- bytesWritten int64
-}
+func (cli *Protocols) ReCalcStats(interval time.Duration) {
+ cli.statsLock.Lock()
+ defer cli.statsLock.Unlock()
+ prevStats, stats := cli.stats, cli.stats
-func CalcStats(prevStats AggStats, interval time.Duration, client *torrent.Client) (result AggStats) {
- var aggBytesCompleted, aggLen int64
- //var aggCompletedPieces, aggNumPieces, aggPartialPieces int
- peers := map[torrent.PeerID]*torrent.PeerConn{}
- torrents := client.Torrents()
- connStats := client.ConnStats()
+ peers := make(map[torrent.PeerID]struct{}, 16)
+ torrents := cli.TorrentClient.Torrents()
+ connStats := cli.TorrentClient.ConnStats()
- result.bytesRead += connStats.BytesReadUsefulIntendedData.Int64()
- result.bytesWritten += connStats.BytesWrittenData.Int64()
+ stats.BytesRead = uint64(connStats.BytesReadUsefulIntendedData.Int64())
+ stats.BytesWritten = uint64(connStats.BytesWrittenData.Int64())
+ stats.BytesTotal, stats.BytesCompleted, stats.ConnectionsTotal, stats.MetadataReady = 0, 0, 0, 0
for _, t := range torrents {
- aggBytesCompleted += t.BytesCompleted()
- aggLen += t.Length()
-
- for _, peer := range t.PeerConns() {
- peers[peer.PeerID] = peer
+ select {
+ case <-t.GotInfo():
+ stats.MetadataReady++
+ for _, peer := range t.PeerConns() {
+ stats.ConnectionsTotal++
+ peers[peer.PeerID] = struct{}{}
+ }
+ stats.BytesCompleted += uint64(t.BytesCompleted())
+ stats.BytesTotal += uint64(t.Length())
+ default:
}
+
+ stats.Completed = stats.Completed && t.Complete.Bool()
}
- result.readBytesPerSec += (result.bytesRead - prevStats.bytesRead) / int64(interval.Seconds())
- result.writeBytesPerSec += (result.bytesWritten - prevStats.bytesWritten) / int64(interval.Seconds())
+ stats.DownloadRate = (stats.BytesRead - prevStats.BytesRead) / uint64(interval.Seconds())
+ stats.UploadRate = (stats.BytesWritten - prevStats.BytesWritten) / uint64(interval.Seconds())
- result.Progress = float32(float64(100) * (float64(aggBytesCompleted) / float64(aggLen)))
+ if stats.BytesTotal == 0 {
+ stats.Progress = 0
+ } else {
+ stats.Progress = float32(float64(100) * (float64(stats.BytesCompleted) / float64(stats.BytesTotal)))
+ if stats.Progress == 100 && !stats.Completed {
+ stats.Progress = 99.99
+ }
+ }
+ stats.PeersUnique = int32(len(peers))
+ stats.FilesTotal = int32(len(torrents))
- result.peersCount = int64(len(peers))
- result.torrentsCount = len(torrents)
- return result
+ cli.stats = stats
}
-func AddTorrentFile(ctx context.Context, torrentFilePath string, torrentClient *torrent.Client) (mi *metainfo.MetaInfo, err error) {
- mi, err = metainfo.LoadFromFile(torrentFilePath)
- if err != nil {
- return nil, err
- }
- mi.AnnounceList = Trackers
+func (cli *Protocols) Stats() AggStats {
+ cli.statsLock.RLock()
+ defer cli.statsLock.RUnlock()
+ return cli.stats
+}
- t := time.Now()
- _, err = torrentClient.AddTorrent(mi)
- if err != nil {
- return mi, err
- }
- took := time.Since(t)
- if took > 3*time.Second {
- log.Info("[torrent] Check validity", "file", torrentFilePath, "took", took)
+func (cli *Protocols) Close() {
+ //for _, tr := range cli.TorrentClient.Torrents() {
+ // go func() {}()
+ // fmt.Printf("alex: CLOse01: %s\n", tr.Name())
+ // tr.DisallowDataUpload()
+ // fmt.Printf("alex: CLOse02: %s\n", tr.Name())
+ // tr.DisallowDataDownload()
+ // fmt.Printf("alex: CLOse03: %s\n", tr.Name())
+ // ch := t.Closed()
+ // tr.Drop()
+ // <-ch
+ //}
+ cli.TorrentClient.Close()
+ cli.DB.Close()
+ if cli.cfg.CompletionCloser != nil {
+ if err := cli.cfg.CompletionCloser.Close(); err != nil {
+ log.Warn("[Snapshots] CompletionCloser", "err", err)
+ }
}
- return mi, nil
}
-// AddTorrentFiles - adding .torrent files to torrentClient (and checking their hashes), if .torrent file
-// added first time - pieces verification process will start (disk IO heavy) - Progress
-// kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again.
-// Don't need call torrent.VerifyData manually
-func AddTorrentFiles(ctx context.Context, snapshotsDir *dir.Rw, torrentClient *torrent.Client) error {
- files, err := AllTorrentPaths(snapshotsDir.Path)
- if err != nil {
- return err
- }
- for _, torrentFilePath := range files {
- if _, err := AddTorrentFile(ctx, torrentFilePath, torrentClient); err != nil {
- return err
- }
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
- }
+func (cli *Protocols) PeerID() []byte {
+ peerID := cli.TorrentClient.PeerID()
+ return peerID[:]
+}
+func (cli *Protocols) StopSeeding(hash metainfo.Hash) error {
+ t, ok := cli.TorrentClient.Torrent(hash)
+ if !ok {
+ return nil
}
-
+ ch := t.Closed()
+ t.Drop()
+ <-ch
return nil
}
-// ResolveAbsentTorrents - add hard-coded hashes (if client doesn't have) as magnet links and download everything
-func ResolveAbsentTorrents(ctx context.Context, torrentClient *torrent.Client, preverifiedHashes []metainfo.Hash, snapshotDir *dir.Rw, silent bool) error {
- mi := &metainfo.MetaInfo{AnnounceList: Trackers}
- for i := range preverifiedHashes {
- if _, ok := torrentClient.Torrent(preverifiedHashes[i]); ok {
- continue
- }
- magnet := mi.Magnet(&preverifiedHashes[i], nil)
- t, err := torrentClient.AddMagnet(magnet.String())
- if err != nil {
- return err
- }
- t.AllowDataDownload()
- t.AllowDataUpload()
- }
- if !silent {
- ctxLocal, cancel := context.WithCancel(ctx)
- defer cancel()
- go LoggingLoop(ctxLocal, torrentClient)
- }
+type AggStats struct {
+ MetadataReady, FilesTotal int32
+ PeersUnique int32
+ ConnectionsTotal uint64
- for _, t := range torrentClient.Torrents() {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-t.GotInfo():
- if !t.Complete.Bool() {
- t.DownloadAll()
- }
- mi := t.Metainfo()
- if err := CreateTorrentFileIfNotExists(snapshotDir, t.Info(), &mi); err != nil {
- return err
- }
- }
- }
+ Completed bool
+ Progress float32
- return nil
-}
+ BytesCompleted, BytesTotal uint64
-//nolint
-func waitForChecksumVerify(ctx context.Context, torrentClient *torrent.Client) {
- //TODO: tr.VerifyData() - find when to call it
- ctx, cancel := context.WithCancel(ctx)
- defer cancel()
- go func() {
- interval := time.Second * 5
- logEvery := time.NewTicker(interval)
- defer logEvery.Stop()
+ UploadRate, DownloadRate uint64
- for {
- select {
- case <-ctx.Done():
- return
- case <-logEvery.C:
- var aggBytesCompleted, aggLen int64
- for _, t := range torrentClient.Torrents() {
- aggBytesCompleted += t.BytesCompleted()
- aggLen += t.Length()
- }
+ BytesRead uint64
+ BytesWritten uint64
+}
- line := fmt.Sprintf(
- "[torrent] verifying snapshots: %s/%s",
- common2.ByteCount(uint64(aggBytesCompleted)),
- common2.ByteCount(uint64(aggLen)),
- )
- log.Info(line)
- }
- }
- }()
- torrentClient.WaitAll() // wait for checksum verify
+// AddTorrentFile - adding .torrent file to torrentClient (and checking their hashes), if .torrent file
+// added first time - pieces verification process will start (disk IO heavy) - Progress
+// kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again.
+// Don't need call torrent.VerifyData manually
+func AddTorrentFile(ctx context.Context, torrentFilePath string, torrentClient *torrent.Client) (*torrent.Torrent, error) {
+ mi, err := metainfo.LoadFromFile(torrentFilePath)
+ if err != nil {
+ return nil, err
+ }
+ mi.AnnounceList = Trackers
+ t, err := torrentClient.AddTorrent(mi)
+ if err != nil {
+ return nil, err
+ }
+ t.DisallowDataDownload()
+ t.AllowDataUpload()
+ return t, nil
}
func VerifyDtaFiles(ctx context.Context, snapshotDir string) error {
@@ -341,12 +320,12 @@ func VerifyDtaFiles(ctx context.Context, snapshotDir string) error {
err = verifyTorrent(&info, snapshotDir, func(i int, good bool) error {
j++
if !good {
- log.Error("[torrent] Verify hash mismatch", "at piece", i, "file", f)
+ log.Error("[Snapshots] Verify hash mismatch", "at piece", i, "file", f)
return fmt.Errorf("invalid file")
}
select {
case <-logEvery.C:
- log.Info("[torrent] Verify", "Progress", fmt.Sprintf("%.2f%%", 100*float64(j)/float64(totalPieces)))
+ log.Info("[Snapshots] Verify", "Progress", fmt.Sprintf("%.2f%%", 100*float64(j)/float64(totalPieces)))
case <-ctx.Done():
return ctx.Err()
default:
@@ -357,6 +336,6 @@ func VerifyDtaFiles(ctx context.Context, snapshotDir string) error {
return err
}
}
- log.Info("[torrent] Verify succeed")
+ log.Info("[Snapshots] Verify succeed")
return nil
}
diff --git a/cmd/downloader/downloader/grpc_server.go b/cmd/downloader/downloader/grpc_server.go
index 06db650ed4..cfb868f4b8 100644
--- a/cmd/downloader/downloader/grpc_server.go
+++ b/cmd/downloader/downloader/grpc_server.go
@@ -3,15 +3,14 @@ package downloader
import (
"context"
"errors"
- "path/filepath"
- "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/gointerfaces"
proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
prototypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types"
"github.com/ledgerwatch/erigon-lib/kv"
+ "github.com/ledgerwatch/log/v3"
"google.golang.org/protobuf/types/known/emptypb"
)
@@ -23,108 +22,79 @@ var (
_ proto_downloader.DownloaderServer = &GrpcServer{}
)
-func NewGrpcServer(db kv.RwDB, client *Protocols, snapshotDir *dir.Rw, silent bool) (*GrpcServer, error) {
+func NewGrpcServer(db kv.RwDB, client *Protocols, snapshotDir *dir.Rw) (*GrpcServer, error) {
sn := &GrpcServer{
db: db,
t: client,
snapshotDir: snapshotDir,
- silent: silent,
}
return sn, nil
}
-func CreateTorrentFilesAndAdd(ctx context.Context, snapshotDir *dir.Rw, torrentClient *torrent.Client) error {
- if err := BuildTorrentFilesIfNeed(ctx, snapshotDir); err != nil {
- return err
- }
- if err := AddTorrentFiles(ctx, snapshotDir, torrentClient); err != nil {
- return err
- }
- for _, t := range torrentClient.Torrents() {
- t.AllowDataUpload()
- if !t.Complete.Bool() {
- t.AllowDataDownload()
- t.DownloadAll()
- }
- }
- return nil
-}
-
type GrpcServer struct {
proto_downloader.UnimplementedDownloaderServer
t *Protocols
db kv.RwDB
snapshotDir *dir.Rw
- silent bool
}
func (s *GrpcServer) Download(ctx context.Context, request *proto_downloader.DownloadRequest) (*emptypb.Empty, error) {
- infoHashes := make([]metainfo.Hash, len(request.Items))
- for i, it := range request.Items {
+ torrentClient := s.t.TorrentClient
+ mi := &metainfo.MetaInfo{AnnounceList: Trackers}
+ for _, it := range request.Items {
if it.TorrentHash == nil {
- if err := BuildTorrentFileIfNeed(ctx, it.Path, s.snapshotDir); err != nil {
- return nil, err
- }
- metaInfo, err := AddTorrentFile(ctx, filepath.Join(s.snapshotDir.Path, it.Path+".torrent"), s.t.TorrentClient)
+ _, err := BuildTorrentAndAdd(ctx, it.Path, s.snapshotDir, s.t.TorrentClient)
if err != nil {
return nil, err
}
- infoHashes[i] = metaInfo.HashInfoBytes()
- } else {
- infoHashes[i] = gointerfaces.ConvertH160toAddress(it.TorrentHash)
+ continue
}
- }
- if err := ResolveAbsentTorrents(ctx, s.t.TorrentClient, infoHashes, s.snapshotDir, s.silent); err != nil {
- return nil, err
- }
- for _, t := range s.t.TorrentClient.Torrents() {
- t.AllowDataDownload()
- t.AllowDataUpload()
- if !t.Complete.Bool() {
- t.DownloadAll()
+
+ hash := Proto2InfoHash(it.TorrentHash)
+ if _, ok := torrentClient.Torrent(hash); ok {
+ continue
}
+
+ magnet := mi.Magnet(&hash, nil)
+ go func(magnetUrl string) {
+ t, err := torrentClient.AddMagnet(magnetUrl)
+ if err != nil {
+ log.Warn("[downloader] add magnet link", "err", err)
+ return
+ }
+ t.DisallowDataDownload()
+ t.AllowDataUpload()
+ <-t.GotInfo()
+ mi := t.Metainfo()
+ if err := CreateTorrentFileIfNotExists(s.snapshotDir, t.Info(), &mi); err != nil {
+ log.Warn("[downloader] create torrent file", "err", err)
+ return
+ }
+ }(magnet.String())
+
}
return &emptypb.Empty{}, nil
}
func (s *GrpcServer) Stats(ctx context.Context, request *proto_downloader.StatsRequest) (*proto_downloader.StatsReply, error) {
- torrents := s.t.TorrentClient.Torrents()
- reply := &proto_downloader.StatsReply{Completed: true, Torrents: int32(len(torrents))}
+ stats := s.t.Stats()
+ return &proto_downloader.StatsReply{
+ MetadataReady: stats.MetadataReady,
+ FilesTotal: stats.FilesTotal,
- peers := map[torrent.PeerID]struct{}{}
+ Completed: stats.Completed,
+ Progress: stats.Progress,
- for _, t := range torrents {
- select {
- case <-ctx.Done():
- return nil, ctx.Err()
- case <-t.GotInfo():
- reply.BytesCompleted += uint64(t.BytesCompleted())
- reply.BytesTotal += uint64(t.Info().TotalLength())
- reply.Completed = reply.Completed && t.Complete.Bool()
- reply.Connections += uint64(len(t.PeerConns()))
+ PeersUnique: stats.PeersUnique,
+ ConnectionsTotal: stats.ConnectionsTotal,
- for _, peer := range t.PeerConns() {
- peers[peer.PeerID] = struct{}{}
- }
- default:
- reply.Completed = false
- }
- }
-
- reply.Peers = int32(len(peers))
- reply.Progress = int32(100 * (float64(reply.BytesCompleted) / float64(reply.BytesTotal)))
- if reply.Progress == 100 && !reply.Completed {
- reply.Progress = 99
- }
- return reply, nil
+ BytesCompleted: stats.BytesCompleted,
+ BytesTotal: stats.BytesTotal,
+ UploadRate: stats.UploadRate,
+ DownloadRate: stats.DownloadRate,
+ }, nil
}
-func Proto2InfoHashes(in []*prototypes.H160) []metainfo.Hash {
- infoHashes := make([]metainfo.Hash, len(in))
- i := 0
- for _, h := range in {
- infoHashes[i] = gointerfaces.ConvertH160toAddress(h)
- i++
- }
- return infoHashes
+func Proto2InfoHash(in *prototypes.H160) metainfo.Hash {
+ return gointerfaces.ConvertH160toAddress(in)
}
diff --git a/cmd/downloader/downloader/torrentcfg/torrentcfg.go b/cmd/downloader/downloader/torrentcfg/torrentcfg.go
index 6ce8d71b27..d42d16d604 100644
--- a/cmd/downloader/downloader/torrentcfg/torrentcfg.go
+++ b/cmd/downloader/downloader/torrentcfg/torrentcfg.go
@@ -25,6 +25,7 @@ type Cfg struct {
*torrent.ClientConfig
DB kv.RwDB
CompletionCloser io.Closer
+ DownloadSlots int
}
func Default() *torrent.ClientConfig {
@@ -33,10 +34,10 @@ func Default() *torrent.ClientConfig {
// enable dht
torrentConfig.NoDHT = true
//torrentConfig.DisableTrackers = true
- //torrentConfig.DisableWebtorrent = true
+ torrentConfig.DisableWebtorrent = true
//torrentConfig.DisableWebseeds = true
- // Increase default timeouts, because we often run on commodity networks
+ // Reduce defaults - to avoid peers with very bad geography
torrentConfig.MinDialTimeout = 1 * time.Second // default: 3sec
torrentConfig.NominalDialTimeout = 10 * time.Second // default: 20sec
torrentConfig.HandshakesTimeout = 1 * time.Second // default: 4sec
@@ -44,14 +45,15 @@ func Default() *torrent.ClientConfig {
return torrentConfig
}
-func New(snapshotsDir *dir.Rw, verbosity lg.Level, natif nat.Interface, downloadRate, uploadRate datasize.ByteSize, port, maxPeers, connsPerFile int, db kv.RwDB) (*Cfg, error) {
+func New(snapshotsDir *dir.Rw, verbosity lg.Level, natif nat.Interface, downloadRate, uploadRate datasize.ByteSize, port, maxPeers, connsPerFile int, db kv.RwDB, downloadSlots int) (*Cfg, error) {
torrentConfig := Default()
// We would-like to reduce amount of goroutines in Erigon, so reducing next params
- torrentConfig.EstablishedConnsPerTorrent = connsPerFile // default: 50
- torrentConfig.TorrentPeersHighWater = maxPeers // default: 500
- torrentConfig.TorrentPeersLowWater = 50 // default: 50
- torrentConfig.HalfOpenConnsPerTorrent = 25 // default: 25
- torrentConfig.TotalHalfOpenConns = 50 // default: 100
+ torrentConfig.EstablishedConnsPerTorrent = connsPerFile // default: 50
+ torrentConfig.HalfOpenConnsPerTorrent = min(25, connsPerFile) // default: 25
+ torrentConfig.TotalHalfOpenConns = 50 // default: 100
+
+ torrentConfig.TorrentPeersHighWater = maxPeers // default: 500
+ torrentConfig.TorrentPeersLowWater = min(50, maxPeers) // default: 50
torrentConfig.ListenPort = port
torrentConfig.Seed = true
@@ -100,5 +102,12 @@ func New(snapshotsDir *dir.Rw, verbosity lg.Level, natif nat.Interface, download
}
m := storage.NewMMapWithCompletion(snapshotsDir.Path, c)
torrentConfig.DefaultStorage = m
- return &Cfg{ClientConfig: torrentConfig, DB: db, CompletionCloser: m}, nil
+ return &Cfg{ClientConfig: torrentConfig, DB: db, CompletionCloser: m, DownloadSlots: downloadSlots}, nil
+}
+
+func min(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
}
diff --git a/cmd/downloader/downloader/util.go b/cmd/downloader/downloader/util.go
index d06e466c5c..4cdb11d6d1 100644
--- a/cmd/downloader/downloader/util.go
+++ b/cmd/downloader/downloader/util.go
@@ -9,9 +9,11 @@ import (
"io"
"os"
"path/filepath"
+ "runtime"
"sync"
"time"
+ "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/mmap_span"
@@ -21,11 +23,12 @@ import (
"github.com/ledgerwatch/erigon/cmd/downloader/trackers"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
"github.com/ledgerwatch/log/v3"
+ "golang.org/x/sync/semaphore"
)
// Trackers - break down by priority tier
var Trackers = [][]string{
- trackers.First(10, trackers.Best),
+ trackers.First(7, trackers.Best),
//trackers.First(3, trackers.Udp),
//trackers.First(3, trackers.Https),
//trackers.First(10, trackers.Ws),
@@ -107,7 +110,10 @@ func BuildTorrentFileIfNeed(ctx context.Context, originalFileName string, root *
if !errors.Is(err, os.ErrNotExist) {
return err
}
- info, err := BuildInfoBytesForFile(root.Path, originalFileName)
+ info := &metainfo.Info{PieceLength: torrentcfg.DefaultPieceSize}
+ if err := info.BuildFromFilePath(filepath.Join(root.Path, originalFileName)); err != nil {
+ return err
+ }
if err != nil {
return err
}
@@ -118,12 +124,24 @@ func BuildTorrentFileIfNeed(ctx context.Context, originalFileName string, root *
return nil
}
+func BuildTorrentAndAdd(ctx context.Context, originalFileName string, snapshotDir *dir.Rw, client *torrent.Client) (*torrent.Torrent, error) {
+ if err := BuildTorrentFileIfNeed(ctx, originalFileName, snapshotDir); err != nil {
+ return nil, err
+ }
+ torrentFilePath := filepath.Join(snapshotDir.Path, originalFileName+".torrent")
+ t, err := AddTorrentFile(ctx, torrentFilePath, client)
+ if err != nil {
+ return nil, err
+ }
+ return t, nil
+}
+
// BuildTorrentFilesIfNeed - create .torrent files from .seg files (big IO) - if .seg files were added manually
-func BuildTorrentFilesIfNeed(ctx context.Context, root *dir.Rw) error {
+func BuildTorrentFilesIfNeed(ctx context.Context, snapshotDir *dir.Rw) error {
logEvery := time.NewTicker(20 * time.Second)
defer logEvery.Stop()
- files, err := allSegmentFiles(root.Path)
+ files, err := allSegmentFiles(snapshotDir.Path)
if err != nil {
return err
}
@@ -133,14 +151,14 @@ func BuildTorrentFilesIfNeed(ctx context.Context, root *dir.Rw) error {
wg.Add(1)
go func(f string, i int) {
defer wg.Done()
- errs <- BuildTorrentFileIfNeed(ctx, f, root)
+ errs <- BuildTorrentFileIfNeed(ctx, f, snapshotDir)
select {
default:
case <-ctx.Done():
errs <- ctx.Err()
case <-logEvery.C:
- log.Info("[torrent] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i, len(files)))
+ log.Info("[Snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i, len(files)))
}
}(f, i)
}
@@ -156,12 +174,50 @@ func BuildTorrentFilesIfNeed(ctx context.Context, root *dir.Rw) error {
return nil
}
-func BuildInfoBytesForFile(root string, fileName string) (*metainfo.Info, error) {
- info := &metainfo.Info{PieceLength: torrentcfg.DefaultPieceSize}
- if err := info.BuildFromFilePath(filepath.Join(root, fileName)); err != nil {
- return nil, err
+// BuildTorrentsAndAdd - create .torrent files from .seg files (big IO) - if .seg files were placed manually to snapshotDir
+func BuildTorrentsAndAdd(ctx context.Context, snapshotDir *dir.Rw, client *torrent.Client) error {
+ logEvery := time.NewTicker(20 * time.Second)
+ defer logEvery.Stop()
+ files, err := allSegmentFiles(snapshotDir.Path)
+ if err != nil {
+ return err
+ }
+ errs := make(chan error, len(files)*2)
+ wg := &sync.WaitGroup{}
+ workers := runtime.GOMAXPROCS(-1) - 1
+ if workers < 1 {
+ workers = 1
+ }
+ var sem = semaphore.NewWeighted(int64(workers))
+ for i, f := range files {
+ wg.Add(1)
+ if err := sem.Acquire(ctx, 1); err != nil {
+ return err
+ }
+ go func(f string, i int) {
+ defer sem.Release(1)
+ defer wg.Done()
+
+ select {
+ case <-ctx.Done():
+ errs <- ctx.Err()
+ default:
+ }
+
+ _, err := BuildTorrentAndAdd(ctx, f, snapshotDir, client)
+ errs <- err
+ }(f, i)
}
- return info, nil
+ go func() {
+ wg.Wait()
+ close(errs)
+ }()
+ for err := range errs {
+ if err != nil {
+ return err
+ }
+ }
+ return nil
}
func CreateTorrentFileIfNotExists(root *dir.Rw, info *metainfo.Info, mi *metainfo.MetaInfo) error {
diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go
index 95c1fcafaa..8cfeb52b58 100644
--- a/cmd/downloader/main.go
+++ b/cmd/downloader/main.go
@@ -43,6 +43,7 @@ var (
natSetting string
torrentVerbosity string
downloadRateStr, uploadRateStr string
+ torrentDownloadSlots int
torrentPort int
torrentMaxPeers int
torrentConnsPerFile int
@@ -63,6 +64,7 @@ func init() {
rootCmd.Flags().IntVar(&torrentPort, "torrent.port", utils.TorrentPortFlag.Value, utils.TorrentPortFlag.Usage)
rootCmd.Flags().IntVar(&torrentMaxPeers, "torrent.maxpeers", utils.TorrentMaxPeersFlag.Value, utils.TorrentMaxPeersFlag.Usage)
rootCmd.Flags().IntVar(&torrentConnsPerFile, "torrent.conns.perfile", utils.TorrentConnsPerFileFlag.Value, utils.TorrentConnsPerFileFlag.Usage)
+ rootCmd.Flags().IntVar(&torrentDownloadSlots, "torrent.download.slots", utils.TorrentDownloadSlotsFlag.Value, utils.TorrentDownloadSlotsFlag.Usage)
withDataDir(printTorrentHashes)
printTorrentHashes.PersistentFlags().BoolVar(&forceRebuild, "rebuild", false, "Force re-create .torrent files")
@@ -148,7 +150,7 @@ func Downloader(ctx context.Context) error {
return err
}
- cfg, err := torrentcfg.New(snapshotDir, torrentLogLevel, natif, downloadRate, uploadRate, torrentPort, torrentMaxPeers, torrentConnsPerFile, db)
+ cfg, err := torrentcfg.New(snapshotDir, torrentLogLevel, natif, downloadRate, uploadRate, torrentPort, torrentMaxPeers, torrentConnsPerFile, db, torrentDownloadSlots)
if err != nil {
return err
}
@@ -160,13 +162,11 @@ func Downloader(ctx context.Context) error {
}
defer protocols.Close()
log.Info("[torrent] Start", "my peerID", fmt.Sprintf("%x", protocols.TorrentClient.PeerID()))
- if err = downloader.CreateTorrentFilesAndAdd(ctx, snapshotDir, protocols.TorrentClient); err != nil {
- return fmt.Errorf("CreateTorrentFilesAndAdd: %w", err)
+ if err := protocols.Start(ctx, false); err != nil {
+ return err
}
- go downloader.LoggingLoop(ctx, protocols.TorrentClient)
-
- bittorrentServer, err := downloader.NewGrpcServer(protocols.DB, protocols, snapshotDir, true)
+ bittorrentServer, err := downloader.NewGrpcServer(protocols.DB, protocols, snapshotDir)
if err != nil {
return fmt.Errorf("new server: %w", err)
}
diff --git a/cmd/observer/observer/crawler.go b/cmd/observer/observer/crawler.go
index 07005dc368..fc1c02d954 100644
--- a/cmd/observer/observer/crawler.go
+++ b/cmd/observer/observer/crawler.go
@@ -5,6 +5,9 @@ import (
"crypto/ecdsa"
"errors"
"fmt"
+ "sync/atomic"
+ "time"
+
"github.com/ledgerwatch/erigon/cmd/observer/database"
"github.com/ledgerwatch/erigon/cmd/observer/utils"
"github.com/ledgerwatch/erigon/core/forkid"
@@ -12,8 +15,6 @@ import (
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/log/v3"
"golang.org/x/sync/semaphore"
- "sync/atomic"
- "time"
)
type Crawler struct {
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 9c805d7c50..e58cccb147 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -648,6 +648,11 @@ var (
Value: "4mb",
Usage: "bytes per second, example: 32mb",
}
+ TorrentDownloadSlotsFlag = cli.IntFlag{
+ Name: "torrent.download.slots",
+ Value: 3,
+ Usage: "amount of files to download in parallel. If network has enough seeders 1-3 slot enough, if network has lack of seeders increase to 5-7 (too big value will slow down everything).",
+ }
TorrentPortFlag = cli.IntFlag{
Name: "torrent.port",
Value: 42069,
@@ -1399,6 +1404,7 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *node.Config, cfg *ethconfig.Conf
ctx.GlobalInt(TorrentMaxPeersFlag.Name),
ctx.GlobalInt(TorrentConnsPerFileFlag.Name),
db,
+ ctx.GlobalInt(TorrentDownloadSlotsFlag.Name),
)
if err != nil {
panic(err)
diff --git a/eth/backend.go b/eth/backend.go
index 7e946ce153..58caf70018 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -21,7 +21,6 @@ import (
"context"
"errors"
"fmt"
- "google.golang.org/protobuf/types/known/emptypb"
"math/big"
"os"
"path/filepath"
@@ -30,6 +29,8 @@ import (
"sync"
"time"
+ "google.golang.org/protobuf/types/known/emptypb"
+
"github.com/holiman/uint256"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/direct"
@@ -324,10 +325,11 @@ func New(stack *node.Node, config *ethconfig.Config, txpoolCfg txpool2.Config, l
if err != nil {
return nil, err
}
- if err = downloader.CreateTorrentFilesAndAdd(ctx, config.SnapshotDir, backend.downloadProtocols.TorrentClient); err != nil {
- return nil, fmt.Errorf("CreateTorrentFilesAndAdd: %w", err)
+ if err := backend.downloadProtocols.Start(ctx, true); err != nil {
+ return nil, err
}
- bittorrentServer, err := downloader.NewGrpcServer(backend.downloadProtocols.DB, backend.downloadProtocols, config.SnapshotDir, false)
+
+ bittorrentServer, err := downloader.NewGrpcServer(backend.downloadProtocols.DB, backend.downloadProtocols, config.SnapshotDir)
if err != nil {
return nil, fmt.Errorf("new server: %w", err)
}
diff --git a/eth/stagedsync/stage_headers.go b/eth/stagedsync/stage_headers.go
index 549dca13a9..491f680d75 100644
--- a/eth/stagedsync/stage_headers.go
+++ b/eth/stagedsync/stage_headers.go
@@ -1193,71 +1193,60 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
// for MVP we sync with Downloader only once, in future will send new snapshots also
func WaitForDownloader(ctx context.Context, tx kv.RwTx, cfg HeadersCfg) error {
snapshotsCfg := snapshothashes.KnownConfig(cfg.chainConfig.ChainName)
- checkStatsEvery := time.NewTicker(5 * time.Second)
- defer checkStatsEvery.Stop()
// send all hashes to the Downloader service
preverified := snapshotsCfg.Preverified
- var prevBytesCompleted uint64
- logEvery := time.NewTicker(logInterval)
- defer logEvery.Stop()
+ req := &proto_downloader.DownloadRequest{Items: make([]*proto_downloader.DownloadItem, len(preverified))}
+ i := 0
for _, p := range preverified {
- req := &proto_downloader.DownloadRequest{Items: make([]*proto_downloader.DownloadItem, 1)}
- req.Items[0] = &proto_downloader.DownloadItem{
+ req.Items[i] = &proto_downloader.DownloadItem{
TorrentHash: downloadergrpc.String2Proto(p.Hash),
Path: p.Name,
}
- for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
- }
- if _, err := cfg.snapshotDownloader.Download(ctx, req); err != nil {
- log.Error("[Snapshots] Can't call downloader", "err", err)
- time.Sleep(10 * time.Second)
- continue
- }
- break
+ i++
+ }
+ log.Info("[Snapshots] Fetching torrent files metadata")
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
}
-
- if reply, err := cfg.snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}); err != nil {
- log.Warn("Error while waiting for snapshots progress", "err", err)
- } else if reply.Completed {
+ if _, err := cfg.snapshotDownloader.Download(ctx, req); err != nil {
+ log.Error("[Snapshots] call downloader", "err", err)
+ time.Sleep(10 * time.Second)
continue
}
+ break
+ }
+ logEvery := time.NewTicker(logInterval / 3)
+ defer logEvery.Stop()
- // Print download progress until all segments are available
- Loop:
- for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-checkStatsEvery.C:
- if reply, err := cfg.snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}); err != nil {
- log.Warn("Error while waiting for snapshots progress", "err", err)
- } else if reply.Completed {
- break Loop
- }
- case <-logEvery.C:
- if reply, err := cfg.snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}); err != nil {
- log.Warn("Error while waiting for snapshots progress", "err", err)
- } else if reply.Completed {
- break Loop
- } else {
- readBytesPerSec := (reply.BytesCompleted - prevBytesCompleted) / uint64(logInterval.Seconds())
- // writeBytesPerSec += (reply.BytesWritten - prevBytesWritten) / int64(logInterval.Seconds())
-
- //readiness := 100 * (float64(reply.BytesCompleted) / float64(reply.BytesTotal))
- log.Info("[Snapshots] download", //"progress", fmt.Sprintf("%.2f%%", readiness),
- "progress", libcommon.ByteCount(reply.BytesCompleted),
- "download", libcommon.ByteCount(readBytesPerSec)+"/s",
- "torrent_peers", reply.Peers,
- "connections", reply.Connections,
- // "upload", libcommon.ByteCount(writeBytesPerSec)+"/s",
- )
- prevBytesCompleted = reply.BytesCompleted
+ // Print download progress until all segments are available
+Loop:
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-logEvery.C:
+ if stats, err := cfg.snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}); err != nil {
+ log.Warn("Error while waiting for snapshots progress", "err", err)
+ } else if stats.Completed {
+ break Loop
+ } else {
+ if stats.MetadataReady < stats.FilesTotal {
+ log.Info(fmt.Sprintf("[Snapshots] Waiting for torrents metadata: %d/%d", stats.MetadataReady, stats.FilesTotal))
+ continue
}
+
+ log.Info("[Snapshots] download",
+ "progress", fmt.Sprintf("%.2f%% %s/%s", stats.Progress, libcommon.ByteCount(stats.BytesCompleted), libcommon.ByteCount(stats.BytesTotal)),
+ "download", libcommon.ByteCount(stats.DownloadRate)+"/s",
+ "upload", libcommon.ByteCount(stats.UploadRate)+"/s",
+ "peers", stats.PeersUnique,
+ "connections", stats.ConnectionsTotal,
+ "files", stats.FilesTotal,
+ )
}
}
}
diff --git a/go.mod b/go.mod
index 2b206802ae..aa641ae8c2 100644
--- a/go.mod
+++ b/go.mod
@@ -35,14 +35,14 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
- github.com/ledgerwatch/erigon-lib v0.0.0-20220425125405-5dc5ab7fe16b
+ github.com/ledgerwatch/erigon-lib v0.0.0-20220427024056-791f257bd412
github.com/ledgerwatch/log/v3 v3.4.1
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/pelletier/go-toml v1.9.5
github.com/pelletier/go-toml/v2 v2.0.0-beta.8
github.com/quasilyte/go-ruleguard/dsl v0.3.19
github.com/rs/cors v1.8.2
- github.com/shirou/gopsutil/v3 v3.22.2
+ github.com/shirou/gopsutil/v3 v3.22.3
github.com/spf13/cobra v1.4.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.1
@@ -61,12 +61,12 @@ require (
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150
golang.org/x/time v0.0.0-20220411224347-583f2d630306
- google.golang.org/grpc v1.45.0
+ google.golang.org/grpc v1.46.0
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0
google.golang.org/protobuf v1.28.0
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6
- modernc.org/sqlite v1.14.2-0.20211125151325-d4ed92c0a70f
+ modernc.org/sqlite v1.17.0
pgregory.net/rapid v0.4.7
)
@@ -150,26 +150,26 @@ require (
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/tidwall/btree v0.7.2-0.20211211132910-4215444137fc // indirect
- github.com/tklauser/go-sysconf v0.3.9 // indirect
- github.com/tklauser/numcpus v0.3.0 // indirect
+ github.com/tklauser/go-sysconf v0.3.10 // indirect
+ github.com/tklauser/numcpus v0.4.0 // indirect
github.com/valyala/fastrand v1.1.0 // indirect
github.com/valyala/histogram v1.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/image v0.0.0-20200927104501-e162460cd6b5 // indirect
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
- golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
lukechampine.com/uint128 v1.1.1 // indirect
- modernc.org/cc/v3 v3.35.18 // indirect
- modernc.org/ccgo/v3 v3.12.73 // indirect
- modernc.org/libc v1.11.82 // indirect
+ modernc.org/cc/v3 v3.35.26 // indirect
+ modernc.org/ccgo/v3 v3.16.2 // indirect
+ modernc.org/libc v1.15.0 // indirect
modernc.org/mathutil v1.4.1 // indirect
- modernc.org/memory v1.0.5 // indirect
+ modernc.org/memory v1.0.7 // indirect
modernc.org/opt v0.1.1 // indirect
modernc.org/strutil v1.1.1 // indirect
modernc.org/token v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 728394e6f6..12521e00b2 100644
--- a/go.sum
+++ b/go.sum
@@ -174,8 +174,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -217,7 +217,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fjl/gencodec v0.0.0-20191126094850-e283372f291f h1:Y/gg/utVetS+WS6htAKCTDralkm/8hLIIUAtLFdbdQ8=
@@ -454,8 +454,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220425125405-5dc5ab7fe16b h1:OhhQT6ynUpkFqfisRCRDkTFhGd2+8Hom7jKDey6O17w=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220425125405-5dc5ab7fe16b/go.mod h1:0VKhW10UjEr7I6DaV+N0KNbXvMygC99qmRl2CfaN9gw=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220427024056-791f257bd412 h1:IAh4RBG0kx8ZtwbZBfwcj6L5iGR0X1orIgeStOdpH7Y=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220427024056-791f257bd412/go.mod h1:Z6hOzGMPdbzDcCs+EV5CEl/a6zOpgXqXL0K5956iXUc=
github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc=
github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
@@ -476,8 +476,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
-github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
+github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -656,8 +656,8 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5P
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
-github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
+github.com/shirou/gopsutil/v3 v3.22.3 h1:UebRzEomgMpv61e3hgD1tGooqX5trFbdU/ehphbHd00=
+github.com/shirou/gopsutil/v3 v3.22.3/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -704,10 +704,10 @@ github.com/tidwall/btree v0.7.2-0.20211211132910-4215444137fc/go.mod h1:LGm8L/DZ
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
-github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
-github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
-github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
+github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
+github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
+github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
+github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/torquem-ch/mdbx-go v0.23.2 h1:7axXl0leix2v8No+mRzeTV32hJrV1817aKhh+hTEpC8=
github.com/torquem-ch/mdbx-go v0.23.2/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E=
@@ -859,8 +859,9 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -929,19 +930,19 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210326220804-49726bf1d181/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211030160813-b3129d9d1021/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1095,8 +1096,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
-google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
+google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -1164,8 +1165,13 @@ modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g
modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.18 h1:rMZhRcWrba0y3nVmdiQ7kxAgOOSq2m2f2VzjHLgEs6U=
modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.20/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.22/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.24/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
+modernc.org/cc/v3 v3.35.25/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
+modernc.org/cc/v3 v3.35.26 h1:S4B+fg6/9krLtfZ9lr7pfKiESopiv+Sm6lUUI3oc0fY=
+modernc.org/cc/v3 v3.35.26/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60=
modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw=
modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI=
@@ -1195,11 +1201,30 @@ modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7I
modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU=
modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc=
modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM=
-modernc.org/ccgo/v3 v3.12.65/go.mod h1:D6hQtKxPNZiY6wDBtehSGKFKmyXn53F8nGTpH+POmS4=
modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ=
modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84=
-modernc.org/ccgo/v3 v3.12.73 h1:AMk4wEpzWjpODXohKvvnlwLob4Xk8tq3we6CwYh88mA=
modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ=
+modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY=
+modernc.org/ccgo/v3 v3.12.84/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w=
+modernc.org/ccgo/v3 v3.12.86/go.mod h1:dN7S26DLTgVSni1PVA3KxxHTcykyDurf3OgUzNqTSrU=
+modernc.org/ccgo/v3 v3.12.90/go.mod h1:obhSc3CdivCRpYZmrvO88TXlW0NvoSVvdh/ccRjJYko=
+modernc.org/ccgo/v3 v3.12.92/go.mod h1:5yDdN7ti9KWPi5bRVWPl8UNhpEAtCjuEE7ayQnzzqHA=
+modernc.org/ccgo/v3 v3.13.1/go.mod h1:aBYVOUfIlcSnrsRVU8VRS35y2DIfpgkmVkYZ0tpIXi4=
+modernc.org/ccgo/v3 v3.15.9/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
+modernc.org/ccgo/v3 v3.15.10/go.mod h1:wQKxoFn0ynxMuCLfFD09c8XPUCc8obfchoVR9Cn0fI8=
+modernc.org/ccgo/v3 v3.15.12/go.mod h1:VFePOWoCd8uDGRJpq/zfJ29D0EVzMSyID8LCMWYbX6I=
+modernc.org/ccgo/v3 v3.15.14/go.mod h1:144Sz2iBCKogb9OKwsu7hQEub3EVgOlyI8wMUPGKUXQ=
+modernc.org/ccgo/v3 v3.15.15/go.mod h1:z5qltXjU4PJl0pE5nhYQCvA9DhPHiWsl5GWl89+NSYE=
+modernc.org/ccgo/v3 v3.15.16/go.mod h1:XbKRMeMWMdq712Tr5ECgATYMrzJ+g9zAZEj2ktzBe24=
+modernc.org/ccgo/v3 v3.15.17/go.mod h1:bofnFkpRFf5gLY+mBZIyTW6FEcp26xi2lgOFk2Rlvs0=
+modernc.org/ccgo/v3 v3.15.19/go.mod h1:TDJj+DxR26pkDteH2E5WQDj/xlmtsX7JdzkJkaZhOVU=
+modernc.org/ccgo/v3 v3.16.0/go.mod h1:w55kPTAqvRMAYS3Lwij6qhqIuBEYS3Z8QtDkjD8cnik=
+modernc.org/ccgo/v3 v3.16.1/go.mod h1:w55kPTAqvRMAYS3Lwij6qhqIuBEYS3Z8QtDkjD8cnik=
+modernc.org/ccgo/v3 v3.16.2 h1:FUklsEMps3Y2heuTOmn/l6mv83nQgCjW3nsU+1JXzuQ=
+modernc.org/ccgo/v3 v3.16.2/go.mod h1:w55kPTAqvRMAYS3Lwij6qhqIuBEYS3Z8QtDkjD8cnik=
+modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
+modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
+modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
@@ -1231,31 +1256,47 @@ modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw=
modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M=
modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18=
modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8=
-modernc.org/libc v1.11.70/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0=
-modernc.org/libc v1.11.82 h1:CSl/6n4odvPYWKKqBtFb8e0ZWVTjxDqwxTjaoee9V7E=
modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI=
+modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE=
+modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY=
+modernc.org/libc v1.11.88/go.mod h1:h3oIVe8dxmTcchcFuCcJ4nAWaoiwzKCdv82MM0oiIdQ=
+modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c=
+modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI=
+modernc.org/libc v1.12.0/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ=
+modernc.org/libc v1.14.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
+modernc.org/libc v1.14.2/go.mod h1:MX1GBLnRLNdvmK9azU9LCxZ5lMyhrbEMK8rG3X/Fe34=
+modernc.org/libc v1.14.3/go.mod h1:GPIvQVOVPizzlqyRX3l756/3ppsAgg1QgPxjr5Q4agQ=
+modernc.org/libc v1.14.6/go.mod h1:2PJHINagVxO4QW/5OQdRrvMYo+bm5ClpUFfyXCYl9ak=
+modernc.org/libc v1.14.7/go.mod h1:f8xfWXW8LW41qb4X5+huVQo5dcfPlq7Cbny2TDheMv0=
+modernc.org/libc v1.14.8/go.mod h1:9+JCLb1MWSY23smyOpIPbd5ED+rSS/ieiDWUpdyO3mo=
+modernc.org/libc v1.14.10/go.mod h1:y1MtIWhwpJFpLYm6grAThtuXJKEsY6xkdZmXbRngIdo=
+modernc.org/libc v1.14.12/go.mod h1:fJdoe23MHu2ruPQkFPPqCpToDi5cckzsbmkI6Ez0LqQ=
+modernc.org/libc v1.15.0 h1:/CTHjQ1QO5mkLDeQICuA9Vh0YvhQTMqtCF2urQTaod8=
+modernc.org/libc v1.15.0/go.mod h1:H1OKCu+NYa9+uQG8WsP7DndMBP61I4PWH8ivWhbdoWQ=
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8=
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc=
-modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14=
modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM=
+modernc.org/memory v1.0.6/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
+modernc.org/memory v1.0.7 h1:UE3cxTRFa5tfUibAV7Jqq8P7zRY0OlJg+yWVIIaluEE=
+modernc.org/memory v1.0.7/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.14.2-0.20211125151325-d4ed92c0a70f h1:yQwkmqKCIgLzFIfjfPfZAAxLZernckpo7zGTv37Ahv0=
-modernc.org/sqlite v1.14.2-0.20211125151325-d4ed92c0a70f/go.mod h1:YT5XFRKOueohjppHO4cHb54eQlnaUGsZMHoryaCpNo4=
+modernc.org/sqlite v1.17.0 h1:yF5JlxCzQOn2WzyfGAPvHbMNx98ifXLno7a97qggXjE=
+modernc.org/sqlite v1.17.0/go.mod h1:yMNaeEckF88G+PcfRcZRwGE+XnBkzWl/j15bPsDm4QM=
modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs=
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
-modernc.org/tcl v1.8.13 h1:V0sTNBw0Re86PvXZxuCub3oO9WrSTqALgrwNZNvLFGw=
-modernc.org/tcl v1.8.13/go.mod h1:V+q/Ef0IJaNUSECieLU4o+8IScapxnMyFV6i/7uQlAY=
+modernc.org/tcl v1.12.0 h1:Mw2Ukszv5qZbwk/wC9HkDjxhPD4exnd/7/zVxqrB4rY=
+modernc.org/tcl v1.12.0/go.mod h1:9zyAWctRV6IAkMTBeGLyYYqcBrTlVy3ubqiY3dzMfwI=
modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.2.19 h1:BGyRFWhDVn5LFS5OcX4Yd/MlpRTOc7hOPTdcIpCiUao=
-modernc.org/z v1.2.19/go.mod h1:+ZpP0pc4zz97eukOzW3xagV/lS82IpPN9NGG5pNF9vY=
+modernc.org/z v1.4.0 h1:IpbQb3bOi5Fz17UVGU/mSor8sKIu/7pdCsmGGnQHcxs=
+modernc.org/z v1.4.0/go.mod h1:x6vxerH3hHCPGA3DAM5pERRzuyJEO4UGVfdQC4NZYl0=
pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=
pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go
index a252294ba9..276b761370 100644
--- a/turbo/cli/default_flags.go
+++ b/turbo/cli/default_flags.go
@@ -76,6 +76,7 @@ var DefaultFlags = []cli.Flag{
utils.TorrentPortFlag,
utils.TorrentMaxPeersFlag,
utils.TorrentConnsPerFileFlag,
+ utils.TorrentDownloadSlotsFlag,
utils.TorrentUploadRateFlag,
utils.TorrentDownloadRateFlag,
utils.TorrentVerbosityFlag,
--
GitLab