From ecb10e854853d79c5eabebe8926ae4631d5bd162 Mon Sep 17 00:00:00 2001
From: Alex Sharov <AskAlexSharov@gmail.com>
Date: Tue, 14 Dec 2021 17:13:17 +0700
Subject: [PATCH] Snapshots download and seed (#3117)
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* Squashed 'interfaces/' content from commit e5b1945d0
git-subtree-dir: interfaces
git-subtree-split: e5b1945d02da7a7f00e2289034ee90a6edd60184
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
* save
---
.gitmodules | 6 +
cmd/downloader/downloader/downloader.go | 291 +++++++++++++
cmd/downloader/downloader/logger.go | 74 ++++
cmd/downloader/downloader/server.go | 121 ++++++
cmd/downloader/downloader/util.go | 177 ++++++++
cmd/downloader/downloadergrpc/client.go | 64 +++
.../generator/commands/metainfo_hash.go | 46 ---
cmd/downloader/{root.go => main.go} | 172 +++++---
cmd/downloader/seeder/main.go | 150 -------
cmd/downloader/trackers/embed.go | 39 ++
cmd/downloader/trackers/trackerslist | 1 +
cmd/hack/hack.go | 3 +-
cmd/integration/commands/stages.go | 30 +-
cmd/rpcdaemon/cli/config.go | 3 +-
cmd/rpcdaemon/interfaces/interfaces.go | 7 +
cmd/sentry/commands/sentry.go | 93 -----
cmd/sentry/main.go | 88 +++-
cmd/sentry/{download => sentry}/broadcast.go | 2 +-
cmd/sentry/{download => sentry}/downloader.go | 6 +-
cmd/sentry/{download => sentry}/sentry.go | 2 +-
cmd/sentry/{download => sentry}/sentry_api.go | 2 +-
.../{download => sentry}/sentry_test.go | 2 +-
cmd/utils/flags.go | 92 +++--
consensus/aura/consensusconfig/embed.go | 7 +-
eth/backend.go | 72 ++--
eth/ethconfig/config.go | 3 +-
eth/stagedsync/stage_headers.go | 357 ++++++++++------
go.mod | 4 +-
go.sum | 4 +-
node/config.go | 2 +
params/config.go | 29 +-
params/networkname/network_name.go | 13 +
params/snapshots.go | 23 --
turbo/cli/default_flags.go | 1 +
turbo/node/node.go | 11 +-
turbo/snapshotsync/block_reader.go | 128 ++++++
turbo/snapshotsync/block_snapshots.go | 113 +++---
turbo/snapshotsync/block_snapshots_test.go | 5 +-
turbo/snapshotsync/bodies_snapshot.go | 1 -
turbo/snapshotsync/build_infobytes.go | 46 ---
turbo/snapshotsync/client.go | 6 +-
turbo/snapshotsync/const.go | 359 ----------------
turbo/snapshotsync/downloader.go | 382 ------------------
turbo/snapshotsync/logger.go | 39 --
turbo/snapshotsync/postprocessing.go | 336 ---------------
turbo/snapshotsync/postprocessing_test.go | 107 -----
turbo/snapshotsync/server.go | 220 ----------
turbo/snapshotsync/snapshot_mode.go | 2 +
turbo/snapshotsync/snapshot_mode_test.go | 2 +
turbo/snapshotsync/snapshothashes/embed.go | 49 +++
.../snapshothashes/erigon-snapshots | 1 +
turbo/snapshotsync/wrapdb.go | 2 +
turbo/stages/headerdownload/header_algos.go | 6 +-
turbo/stages/mock_sentry.go | 15 +-
turbo/stages/stageloop.go | 11 +-
55 files changed, 1643 insertions(+), 2184 deletions(-)
create mode 100644 cmd/downloader/downloader/downloader.go
create mode 100644 cmd/downloader/downloader/logger.go
create mode 100644 cmd/downloader/downloader/server.go
create mode 100644 cmd/downloader/downloader/util.go
create mode 100644 cmd/downloader/downloadergrpc/client.go
delete mode 100644 cmd/downloader/generator/commands/metainfo_hash.go
rename cmd/downloader/{root.go => main.go} (50%)
delete mode 100644 cmd/downloader/seeder/main.go
create mode 100644 cmd/downloader/trackers/embed.go
create mode 160000 cmd/downloader/trackers/trackerslist
delete mode 100644 cmd/sentry/commands/sentry.go
rename cmd/sentry/{download => sentry}/broadcast.go (99%)
rename cmd/sentry/{download => sentry}/downloader.go (99%)
rename cmd/sentry/{download => sentry}/sentry.go (99%)
rename cmd/sentry/{download => sentry}/sentry_api.go (99%)
rename cmd/sentry/{download => sentry}/sentry_test.go (99%)
create mode 100644 params/networkname/network_name.go
delete mode 100644 params/snapshots.go
delete mode 100644 turbo/snapshotsync/bodies_snapshot.go
delete mode 100644 turbo/snapshotsync/build_infobytes.go
delete mode 100644 turbo/snapshotsync/downloader.go
delete mode 100644 turbo/snapshotsync/logger.go
delete mode 100644 turbo/snapshotsync/postprocessing.go
delete mode 100644 turbo/snapshotsync/postprocessing_test.go
delete mode 100644 turbo/snapshotsync/server.go
create mode 100644 turbo/snapshotsync/snapshothashes/embed.go
create mode 160000 turbo/snapshotsync/snapshothashes/erigon-snapshots
diff --git a/.gitmodules b/.gitmodules
index 5bc6224029..10c5825620 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,9 @@
[submodule "libmdbx"]
path = libmdbx
url = https://github.com/erthink/libmdbx
+[submodule "turbo/snapshotsync/snapshothashes/erigon-snapshots"]
+ path = turbo/snapshotsync/snapshothashes/erigon-snapshots
+ url = https://github.com/ledgerwatch/erigon-snapshot.git
+[submodule "cmd/downloader/trackers/trackerslist"]
+ path = cmd/downloader/trackers/trackerslist
+ url = https://github.com/ngosang/trackerslist.git
diff --git a/cmd/downloader/downloader/downloader.go b/cmd/downloader/downloader/downloader.go
new file mode 100644
index 0000000000..604db33e48
--- /dev/null
+++ b/cmd/downloader/downloader/downloader.go
@@ -0,0 +1,291 @@
+package downloader
+
+import (
+ "context"
+ "fmt"
+ "path/filepath"
+ "sync"
+ "time"
+
+ lg "github.com/anacrolix/log"
+ "github.com/anacrolix/torrent"
+ "github.com/anacrolix/torrent/metainfo"
+ "github.com/anacrolix/torrent/storage"
+ "github.com/dustin/go-humanize"
+ "github.com/ledgerwatch/erigon-lib/kv"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
+ "github.com/ledgerwatch/log/v3"
+)
+
+type Client struct {
+ Cli *torrent.Client
+ pieceCompletionStore storage.PieceCompletion
+}
+
+func New(snapshotsDir string, seeding bool, peerID string) (*Client, error) {
+ torrentConfig := DefaultTorrentConfig()
+ torrentConfig.Seed = seeding
+ torrentConfig.DataDir = snapshotsDir
+ torrentConfig.UpnpID = torrentConfig.UpnpID + "leecher"
+ torrentConfig.PeerID = peerID
+
+ progressStore, err := storage.NewBoltPieceCompletion(snapshotsDir)
+ if err != nil {
+ panic(err)
+ }
+ torrentConfig.DefaultStorage = storage.NewMMapWithCompletion(snapshotsDir, progressStore)
+
+ torrentClient, err := torrent.NewClient(torrentConfig)
+ if err != nil {
+ log.Error("Fail to start torrnet client", "err", err)
+ return nil, fmt.Errorf("fail to start: %w", err)
+ }
+
+ log.Info(fmt.Sprintf("Seeding: %t, my peerID: %x", seeding, torrentClient.PeerID()))
+
+ return &Client{
+ Cli: torrentClient,
+ pieceCompletionStore: progressStore,
+ }, nil
+}
+
+func DefaultTorrentConfig() *torrent.ClientConfig {
+ torrentConfig := torrent.NewDefaultClientConfig()
+ torrentConfig.ListenPort = 0
+ // debug
+ torrentConfig.Debug = false
+ torrentConfig.Logger = NewAdapterLogger()
+ torrentConfig.Logger = torrentConfig.Logger.FilterLevel(lg.Debug)
+
+ // enable dht
+ torrentConfig.NoDHT = true
+ torrentConfig.DisableTrackers = false
+ //torrentConfig.DisableWebtorrent = true
+ //torrentConfig.DisableWebseeds = true
+
+ // Increase default timeouts, because we often run on commodity networks
+ torrentConfig.MinDialTimeout = 6 * time.Second // default: 3sec
+ torrentConfig.NominalDialTimeout = 20 * time.Second // default: 20sec
+ torrentConfig.HandshakesTimeout = 8 * time.Second // default: 4sec
+
+ //torrentConfig.MinPeerExtensions.SetBit(peer_protocol.ExtensionBitFast, true)
+ return torrentConfig
+}
+
+func (cli *Client) SavePeerID(db kv.Putter) error {
+ return db.Put(kv.BittorrentInfo, []byte(kv.BittorrentPeerID), cli.PeerID())
+}
+
+func (cli *Client) Close() {
+ cli.pieceCompletionStore.Close()
+ cli.Cli.Close()
+}
+
+func (cli *Client) PeerID() []byte {
+ peerID := cli.Cli.PeerID()
+ return peerID[:]
+}
+
+func MainLoop(ctx context.Context, torrentClient *torrent.Client) {
+ interval := time.Second * 5
+ logEvery := time.NewTicker(interval)
+ defer logEvery.Stop()
+ for {
+ var prevBytesReadUsefulData, aggByteRate int64
+ select {
+ case <-ctx.Done():
+ return
+ case <-logEvery.C:
+ torrents := torrentClient.Torrents()
+ allComplete := true
+ gotInfo := 0
+ for _, t := range torrents {
+ select {
+ case <-t.GotInfo(): // all good
+ gotInfo++
+ default:
+ t.AllowDataUpload()
+ t.AllowDataDownload()
+ }
+ allComplete = allComplete && t.Complete.Bool()
+ }
+ if gotInfo < len(torrents) {
+ log.Info(fmt.Sprintf("[torrent] Waiting for torrents metadata: %d/%d", gotInfo, len(torrents)))
+ continue
+ }
+ if allComplete {
+ peers := map[torrent.PeerID]struct{}{}
+ for _, t := range torrentClient.Torrents() {
+ for _, peer := range t.PeerConns() {
+ peers[peer.PeerID] = struct{}{}
+ }
+ }
+ log.Info("[torrent] Seeding", "peers", len(peers), "torrents", len(torrents))
+ continue
+ }
+
+ var aggBytesCompleted, aggLen int64
+ //var aggCompletedPieces, aggNumPieces, aggPartialPieces int
+ peers := map[torrent.PeerID]*torrent.PeerConn{}
+
+ for _, t := range torrents {
+ stats := t.Stats()
+ /*
+ var completedPieces, partialPieces int
+ psrs := t.PieceStateRuns()
+ for _, r := range psrs {
+ if r.Complete {
+ completedPieces += r.Length
+ }
+ if r.Partial {
+ partialPieces += r.Length
+ }
+ }
+ aggCompletedPieces += completedPieces
+ aggPartialPieces += partialPieces
+ aggNumPieces = t.NumPieces()
+ */
+
+ byteRate := int64(time.Second)
+ bytesReadUsefulData := stats.BytesReadUsefulData.Int64()
+ byteRate *= stats.BytesReadUsefulData.Int64() - prevBytesReadUsefulData
+ byteRate /= int64(interval)
+ aggByteRate += byteRate
+
+ prevBytesReadUsefulData = bytesReadUsefulData
+ aggBytesCompleted += t.BytesCompleted()
+ aggLen += t.Length()
+
+ for _, peer := range t.PeerConns() {
+ peers[peer.PeerID] = peer
+ }
+
+ }
+
+ line := fmt.Sprintf(
+ "[torrent] Downloading: %d%%, %v/s, peers: %d",
+ int(100*(float64(aggBytesCompleted)/float64(aggLen))),
+ //humanize.Bytes(uint64(aggBytesCompleted)),
+ // humanize.Bytes(uint64(aggLen)),
+ humanize.Bytes(uint64(aggByteRate)),
+ len(peers),
+ )
+ log.Info(line)
+ }
+ }
+}
+
+func (cli *Client) StopSeeding(hash metainfo.Hash) error {
+ t, ok := cli.Cli.Torrent(hash)
+ if !ok {
+ return nil
+ }
+ ch := t.Closed()
+ t.Drop()
+ <-ch
+ return nil
+}
+
+// AddTorrentFiles - adding .torrent files to torrentClient (and checking their hashes), if .torrent file
+// added first time - pieces verification process will start (disk IO heavy) - progress
+// kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again.
+// Don't need call torrent.VerifyData manually
+func AddTorrentFiles(ctx context.Context, snapshotsDir string, torrentClient *torrent.Client, preverifiedHashes snapshothashes.Preverified) error {
+ if err := ForEachTorrentFile(snapshotsDir, func(torrentFilePath string) error {
+ mi, err := metainfo.LoadFromFile(torrentFilePath)
+ if err != nil {
+ return err
+ }
+ mi.AnnounceList = Trackers
+
+ // skip non-preverified files
+ _, torrentFileName := filepath.Split(torrentFilePath)
+ segmentFileName := segmentFileNameFromTorrentFileName(torrentFileName)
+ hashString, ok := preverifiedHashes[segmentFileName]
+ if !ok {
+ return nil
+ }
+ expect := metainfo.NewHashFromHex(hashString)
+ if mi.HashInfoBytes() != expect {
+ return fmt.Errorf("file %s has unexpected hash %x, expected %x", torrentFileName, mi.HashInfoBytes(), expect)
+ }
+
+ if _, err = torrentClient.AddTorrent(mi); err != nil {
+ return err
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ waitForChecksumVerify(ctx, torrentClient)
+ return nil
+}
+
+// ResolveAbsentTorrents - add hard-coded hashes (if client doesn't have) as magnet links and download everything
+func ResolveAbsentTorrents(ctx context.Context, torrentClient *torrent.Client, preverifiedHashes []metainfo.Hash, snapshotDir string) error {
+ mi := &metainfo.MetaInfo{AnnounceList: Trackers}
+ wg := &sync.WaitGroup{}
+ for _, infoHash := range preverifiedHashes {
+ if _, ok := torrentClient.Torrent(infoHash); ok {
+ continue
+ }
+ magnet := mi.Magnet(&infoHash, nil)
+ t, err := torrentClient.AddMagnet(magnet.String())
+ if err != nil {
+ return err
+ }
+ t.AllowDataDownload()
+ t.AllowDataUpload()
+
+ wg.Add(1)
+ go func(t *torrent.Torrent, infoHash metainfo.Hash) {
+ defer wg.Done()
+
+ select {
+ case <-ctx.Done():
+ t.Drop()
+ return
+ case <-t.GotInfo():
+ mi := t.Metainfo()
+ _ = CreateTorrentFileIfNotExists(snapshotDir, t.Info(), &mi)
+ }
+ }(t, infoHash)
+ }
+
+ wg.Wait()
+
+ return nil
+}
+
+func waitForChecksumVerify(ctx context.Context, torrentClient *torrent.Client) {
+ //TODO: tr.VerifyData() - find when to call it
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ go func() {
+ interval := time.Second * 5
+ logEvery := time.NewTicker(interval)
+ defer logEvery.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-logEvery.C:
+ var aggBytesCompleted, aggLen int64
+ for _, t := range torrentClient.Torrents() {
+ aggBytesCompleted += t.BytesCompleted()
+ aggLen += t.Length()
+ }
+
+ line := fmt.Sprintf(
+ "[torrent] verifying snapshots: %s/%s",
+ humanize.Bytes(uint64(aggBytesCompleted)),
+ humanize.Bytes(uint64(aggLen)),
+ )
+ log.Info(line)
+ }
+ }
+ }()
+ torrentClient.WaitAll() // wait for checksum verify
+}
diff --git a/cmd/downloader/downloader/logger.go b/cmd/downloader/downloader/logger.go
new file mode 100644
index 0000000000..b8a6d204f5
--- /dev/null
+++ b/cmd/downloader/downloader/logger.go
@@ -0,0 +1,74 @@
+package downloader
+
+import (
+ stdlog "log"
+ "strings"
+
+ utp "github.com/anacrolix/go-libutp"
+ lg "github.com/anacrolix/log"
+ "github.com/ledgerwatch/log/v3"
+)
+
+func init() {
+ lg.Default = NewAdapterLogger()
+ utp.Logger = stdlog.New(NullWriter(1), "", stdlog.LstdFlags)
+}
+
+func NewAdapterLogger() lg.Logger {
+ return lg.Logger{
+ LoggerImpl: lg.LoggerImpl(adapterLogger{}),
+ }
+}
+
+type adapterLogger struct{}
+
+func (b adapterLogger) Log(msg lg.Msg) {
+ lvl, ok := msg.GetLevel()
+ if !ok {
+ lvl = lg.Info
+ }
+
+ switch lvl {
+ case lg.Debug:
+ log.Debug(msg.String())
+ case lg.Info:
+ str := msg.String()
+ if strings.Contains(str, "EOF") { // suppress useless errors
+ break
+ }
+
+ log.Info(str)
+ case lg.Warning:
+ str := msg.String()
+ if strings.Contains(str, "could not find offer for id") { // suppress useless errors
+ break
+ }
+
+ log.Warn(str)
+ case lg.Error:
+ str := msg.String()
+ if strings.Contains(str, "EOF") { // suppress useless errors
+ break
+ }
+
+ log.Error(str)
+ case lg.Critical:
+ str := msg.String()
+ if strings.Contains(str, "EOF") { // suppress useless errors
+ break
+ }
+ if strings.Contains(str, "don't want conns") { // suppress useless errors
+ break
+ }
+
+ log.Error(str)
+ default:
+ log.Warn("unknown log type", "msg", msg.String())
+ }
+}
+
+// NullWriter implements the io.Write interface but doesn't do anything.
+type NullWriter int
+
+// Write implements the io.Write interface but is a noop.
+func (NullWriter) Write([]byte) (int, error) { return 0, nil }
diff --git a/cmd/downloader/downloader/server.go b/cmd/downloader/downloader/server.go
new file mode 100644
index 0000000000..7b78c2d317
--- /dev/null
+++ b/cmd/downloader/downloader/server.go
@@ -0,0 +1,121 @@
+package downloader
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/anacrolix/torrent"
+ "github.com/anacrolix/torrent/metainfo"
+ "github.com/ledgerwatch/erigon-lib/gointerfaces"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
+ prototypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types"
+ "github.com/ledgerwatch/erigon-lib/kv"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
+ "google.golang.org/protobuf/types/known/emptypb"
+)
+
+var (
+ ErrNotSupportedNetworkID = errors.New("not supported network id")
+ ErrNotSupportedSnapshot = errors.New("not supported snapshot for this network id")
+)
+var (
+ _ proto_downloader.DownloaderServer = &SNDownloaderServer{}
+)
+
+func NewServer(db kv.RwDB, client *Client, snapshotDir string) (*SNDownloaderServer, error) {
+ sn := &SNDownloaderServer{
+ db: db,
+ t: client,
+ snapshotDir: snapshotDir,
+ }
+ return sn, nil
+}
+
+func Stop(torrentClient *torrent.Client) {
+ for _, t := range torrentClient.Torrents() {
+ t.DisallowDataDownload()
+ t.DisallowDataUpload()
+ }
+}
+
+func Start(ctx context.Context, snapshotDir string, torrentClient *torrent.Client, config *snapshothashes.Config) error {
+ if err := BuildTorrentFilesIfNeed(ctx, snapshotDir); err != nil {
+ return err
+ }
+ if err := AddTorrentFiles(ctx, snapshotDir, torrentClient, config.Preverified); err != nil {
+ return err
+ }
+ for _, t := range torrentClient.Torrents() {
+ t.AllowDataDownload()
+ t.AllowDataUpload()
+ }
+
+ return nil
+}
+
+type SNDownloaderServer struct {
+ proto_downloader.UnimplementedDownloaderServer
+ t *Client
+ db kv.RwDB
+ snapshotDir string
+}
+
+func (s *SNDownloaderServer) Download(ctx context.Context, request *proto_downloader.DownloadRequest) (*emptypb.Empty, error) {
+ infoHashes := make([]metainfo.Hash, len(request.Items))
+ for i, it := range request.Items {
+ //TODO: if hash is empty - create .torrent file from path file (if it exists)
+ infoHashes[i] = gointerfaces.ConvertH160toAddress(it.TorrentHash)
+ }
+ ctx, cancel := context.WithTimeout(ctx, time.Minute*10)
+ defer cancel()
+ if err := ResolveAbsentTorrents(ctx, s.t.Cli, infoHashes, s.snapshotDir); err != nil {
+ return nil, err
+ }
+ for _, t := range s.t.Cli.Torrents() {
+ t.AllowDataDownload()
+ t.AllowDataUpload()
+ t.DownloadAll()
+ }
+ return &emptypb.Empty{}, nil
+}
+
+func (s *SNDownloaderServer) Stats(ctx context.Context, request *proto_downloader.StatsRequest) (*proto_downloader.StatsReply, error) {
+ torrents := s.t.Cli.Torrents()
+ reply := &proto_downloader.StatsReply{Completed: true, Torrents: int32(len(torrents))}
+
+ peers := map[torrent.PeerID]struct{}{}
+
+ for _, t := range torrents {
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-t.GotInfo():
+ reply.BytesCompleted += uint64(t.BytesCompleted())
+ reply.BytesTotal += uint64(t.Info().TotalLength())
+ reply.Completed = reply.Completed && t.Complete.Bool()
+ for _, peer := range t.PeerConns() {
+ peers[peer.PeerID] = struct{}{}
+ }
+ default:
+ reply.Completed = false
+ }
+ }
+
+ reply.Peers = int32(len(peers))
+ reply.Progress = int32(100 * (float64(reply.BytesCompleted) / float64(reply.BytesTotal)))
+ if reply.Progress == 100 && !reply.Completed {
+ reply.Progress = 99
+ }
+ return reply, nil
+}
+
+func Proto2InfoHashes(in []*prototypes.H160) []metainfo.Hash {
+ infoHashes := make([]metainfo.Hash, len(in))
+ i := 0
+ for _, h := range in {
+ infoHashes[i] = gointerfaces.ConvertH160toAddress(h)
+ i++
+ }
+ return infoHashes
+}
diff --git a/cmd/downloader/downloader/util.go b/cmd/downloader/downloader/util.go
new file mode 100644
index 0000000000..fae51be87c
--- /dev/null
+++ b/cmd/downloader/downloader/util.go
@@ -0,0 +1,177 @@
+package downloader
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path"
+ "path/filepath"
+ "time"
+
+ "github.com/anacrolix/torrent/bencode"
+ "github.com/anacrolix/torrent/metainfo"
+ "github.com/ledgerwatch/erigon/cmd/downloader/trackers"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/log/v3"
+)
+
+// DefaultPieceSize - Erigon serves many big files, bigger pieces will reduce
+// amount of network announcements, but can't go over 2Mb
+// see https://wiki.theory.org/BitTorrentSpecification#Metainfo_File_Structure
+const DefaultPieceSize = 2 * 1024 * 1024
+
+// Trackers - break down by priority tier
+var Trackers = [][]string{
+ trackers.Best, trackers.Ws, // trackers.Udp, trackers.Https, trackers.Http,
+}
+
+func allTorrentFiles(dir string) ([]string, error) {
+ files, err := ioutil.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+ var res []string
+ for _, f := range files {
+ if !snapshotsync.IsCorrectFileName(f.Name()) {
+ continue
+ }
+ if f.Size() == 0 {
+ continue
+ }
+ if filepath.Ext(f.Name()) != ".torrent" { // filter out only compressed files
+ continue
+ }
+ res = append(res, f.Name())
+ }
+ return res, nil
+}
+func allSegmentFiles(dir string) ([]string, error) {
+ files, err := ioutil.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+ var res []string
+ for _, f := range files {
+ if !snapshotsync.IsCorrectFileName(f.Name()) {
+ continue
+ }
+ if f.Size() == 0 {
+ continue
+ }
+ if filepath.Ext(f.Name()) != ".seg" { // filter out only compressed files
+ continue
+ }
+ res = append(res, f.Name())
+ }
+ return res, nil
+}
+
+func ForEachTorrentFile(root string, walker func(torrentFileName string) error) error {
+ files, err := allTorrentFiles(root)
+ if err != nil {
+ return err
+ }
+ for _, f := range files {
+ torrentFileName := filepath.Join(root, f)
+ if _, err := os.Stat(torrentFileName); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ return err
+ }
+ if err := walker(torrentFileName); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// BuildTorrentFilesIfNeed - create .torrent files from .seg files (big IO) - if .seg files were added manually
+func BuildTorrentFilesIfNeed(ctx context.Context, root string) error {
+ logEvery := time.NewTicker(20 * time.Second)
+ defer logEvery.Stop()
+
+ files, err := allSegmentFiles(root)
+ if err != nil {
+ return err
+ }
+ for i, f := range files {
+ torrentFileName := path.Join(root, f+".torrent")
+ if _, err := os.Stat(torrentFileName); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ info, err := BuildInfoBytesForFile(root, f)
+ if err != nil {
+ return err
+ }
+ if err := CreateTorrentFile(root, info, nil); err != nil {
+ return err
+ }
+ }
+
+ select {
+ default:
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-logEvery.C:
+ log.Info("[torrent] Create .torrent files", "progress", fmt.Sprintf("%d/%d", i, len(files)))
+ }
+ }
+ return nil
+}
+
+func BuildInfoBytesForFile(root string, fileName string) (*metainfo.Info, error) {
+ info := &metainfo.Info{PieceLength: DefaultPieceSize}
+ if err := info.BuildFromFilePath(filepath.Join(root, fileName)); err != nil {
+ return nil, err
+ }
+ return info, nil
+}
+
+func CreateTorrentFileIfNotExists(root string, info *metainfo.Info, mi *metainfo.MetaInfo) error {
+ torrentFileName := filepath.Join(root, info.Name+".torrent")
+ if _, err := os.Stat(torrentFileName); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return CreateTorrentFile(root, info, mi)
+ }
+ return err
+ }
+ return nil
+}
+
+func CreateTorrentFile(root string, info *metainfo.Info, mi *metainfo.MetaInfo) error {
+ if mi == nil {
+ infoBytes, err := bencode.Marshal(info)
+ if err != nil {
+ return err
+ }
+ mi = &metainfo.MetaInfo{
+ CreationDate: time.Now().Unix(),
+ CreatedBy: "erigon",
+ InfoBytes: infoBytes,
+ AnnounceList: Trackers,
+ }
+ } else {
+ mi.AnnounceList = Trackers
+ }
+ torrentFileName := filepath.Join(root, info.Name+".torrent")
+
+ file, err := os.Create(torrentFileName)
+ if err != nil {
+ return err
+ }
+ defer file.Sync()
+ defer file.Close()
+ if err := mi.Write(file); err != nil {
+ return err
+ }
+ return nil
+}
+
+func segmentFileNameFromTorrentFileName(in string) string {
+ ext := filepath.Ext(in)
+ return in[0 : len(in)-len(ext)]
+}
diff --git a/cmd/downloader/downloadergrpc/client.go b/cmd/downloader/downloadergrpc/client.go
new file mode 100644
index 0000000000..47ae9492f2
--- /dev/null
+++ b/cmd/downloader/downloadergrpc/client.go
@@ -0,0 +1,64 @@
+package downloadergrpc
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/anacrolix/torrent/metainfo"
+ "github.com/c2h5oh/datasize"
+ "github.com/ledgerwatch/erigon-lib/gointerfaces"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
+ prototypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types"
+ "github.com/ledgerwatch/erigon/common"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/backoff"
+ "google.golang.org/grpc/keepalive"
+)
+
+func NewClient(ctx context.Context, downloaderAddr string) (proto_downloader.DownloaderClient, error) {
+ // creating grpc client connection
+ var dialOpts []grpc.DialOption
+
+ backoffCfg := backoff.DefaultConfig
+ backoffCfg.BaseDelay = 500 * time.Millisecond
+ backoffCfg.MaxDelay = 10 * time.Second
+ dialOpts = []grpc.DialOption{
+ grpc.WithConnectParams(grpc.ConnectParams{Backoff: backoffCfg, MinConnectTimeout: 10 * time.Minute}),
+ grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(16 * datasize.MB))),
+ grpc.WithKeepaliveParams(keepalive.ClientParameters{}),
+ }
+
+ dialOpts = append(dialOpts, grpc.WithInsecure())
+ conn, err := grpc.DialContext(ctx, downloaderAddr, dialOpts...)
+ if err != nil {
+ return nil, fmt.Errorf("creating client connection to sentry P2P: %w", err)
+ }
+ return proto_downloader.NewDownloaderClient(conn), nil
+}
+
+func InfoHashes2Proto(in []metainfo.Hash) []*prototypes.H160 {
+ infoHashes := make([]*prototypes.H160, len(in))
+ i := 0
+ for _, h := range in {
+ infoHashes[i] = gointerfaces.ConvertAddressToH160(h)
+ i++
+ }
+ return infoHashes
+}
+
+func Strings2Proto(in []string) []*prototypes.H160 {
+ infoHashes := make([]*prototypes.H160, len(in))
+ i := 0
+ for _, h := range in {
+ infoHashes[i] = String2Proto(h)
+ i++
+ }
+ return infoHashes
+}
+
+func String2Proto(in string) *prototypes.H160 {
+ var infoHash [20]byte
+ copy(infoHash[:], common.FromHex(in))
+ return gointerfaces.ConvertAddressToH160(infoHash)
+}
diff --git a/cmd/downloader/generator/commands/metainfo_hash.go b/cmd/downloader/generator/commands/metainfo_hash.go
deleted file mode 100644
index 81efd1b2f9..0000000000
--- a/cmd/downloader/generator/commands/metainfo_hash.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package commands
-
-import (
- "errors"
- "fmt"
- "time"
-
- "github.com/anacrolix/torrent/bencode"
- "github.com/anacrolix/torrent/metainfo"
- "github.com/ledgerwatch/erigon/common"
- "github.com/ledgerwatch/erigon/turbo/snapshotsync"
- "github.com/spf13/cobra"
-)
-
-func init() {
- rootCmd.AddCommand(snapshotMetainfoCmd)
-}
-
-func PrintMetaInfoHash(path string) error {
- t := time.Now()
- mi := metainfo.MetaInfo{}
- info, err := snapshotsync.BuildInfoBytesForSnapshot(path, snapshotsync.MdbxFilename)
- if err != nil {
- return err
- }
- mi.InfoBytes, err = bencode.Marshal(info)
- if err != nil {
- return err
- }
-
- fmt.Println("infohash:", mi.HashInfoBytes().String())
- fmt.Println("infobytes:", common.Bytes2Hex(mi.InfoBytes))
- fmt.Println("It took", time.Since(t))
- return nil
-}
-
-var snapshotMetainfoCmd = &cobra.Command{
- Use: "snapshotMetainfo",
- Short: "Calculate snapshot metainfo",
- RunE: func(cmd *cobra.Command, args []string) error {
- if len(args) == 0 {
- return errors.New("empty path")
- }
- return PrintMetaInfoHash(args[0])
- },
-}
diff --git a/cmd/downloader/root.go b/cmd/downloader/main.go
similarity index 50%
rename from cmd/downloader/root.go
rename to cmd/downloader/main.go
index d9c76c972e..80a9207dff 100644
--- a/cmd/downloader/root.go
+++ b/cmd/downloader/main.go
@@ -2,20 +2,26 @@ package main
import (
"context"
+ "encoding/json"
"fmt"
"net"
"os"
"path"
"time"
+ "github.com/anacrolix/torrent/metainfo"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
- proto_snap "github.com/ledgerwatch/erigon-lib/gointerfaces/snapshotsync"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
+ "github.com/ledgerwatch/erigon-lib/kv"
+ "github.com/ledgerwatch/erigon-lib/kv/mdbx"
+ "github.com/ledgerwatch/erigon/cmd/downloader/downloader"
+ "github.com/ledgerwatch/erigon/cmd/hack/tool"
"github.com/ledgerwatch/erigon/cmd/utils"
"github.com/ledgerwatch/erigon/common/paths"
"github.com/ledgerwatch/erigon/internal/debug"
"github.com/ledgerwatch/erigon/params"
- "github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/log/v3"
"github.com/spf13/cobra"
"google.golang.org/grpc"
@@ -30,21 +36,26 @@ var (
datadir string
seeding bool
downloaderApiAddr string
- healthCheck bool
)
func init() {
flags := append(debug.Flags, utils.MetricFlags...)
utils.CobraFlags(rootCmd, flags)
- rootCmd.Flags().StringVar(&datadir, utils.DataDirFlag.Name, paths.DefaultDataDir(), utils.DataDirFlag.Usage)
- if err := rootCmd.MarkFlagDirname(utils.DataDirFlag.Name); err != nil {
- panic(err)
- }
+ withDatadir(rootCmd)
rootCmd.PersistentFlags().BoolVar(&seeding, "seeding", true, "Seed snapshots")
rootCmd.Flags().StringVar(&downloaderApiAddr, "downloader.api.addr", "127.0.0.1:9093", "external downloader api network address, for example: 127.0.0.1:9093 serves remote downloader interface")
- rootCmd.Flags().BoolVar(&healthCheck, "healthcheck", false, "Enable grpc health check")
+
+ withDatadir(printInfoHashes)
+ rootCmd.AddCommand(printInfoHashes)
+}
+
+func withDatadir(cmd *cobra.Command) {
+ cmd.Flags().StringVar(&datadir, utils.DataDirFlag.Name, paths.DefaultDataDir(), utils.DataDirFlag.Usage)
+ if err := cmd.MarkFlagDirname(utils.DataDirFlag.Name); err != nil {
+ panic(err)
+ }
}
func main() {
@@ -70,64 +81,111 @@ var rootCmd = &cobra.Command{
debug.Exit()
},
RunE: func(cmd *cobra.Command, args []string) error {
- ctx := cmd.Context()
- snapshotsDir := path.Join(datadir, "snapshots")
- log.Info("Run snapshot downloader", "addr", downloaderApiAddr, "datadir", datadir, "seeding", seeding)
+ if err := Downloader(cmd.Context(), cmd); err != nil {
+ log.Error("Downloader", "err", err)
+ return nil
+ }
+ return nil
+ },
+}
- bittorrentServer, err := snapshotsync.NewServer(snapshotsDir, seeding)
+func Downloader(ctx context.Context, cmd *cobra.Command) error {
+ var cc *params.ChainConfig
+ {
+ chaindataDir := path.Join(datadir, "chaindata")
+ if err := os.MkdirAll(chaindataDir, 0755); err != nil {
+ return err
+ }
+ chaindata, err := mdbx.Open(chaindataDir, log.New(), true)
+ if err != nil {
+ return fmt.Errorf("%w, path: %s", err, chaindataDir)
+ }
+ cc = tool.ChainConfigFromDB(chaindata)
+ chaindata.Close()
+ }
+
+ snapshotsDir := path.Join(datadir, "snapshots")
+ log.Info("Run snapshot downloader", "addr", downloaderApiAddr, "datadir", datadir, "seeding", seeding)
+ if err := os.MkdirAll(snapshotsDir, 0755); err != nil {
+ return err
+ }
+
+ db := mdbx.MustOpen(snapshotsDir + "/db")
+ var t *downloader.Client
+ if err := db.Update(context.Background(), func(tx kv.RwTx) error {
+ peerID, err := tx.GetOne(kv.BittorrentInfo, []byte(kv.BittorrentPeerID))
if err != nil {
- return fmt.Errorf("new server: %w", err)
+ return fmt.Errorf("get peer id: %w", err)
}
- log.Info("Load")
- err = bittorrentServer.Load()
+ t, err = downloader.New(snapshotsDir, seeding, string(peerID))
if err != nil {
- return fmt.Errorf("load: %w", err)
+ return err
+ }
+ if len(peerID) == 0 {
+ err = t.SavePeerID(tx)
+ if err != nil {
+ return fmt.Errorf("save peer id: %w", err)
+ }
}
+ return nil
+ }); err != nil {
+ return err
+ }
+ defer t.Close()
+
+ bittorrentServer, err := downloader.NewServer(db, t, snapshotsDir)
+ if err != nil {
+ return fmt.Errorf("new server: %w", err)
+ }
+
+ snapshotsCfg := snapshothashes.KnownConfig(cc.ChainName)
+ err = downloader.Start(ctx, snapshotsDir, t.Cli, snapshotsCfg)
+ if err != nil {
+ return fmt.Errorf("start: %w", err)
+ }
+
+ go downloader.MainLoop(ctx, t.Cli)
- go func() {
- _, err := bittorrentServer.Download(ctx, &proto_snap.DownloadSnapshotRequest{
- NetworkId: params.MainnetChainConfig.ChainID.Uint64(),
- Type: snapshotsync.GetAvailableSnapshotTypes(params.MainnetChainConfig.ChainID.Uint64()),
- })
+ grpcServer, err := StartGrpc(bittorrentServer, downloaderApiAddr, nil)
+ if err != nil {
+ return err
+ }
+ <-cmd.Context().Done()
+ grpcServer.GracefulStop()
+ return nil
+}
+
+var printInfoHashes = &cobra.Command{
+ Use: "print_info_hashes",
+ Example: "go run ./cmd/downloader print_info_hashes --datadir <your_datadir> ",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ snapshotsDir := path.Join(datadir, "snapshots")
+
+ res := map[string]string{}
+ if err := downloader.ForEachTorrentFile(snapshotsDir, func(torrentFilePath string) error {
+ mi, err := metainfo.LoadFromFile(torrentFilePath)
if err != nil {
- log.Error("Download failed", "err", err, "networkID", params.MainnetChainConfig.ChainID.Uint64())
+ return err
}
- }()
- go func() {
- for {
- select {
- case <-cmd.Context().Done():
- return
- default:
- }
-
- snapshots, err := bittorrentServer.Snapshots(ctx, &proto_snap.SnapshotsRequest{
- NetworkId: params.MainnetChainConfig.ChainID.Uint64(),
- })
- if err != nil {
- log.Error("get snapshots", "err", err)
- time.Sleep(time.Minute)
- continue
- }
- stats := bittorrentServer.Stats(context.Background())
- for _, v := range snapshots.Info {
- log.Info("Snapshot "+v.Type.String(), "%", v.Readiness, "peers", stats[v.Type.String()].ConnectedSeeders)
- }
- time.Sleep(time.Minute)
+ info, err := mi.UnmarshalInfo()
+ if err != nil {
+ return err
}
- }()
- grpcServer, err := StartGrpc(bittorrentServer, downloaderApiAddr, nil, healthCheck)
+ res[info.Name] = mi.HashInfoBytes().String()
+ return nil
+ }); err != nil {
+ return err
+ }
+ b, err := json.Marshal(res)
if err != nil {
return err
}
- <-cmd.Context().Done()
- grpcServer.GracefulStop()
-
+ fmt.Printf("%s\n", b)
return nil
},
}
-func StartGrpc(snServer *snapshotsync.SNDownloaderServer, addr string, creds *credentials.TransportCredentials, healthCheck bool) (*grpc.Server, error) {
+func StartGrpc(snServer *downloader.SNDownloaderServer, addr string, creds *credentials.TransportCredentials) (*grpc.Server, error) {
lis, err := net.Listen("tcp", addr)
if err != nil {
return nil, fmt.Errorf("could not create listener: %w, addr=%s", err, addr)
@@ -162,22 +220,18 @@ func StartGrpc(snServer *snapshotsync.SNDownloaderServer, addr string, creds *cr
grpcServer := grpc.NewServer(opts...)
reflection.Register(grpcServer) // Register reflection service on gRPC server.
if snServer != nil {
- proto_snap.RegisterDownloaderServer(grpcServer, snServer)
- }
- var healthServer *health.Server
- if healthCheck {
- healthServer = health.NewServer()
- grpc_health_v1.RegisterHealthServer(grpcServer, healthServer)
+ proto_downloader.RegisterDownloaderServer(grpcServer, snServer)
}
//if metrics.Enabled {
// grpc_prometheus.Register(grpcServer)
//}
+ healthServer := health.NewServer()
+ grpc_health_v1.RegisterHealthServer(grpcServer, healthServer)
+
go func() {
- if healthCheck {
- defer healthServer.Shutdown()
- }
+ defer healthServer.Shutdown()
if err := grpcServer.Serve(lis); err != nil {
log.Error("gRPC server stop", "err", err)
}
diff --git a/cmd/downloader/seeder/main.go b/cmd/downloader/seeder/main.go
deleted file mode 100644
index e2c0f3adb7..0000000000
--- a/cmd/downloader/seeder/main.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "time"
-
- lg "github.com/anacrolix/log"
- "github.com/anacrolix/torrent"
- "github.com/anacrolix/torrent/bencode"
- "github.com/anacrolix/torrent/metainfo"
- libcommon "github.com/ledgerwatch/erigon-lib/common"
- "github.com/ledgerwatch/erigon/cmd/utils"
- "github.com/ledgerwatch/erigon/common"
- "github.com/ledgerwatch/erigon/internal/debug"
- trnt "github.com/ledgerwatch/erigon/turbo/snapshotsync"
- "github.com/ledgerwatch/log/v3"
- "github.com/spf13/cobra"
-)
-
-func init() {
- utils.CobraFlags(rootCmd, append(debug.Flags, utils.MetricFlags...))
-}
-
-func main() {
- ctx, cancel := utils.RootContext()
- defer cancel()
-
- if err := rootCmd.ExecuteContext(ctx); err != nil {
- fmt.Println(err)
- os.Exit(1)
- }
-}
-
-var rootCmd = &cobra.Command{
- Use: "seed",
- Short: "seed snapshot",
- PersistentPreRun: func(cmd *cobra.Command, args []string) {
- if err := debug.SetupCobra(cmd); err != nil {
- panic(err)
- }
- },
- PersistentPostRun: func(cmd *cobra.Command, args []string) {
- debug.Exit()
- },
- Args: cobra.ExactArgs(1),
- ArgAliases: []string{"snapshots dir"},
- RunE: func(cmd *cobra.Command, args []string) error {
- return Seed(cmd.Context(), args[0])
- },
-}
-
-func Seed(ctx context.Context, datadir string) error {
- defer func() {
- //hack origin lib don't have proper close handling
- time.Sleep(time.Second * 5)
- }()
- datadir = filepath.Dir(datadir)
- ctx, cancel := context.WithCancel(ctx)
- defer cancel()
-
- cfg := trnt.DefaultTorrentConfig()
- cfg.NoDHT = false
- cfg.DisableTrackers = false
- cfg.Seed = true
- cfg.Debug = false
- cfg.Logger = cfg.Logger.FilterLevel(lg.Info)
- cfg.DataDir = datadir
-
- pathes := []string{
- cfg.DataDir + "/headers",
- cfg.DataDir + "/bodies",
- cfg.DataDir + "/state",
- }
-
- cl, err := torrent.NewClient(cfg)
- if err != nil {
- return err
- }
- defer cl.Close()
-
- torrents := make([]*torrent.Torrent, len(pathes))
- for i, v := range pathes {
- i := i
- mi := &metainfo.MetaInfo{
- CreationDate: time.Now().Unix(),
- CreatedBy: "erigon",
- AnnounceList: trnt.Trackers,
- }
-
- if _, err := os.Stat(v); os.IsNotExist(err) {
- fmt.Println(err)
- continue
- } else if err != nil {
- return err
- }
- tt := time.Now()
- if common.IsCanceled(ctx) {
- return libcommon.ErrStopped
- }
- info, err := trnt.BuildInfoBytesForSnapshot(v, trnt.MdbxFilename)
- if err != nil {
- return err
- }
-
- mi.InfoBytes, err = bencode.Marshal(info)
- if err != nil {
- return err
- }
-
- torrents[i], _, err = cl.AddTorrentSpec(&torrent.TorrentSpec{
- Trackers: trnt.Trackers,
- InfoHash: mi.HashInfoBytes(),
- InfoBytes: mi.InfoBytes,
- ChunkSize: trnt.DefaultChunkSize,
- })
- if err != nil {
- return err
- }
-
- log.Info("Torrent added", "name", torrents[i].Info().Name, "path", v, "t", time.Since(tt))
-
- if !torrents[i].Seeding() {
- log.Warn(torrents[i].Name() + " not seeding")
- }
-
- if common.IsCanceled(ctx) {
- return libcommon.ErrStopped
- }
- }
-
- go func() {
- ticker := time.NewTicker(10 * time.Second)
- for range ticker.C {
- for _, t := range cl.Torrents() {
- log.Info("Snapshot stats", "snapshot", t.Name(), "active peers", t.Stats().ActivePeers, "seeding", t.Seeding(), "hash", t.Metainfo().HashInfoBytes().String())
- }
-
- if common.IsCanceled(ctx) {
- ticker.Stop()
- return
- }
- }
- }()
-
- <-ctx.Done()
- return nil
-}
diff --git a/cmd/downloader/trackers/embed.go b/cmd/downloader/trackers/embed.go
new file mode 100644
index 0000000000..a0a0f7162d
--- /dev/null
+++ b/cmd/downloader/trackers/embed.go
@@ -0,0 +1,39 @@
+package trackers
+
+import (
+ _ "embed"
+ "strings"
+)
+
+//go:embed trackerslist/trackers_best.txt
+var best string
+var Best = strings.Split(best, "\n\n")
+
+//go:embed trackerslist/trackers_all_https.txt
+var https string
+var Https = withoutBest(strings.Split(https, "\n\n"))
+
+//go:embed trackerslist/trackers_all_http.txt
+var http string
+var Http = withoutBest(strings.Split(http, "\n\n"))
+
+//go:embed trackerslist/trackers_all_udp.txt
+var udp string
+var Udp = withoutBest(strings.Split(udp, "\n\n"))
+
+//go:embed trackerslist/trackers_all_ws.txt
+var ws string
+var Ws = withoutBest(strings.Split(ws, "\n\n"))
+
+func withoutBest(in []string) (res []string) {
+Loop:
+ for _, tracker := range in {
+ for _, bestItem := range Best {
+ if tracker == bestItem {
+ continue Loop
+ }
+ }
+ res = append(res, tracker)
+ }
+ return res
+}
diff --git a/cmd/downloader/trackers/trackerslist b/cmd/downloader/trackers/trackerslist
new file mode 160000
index 0000000000..40110ecd39
--- /dev/null
+++ b/cmd/downloader/trackers/trackerslist
@@ -0,0 +1 @@
+Subproject commit 40110ecd394cd66c749ca5fe278e36cb444bdbf8
diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
index f9dfa41c9e..618a379f11 100644
--- a/cmd/hack/hack.go
+++ b/cmd/hack/hack.go
@@ -60,6 +60,7 @@ import (
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/rlp"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/erigon/turbo/trie"
"github.com/ledgerwatch/log/v3"
"github.com/wcharczuk/go-chart/v2"
@@ -2654,7 +2655,7 @@ func checkBlockSnapshot(chaindata string) error {
chainID, _ := uint256.FromBig(chainConfig.ChainID)
_ = chainID
- snapshots := snapshotsync.NewAllSnapshots(path.Join(dataDir, "snapshots"), params.KnownSnapshots(chainConfig.ChainName))
+ snapshots := snapshotsync.NewAllSnapshots(path.Join(dataDir, "snapshots"), snapshothashes.KnownConfig(chainConfig.ChainName))
snapshots.ReopenSegments()
snapshots.ReopenIndices()
//if err := snapshots.BuildIndices(context.Background(), *chainID); err != nil {
diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go
index 8c5a9ffd6c..0127c313d9 100644
--- a/cmd/integration/commands/stages.go
+++ b/cmd/integration/commands/stages.go
@@ -14,7 +14,7 @@ import (
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/cmd/rpcdaemon/interfaces"
- "github.com/ledgerwatch/erigon/cmd/sentry/download"
+ "github.com/ledgerwatch/erigon/cmd/sentry/sentry"
"github.com/ledgerwatch/erigon/cmd/utils"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/dbutils"
@@ -33,7 +33,9 @@ import (
"github.com/ledgerwatch/erigon/migrations"
"github.com/ledgerwatch/erigon/p2p"
"github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/erigon/params/networkname"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
stages2 "github.com/ledgerwatch/erigon/turbo/stages"
"github.com/ledgerwatch/log/v3"
"github.com/ledgerwatch/secp256k1"
@@ -996,25 +998,25 @@ func byChain() (*core.Genesis, *params.ChainConfig) {
var genesis *core.Genesis
switch chain {
- case "", params.MainnetChainName:
+ case "", networkname.MainnetChainName:
chainConfig = params.MainnetChainConfig
genesis = core.DefaultGenesisBlock()
- case params.RopstenChainName:
+ case networkname.RopstenChainName:
chainConfig = params.RopstenChainConfig
genesis = core.DefaultRopstenGenesisBlock()
- case params.GoerliChainName:
+ case networkname.GoerliChainName:
chainConfig = params.GoerliChainConfig
genesis = core.DefaultGoerliGenesisBlock()
- case params.RinkebyChainName:
+ case networkname.RinkebyChainName:
chainConfig = params.RinkebyChainConfig
genesis = core.DefaultRinkebyGenesisBlock()
- case params.SokolChainName:
+ case networkname.SokolChainName:
chainConfig = params.SokolChainConfig
genesis = core.DefaultSokolGenesisBlock()
- case params.KovanChainName:
+ case networkname.KovanChainName:
chainConfig = params.KovanChainConfig
genesis = core.DefaultKovanGenesisBlock()
- case params.FermionChainName:
+ case networkname.FermionChainName:
chainConfig = params.FermionChainConfig
genesis = core.DefaultFermionGenesisBlock()
}
@@ -1031,11 +1033,11 @@ func allSnapshots(cc *params.ChainConfig) *snapshotsync.AllSnapshots {
Enabled: true,
Dir: path.Join(datadir, "snapshots"),
}
- _allSnapshotsSingleton = snapshotsync.NewAllSnapshots(snapshotCfg.Dir, params.KnownSnapshots(cc.ChainName))
+ _allSnapshotsSingleton = snapshotsync.NewAllSnapshots(snapshotCfg.Dir, snapshothashes.KnownConfig(cc.ChainName))
if err := _allSnapshotsSingleton.ReopenSegments(); err != nil {
panic(err)
}
- if err := _allSnapshotsSingleton.ReopenIndices(); err != nil {
+ if err := _allSnapshotsSingleton.ReopenSomeIndices(snapshotsync.AllSnapshotTypes...); err != nil {
panic(err)
}
}
@@ -1098,7 +1100,7 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig)
must(batchSize.UnmarshalText([]byte(batchSizeStr)))
blockDownloaderWindow := 65536
- downloadServer, err := download.NewControlServer(db, "", chainConfig, genesisBlock.Hash(), engine, 1, nil, blockDownloaderWindow)
+ sentryControlServer, err := sentry.NewControlServer(db, "", chainConfig, genesisBlock.Hash(), engine, 1, nil, blockDownloaderWindow)
if err != nil {
panic(err)
}
@@ -1111,7 +1113,11 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig)
cfg.Miner = *miningConfig
}
- sync, err := stages2.NewStagedSync(context.Background(), logger, db, p2p.Config{}, cfg, chainConfig.TerminalTotalDifficulty, downloadServer, tmpdir, nil, nil, nil, nil)
+ sync, err := stages2.NewStagedSync(context.Background(), logger, db, p2p.Config{}, cfg,
+ chainConfig.TerminalTotalDifficulty, sentryControlServer, tmpdir,
+ nil, nil, nil, nil,
+ nil,
+ )
if err != nil {
panic(err)
}
diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go
index 8a881c09d9..cb57e51b80 100644
--- a/cmd/rpcdaemon/cli/config.go
+++ b/cmd/rpcdaemon/cli/config.go
@@ -31,6 +31,7 @@ import (
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/rpc"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/log/v3"
"github.com/spf13/cobra"
"google.golang.org/grpc"
@@ -277,7 +278,7 @@ func RemoteServices(ctx context.Context, cfg Flags, logger log.Logger, rootCance
return nil, nil, nil, nil, nil, nil, fmt.Errorf("chain config not found in db. Need start erigon at least once on this db")
}
- allSnapshots := snapshotsync.NewAllSnapshots(cfg.Snapshot.Dir, params.KnownSnapshots(cc.ChainName))
+ allSnapshots := snapshotsync.NewAllSnapshots(cfg.Snapshot.Dir, snapshothashes.KnownConfig(cc.ChainName))
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
diff --git a/cmd/rpcdaemon/interfaces/interfaces.go b/cmd/rpcdaemon/interfaces/interfaces.go
index 7b3346f8b4..4143133563 100644
--- a/cmd/rpcdaemon/interfaces/interfaces.go
+++ b/cmd/rpcdaemon/interfaces/interfaces.go
@@ -6,6 +6,7 @@ import (
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/core/types"
+ "github.com/ledgerwatch/erigon/rlp"
)
type BlockReader interface {
@@ -17,7 +18,13 @@ type HeaderReader interface {
HeaderByNumber(ctx context.Context, tx kv.Getter, blockHeight uint64) (*types.Header, error)
}
+type BodyReader interface {
+ Body(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (body *types.Body, err error)
+ BodyRlp(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (bodyRlp rlp.RawValue, err error)
+}
+
type FullBlockReader interface {
BlockReader
+ BodyReader
HeaderReader
}
diff --git a/cmd/sentry/commands/sentry.go b/cmd/sentry/commands/sentry.go
deleted file mode 100644
index c9a1670883..0000000000
--- a/cmd/sentry/commands/sentry.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package commands
-
-import (
- "fmt"
- "os"
- "path"
-
- "github.com/ledgerwatch/erigon/cmd/sentry/download"
- "github.com/ledgerwatch/erigon/cmd/utils"
- "github.com/ledgerwatch/erigon/common/paths"
- "github.com/ledgerwatch/erigon/eth/protocols/eth"
- "github.com/ledgerwatch/erigon/internal/debug"
- node2 "github.com/ledgerwatch/erigon/turbo/node"
- "github.com/spf13/cobra"
-)
-
-var (
- sentryAddr string // Address of the sentry <host>:<port>
- chaindata string // Path to chaindata
- datadir string // Path to td working dir
-
- natSetting string // NAT setting
- port int // Listening port
- staticPeers []string // static peers
- trustedPeers []string // trusted peers
- discoveryDNS []string
- nodiscover bool // disable sentry's discovery mechanism
- protocol string
- netRestrict string // CIDR to restrict peering to
- healthCheck bool
-)
-
-func init() {
- utils.CobraFlags(rootCmd, append(debug.Flags, utils.MetricFlags...))
-
- rootCmd.Flags().StringVar(&natSetting, "nat", "", `NAT port mapping mechanism (any|none|upnp|pmp|extip:<IP>)
- "" or "none" default - do not nat
- "extip:77.12.33.4" will assume the local machine is reachable on the given IP
- "any" uses the first auto-detected mechanism
- "upnp" uses the Universal Plug and Play protocol
- "pmp" uses NAT-PMP with an auto-detected gateway address
- "pmp:192.168.0.1" uses NAT-PMP with the given gateway address
-`)
- rootCmd.Flags().IntVar(&port, "port", 30303, "p2p port number")
- rootCmd.Flags().StringVar(&sentryAddr, "sentry.api.addr", "localhost:9091", "grpc addresses")
- rootCmd.Flags().StringVar(&protocol, "p2p.protocol", "eth66", "eth66")
- rootCmd.Flags().StringSliceVar(&staticPeers, "staticpeers", []string{}, "static peer list [enode]")
- rootCmd.Flags().StringSliceVar(&trustedPeers, "trustedpeers", []string{}, "trusted peer list [enode]")
- rootCmd.Flags().StringSliceVar(&discoveryDNS, utils.DNSDiscoveryFlag.Name, []string{}, utils.DNSDiscoveryFlag.Usage)
- rootCmd.Flags().BoolVar(&nodiscover, utils.NoDiscoverFlag.Name, false, utils.NoDiscoverFlag.Usage)
- rootCmd.Flags().StringVar(&netRestrict, "netrestrict", "", "CIDR range to accept peers from <CIDR>")
- rootCmd.Flags().StringVar(&datadir, utils.DataDirFlag.Name, paths.DefaultDataDir(), utils.DataDirFlag.Usage)
- rootCmd.Flags().BoolVar(&healthCheck, utils.HealthCheckFlag.Name, false, utils.HealthCheckFlag.Usage)
- if err := rootCmd.MarkFlagDirname(utils.DataDirFlag.Name); err != nil {
- panic(err)
- }
-
-}
-
-var rootCmd = &cobra.Command{
- Use: "sentry",
- Short: "Run p2p sentry",
- PersistentPreRun: func(cmd *cobra.Command, args []string) {
- if err := debug.SetupCobra(cmd); err != nil {
- panic(err)
- }
- if chaindata == "" {
- chaindata = path.Join(datadir, "chaindata")
- }
- },
- PersistentPostRun: func(cmd *cobra.Command, args []string) {
- debug.Exit()
- },
- RunE: func(cmd *cobra.Command, args []string) error {
- p := eth.ETH66
-
- nodeConfig := node2.NewNodeConfig()
- p2pConfig, err := utils.NewP2PConfig(nodiscover, datadir, netRestrict, natSetting, nodeConfig.NodeName(), staticPeers, trustedPeers, uint(port), uint(p))
- if err != nil {
- return err
- }
- return download.Sentry(datadir, sentryAddr, discoveryDNS, p2pConfig, uint(p), healthCheck)
- },
-}
-
-func Execute() {
- ctx, cancel := utils.RootContext()
- defer cancel()
- if err := rootCmd.ExecuteContext(ctx); err != nil {
- fmt.Println(err)
- os.Exit(1)
- }
-}
diff --git a/cmd/sentry/main.go b/cmd/sentry/main.go
index b212bb0493..45f42c69d6 100644
--- a/cmd/sentry/main.go
+++ b/cmd/sentry/main.go
@@ -1,11 +1,95 @@
package main
import (
- "github.com/ledgerwatch/erigon/cmd/sentry/commands"
+ "fmt"
+ "os"
+ "path"
+
+ "github.com/ledgerwatch/erigon/cmd/sentry/sentry"
+ "github.com/ledgerwatch/erigon/cmd/utils"
+ "github.com/ledgerwatch/erigon/common/paths"
+ "github.com/ledgerwatch/erigon/eth/protocols/eth"
+ "github.com/ledgerwatch/erigon/internal/debug"
+ node2 "github.com/ledgerwatch/erigon/turbo/node"
+ "github.com/spf13/cobra"
)
// generate the messages
+var (
+ sentryAddr string // Address of the sentry <host>:<port>
+ chaindata string // Path to chaindata
+ datadir string // Path to td working dir
+
+ natSetting string // NAT setting
+ port int // Listening port
+ staticPeers []string // static peers
+ trustedPeers []string // trusted peers
+ discoveryDNS []string
+ nodiscover bool // disable sentry's discovery mechanism
+ protocol string
+ netRestrict string // CIDR to restrict peering to
+ healthCheck bool
+)
+
+func init() {
+ utils.CobraFlags(rootCmd, append(debug.Flags, utils.MetricFlags...))
+
+ rootCmd.Flags().StringVar(&natSetting, "nat", "", `NAT port mapping mechanism (any|none|upnp|pmp|extip:<IP>)
+ "" or "none" default - do not nat
+ "extip:77.12.33.4" will assume the local machine is reachable on the given IP
+ "any" uses the first auto-detected mechanism
+ "upnp" uses the Universal Plug and Play protocol
+ "pmp" uses NAT-PMP with an auto-detected gateway address
+ "pmp:192.168.0.1" uses NAT-PMP with the given gateway address
+`)
+ rootCmd.Flags().IntVar(&port, "port", 30303, "p2p port number")
+ rootCmd.Flags().StringVar(&sentryAddr, "sentry.api.addr", "localhost:9091", "grpc addresses")
+ rootCmd.Flags().StringVar(&protocol, "p2p.protocol", "eth66", "eth66")
+ rootCmd.Flags().StringSliceVar(&staticPeers, "staticpeers", []string{}, "static peer list [enode]")
+ rootCmd.Flags().StringSliceVar(&trustedPeers, "trustedpeers", []string{}, "trusted peer list [enode]")
+ rootCmd.Flags().StringSliceVar(&discoveryDNS, utils.DNSDiscoveryFlag.Name, []string{}, utils.DNSDiscoveryFlag.Usage)
+ rootCmd.Flags().BoolVar(&nodiscover, utils.NoDiscoverFlag.Name, false, utils.NoDiscoverFlag.Usage)
+ rootCmd.Flags().StringVar(&netRestrict, "netrestrict", "", "CIDR range to accept peers from <CIDR>")
+ rootCmd.Flags().StringVar(&datadir, utils.DataDirFlag.Name, paths.DefaultDataDir(), utils.DataDirFlag.Usage)
+ rootCmd.Flags().BoolVar(&healthCheck, utils.HealthCheckFlag.Name, false, utils.HealthCheckFlag.Usage)
+ if err := rootCmd.MarkFlagDirname(utils.DataDirFlag.Name); err != nil {
+ panic(err)
+ }
+
+}
+
+var rootCmd = &cobra.Command{
+ Use: "sentry",
+ Short: "Run p2p sentry",
+ PersistentPreRun: func(cmd *cobra.Command, args []string) {
+ if err := debug.SetupCobra(cmd); err != nil {
+ panic(err)
+ }
+ if chaindata == "" {
+ chaindata = path.Join(datadir, "chaindata")
+ }
+ },
+ PersistentPostRun: func(cmd *cobra.Command, args []string) {
+ debug.Exit()
+ },
+ RunE: func(cmd *cobra.Command, args []string) error {
+ p := eth.ETH66
+
+ nodeConfig := node2.NewNodeConfig()
+ p2pConfig, err := utils.NewP2PConfig(nodiscover, datadir, netRestrict, natSetting, nodeConfig.NodeName(), staticPeers, trustedPeers, uint(port), uint(p))
+ if err != nil {
+ return err
+ }
+ return sentry.Sentry(datadir, sentryAddr, discoveryDNS, p2pConfig, uint(p), healthCheck)
+ },
+}
+
func main() {
- commands.Execute()
+ ctx, cancel := utils.RootContext()
+ defer cancel()
+ if err := rootCmd.ExecuteContext(ctx); err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
}
diff --git a/cmd/sentry/download/broadcast.go b/cmd/sentry/sentry/broadcast.go
similarity index 99%
rename from cmd/sentry/download/broadcast.go
rename to cmd/sentry/sentry/broadcast.go
index fce68ee7b9..09d60d4ab8 100644
--- a/cmd/sentry/download/broadcast.go
+++ b/cmd/sentry/sentry/broadcast.go
@@ -1,4 +1,4 @@
-package download
+package sentry
import (
"context"
diff --git a/cmd/sentry/download/downloader.go b/cmd/sentry/sentry/downloader.go
similarity index 99%
rename from cmd/sentry/download/downloader.go
rename to cmd/sentry/sentry/downloader.go
index 6d48755174..9d1411079b 100644
--- a/cmd/sentry/download/downloader.go
+++ b/cmd/sentry/sentry/downloader.go
@@ -1,4 +1,4 @@
-package download
+package sentry
import (
"bytes"
@@ -753,9 +753,7 @@ func makeStatusData(s *ControlServerImpl) *proto_sentry.StatusData {
}
}
-// Methods of Core called by sentry
-
-func GrpcSentryClient(ctx context.Context, sentryAddr string) (*direct.SentryClientRemote, error) {
+func GrpcClient(ctx context.Context, sentryAddr string) (*direct.SentryClientRemote, error) {
// creating grpc client connection
var dialOpts []grpc.DialOption
diff --git a/cmd/sentry/download/sentry.go b/cmd/sentry/sentry/sentry.go
similarity index 99%
rename from cmd/sentry/download/sentry.go
rename to cmd/sentry/sentry/sentry.go
index a3a56cb112..f70a839d4f 100644
--- a/cmd/sentry/download/sentry.go
+++ b/cmd/sentry/sentry/sentry.go
@@ -1,4 +1,4 @@
-package download
+package sentry
import (
"bytes"
diff --git a/cmd/sentry/download/sentry_api.go b/cmd/sentry/sentry/sentry_api.go
similarity index 99%
rename from cmd/sentry/download/sentry_api.go
rename to cmd/sentry/sentry/sentry_api.go
index 6cefab0d18..ae12dc8c69 100644
--- a/cmd/sentry/download/sentry_api.go
+++ b/cmd/sentry/sentry/sentry_api.go
@@ -1,4 +1,4 @@
-package download
+package sentry
import (
"context"
diff --git a/cmd/sentry/download/sentry_test.go b/cmd/sentry/sentry/sentry_test.go
similarity index 99%
rename from cmd/sentry/download/sentry_test.go
rename to cmd/sentry/sentry/sentry_test.go
index 95a989e54c..d8316129e9 100644
--- a/cmd/sentry/download/sentry_test.go
+++ b/cmd/sentry/sentry/sentry_test.go
@@ -1,4 +1,4 @@
-package download
+package sentry
import (
"context"
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 70535d1055..60ed95aecf 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -33,6 +33,7 @@ import (
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/txpool"
"github.com/ledgerwatch/erigon/eth/protocols/eth"
+ "github.com/ledgerwatch/erigon/params/networkname"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/urfave/cli"
@@ -122,7 +123,7 @@ var (
ChainFlag = cli.StringFlag{
Name: "chain",
Usage: "Name of the testnet to join",
- Value: params.MainnetChainName,
+ Value: networkname.MainnetChainName,
}
IdentityFlag = cli.StringFlag{
Name: "identity",
@@ -412,6 +413,11 @@ var (
Name: "sentry.api.addr",
Usage: "comma separated sentry addresses '<host>:<port>,<host>:<port>'",
}
+ DownloaderAddrFlag = cli.StringFlag{
+ Name: "downloader.api.addr",
+ Value: "127.0.0.1:9093",
+ Usage: "downloader address '<host>:<port>'",
+ }
BootnodesFlag = cli.StringFlag{
Name: "bootnodes",
Usage: "Comma separated enode URLs for P2P discovery bootstrap",
@@ -601,19 +607,19 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
} else {
chain := ctx.GlobalString(ChainFlag.Name)
switch chain {
- case params.RopstenChainName:
+ case networkname.RopstenChainName:
urls = params.RopstenBootnodes
- case params.RinkebyChainName:
+ case networkname.RinkebyChainName:
urls = params.RinkebyBootnodes
- case params.GoerliChainName:
+ case networkname.GoerliChainName:
urls = params.GoerliBootnodes
- case params.ErigonMineName:
+ case networkname.ErigonMineName:
urls = params.ErigonBootnodes
- case params.SokolChainName:
+ case networkname.SokolChainName:
urls = params.SokolBootnodes
- case params.KovanChainName:
+ case networkname.KovanChainName:
urls = params.KovanBootnodes
- case params.FermionChainName:
+ case networkname.FermionChainName:
urls = params.FermionBootnodes
default:
if cfg.BootstrapNodes != nil {
@@ -635,19 +641,19 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
chain := ctx.GlobalString(ChainFlag.Name)
switch chain {
- case params.RopstenChainName:
+ case networkname.RopstenChainName:
urls = params.RopstenBootnodes
- case params.RinkebyChainName:
+ case networkname.RinkebyChainName:
urls = params.RinkebyBootnodes
- case params.GoerliChainName:
+ case networkname.GoerliChainName:
urls = params.GoerliBootnodes
- case params.ErigonMineName:
+ case networkname.ErigonMineName:
urls = params.ErigonBootnodes
- case params.SokolChainName:
+ case networkname.SokolChainName:
urls = params.SokolBootnodes
- case params.KovanChainName:
+ case networkname.KovanChainName:
urls = params.KovanBootnodes
- case params.FermionChainName:
+ case networkname.FermionChainName:
urls = params.FermionBootnodes
default:
if cfg.BootstrapNodesV5 != nil {
@@ -821,7 +827,7 @@ func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) {
}
}
- if ctx.GlobalString(ChainFlag.Name) == params.DevChainName {
+ if ctx.GlobalString(ChainFlag.Name) == networkname.DevChainName {
if etherbase == "" {
cfg.Miner.SigKey = core.DevnetSignPrivateKey
cfg.Miner.Etherbase = core.DevnetEtherbase
@@ -829,9 +835,9 @@ func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) {
setSigKey(ctx, cfg)
}
- if ctx.GlobalString(ChainFlag.Name) == params.FermionChainName {
+ if ctx.GlobalString(ChainFlag.Name) == networkname.FermionChainName {
if ctx.GlobalIsSet(MiningEnabledFlag.Name) && !ctx.GlobalIsSet(MinerSigningKeyFileFlag.Name) {
- panic(fmt.Sprintf("Flag --%s is required in %s chain with --%s flag", MinerSigningKeyFileFlag.Name, params.FermionChainName, MiningEnabledFlag.Name))
+ panic(fmt.Sprintf("Flag --%s is required in %s chain with --%s flag", MinerSigningKeyFileFlag.Name, networkname.FermionChainName, MiningEnabledFlag.Name))
}
setSigKey(ctx, cfg)
if cfg.Miner.SigKey != nil {
@@ -876,7 +882,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config, nodeName, dataDir string) {
cfg.NetRestrict = list
}
- if ctx.GlobalString(ChainFlag.Name) == params.DevChainName {
+ if ctx.GlobalString(ChainFlag.Name) == networkname.DevChainName {
// --dev mode can't use p2p networking.
// cfg.MaxPeers = 0 // It can have peers otherwise local sync is not possible
cfg.ListenAddr = ":0"
@@ -890,6 +896,8 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
setDataDir(ctx, cfg)
setNodeUserIdent(ctx, cfg)
SetP2PConfig(ctx, &cfg.P2P, cfg.NodeName(), cfg.DataDir)
+
+ cfg.DownloaderAddr = strings.TrimSpace(ctx.GlobalString(DownloaderAddrFlag.Name))
}
func SetNodeConfigCobra(cmd *cobra.Command, cfg *node.Config) {
@@ -905,17 +913,17 @@ func DataDirForNetwork(datadir string, network string) string {
}
switch network {
- case params.DevChainName:
+ case networkname.DevChainName:
return "" // unless explicitly requested, use memory databases
- case params.RinkebyChainName:
+ case networkname.RinkebyChainName:
return filepath.Join(datadir, "rinkeby")
- case params.GoerliChainName:
+ case networkname.GoerliChainName:
filepath.Join(datadir, "goerli")
- case params.SokolChainName:
+ case networkname.SokolChainName:
return filepath.Join(datadir, "sokol")
- case params.KovanChainName:
+ case networkname.KovanChainName:
return filepath.Join(datadir, "kovan")
- case params.FermionChainName:
+ case networkname.FermionChainName:
return filepath.Join(datadir, "fermion")
default:
return datadir
@@ -1264,51 +1272,51 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *node.Config, cfg *ethconfig.Conf
if cfg.NetworkID == 1 {
SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
}
- case params.MainnetChainName:
+ case networkname.MainnetChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 1
}
cfg.Genesis = core.DefaultGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
- case params.RopstenChainName:
+ case networkname.RopstenChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 3
}
cfg.Genesis = core.DefaultRopstenGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.RopstenGenesisHash)
- case params.RinkebyChainName:
+ case networkname.RinkebyChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 4
}
cfg.Genesis = core.DefaultRinkebyGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.RinkebyGenesisHash)
- case params.GoerliChainName:
+ case networkname.GoerliChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 5
}
cfg.Genesis = core.DefaultGoerliGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash)
- case params.ErigonMineName:
+ case networkname.ErigonMineName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = new(big.Int).SetBytes([]byte("erigon-mine")).Uint64() // erigon-mine
}
cfg.Genesis = core.DefaultErigonGenesisBlock()
- case params.SokolChainName:
+ case networkname.SokolChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 77
}
cfg.Genesis = core.DefaultSokolGenesisBlock()
- case params.KovanChainName:
+ case networkname.KovanChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 42
}
cfg.Genesis = core.DefaultKovanGenesisBlock()
- case params.FermionChainName:
+ case networkname.FermionChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 1212120
}
cfg.Genesis = core.DefaultFermionGenesisBlock()
- case params.DevChainName:
+ case networkname.DevChainName:
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkID = 1337
}
@@ -1372,21 +1380,21 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
var genesis *core.Genesis
chain := ctx.GlobalString(ChainFlag.Name)
switch chain {
- case params.RopstenChainName:
+ case networkname.RopstenChainName:
genesis = core.DefaultRopstenGenesisBlock()
- case params.RinkebyChainName:
+ case networkname.RinkebyChainName:
genesis = core.DefaultRinkebyGenesisBlock()
- case params.GoerliChainName:
+ case networkname.GoerliChainName:
genesis = core.DefaultGoerliGenesisBlock()
- case params.ErigonMineName:
+ case networkname.ErigonMineName:
genesis = core.DefaultErigonGenesisBlock()
- case params.SokolChainName:
+ case networkname.SokolChainName:
genesis = core.DefaultSokolGenesisBlock()
- case params.KovanChainName:
+ case networkname.KovanChainName:
genesis = core.DefaultKovanGenesisBlock()
- case params.FermionChainName:
+ case networkname.FermionChainName:
genesis = core.DefaultFermionGenesisBlock()
- case params.DevChainName:
+ case networkname.DevChainName:
Fatalf("Developer chains are ephemeral")
}
return genesis
diff --git a/consensus/aura/consensusconfig/embed.go b/consensus/aura/consensusconfig/embed.go
index 883c4c3d04..48deb45aa2 100644
--- a/consensus/aura/consensusconfig/embed.go
+++ b/consensus/aura/consensusconfig/embed.go
@@ -2,7 +2,8 @@ package consensusconfig
import (
_ "embed"
- "github.com/ledgerwatch/erigon/params"
+
+ "github.com/ledgerwatch/erigon/params/networkname"
)
//go:embed poasokol.json
@@ -13,9 +14,9 @@ var Kovan []byte
func GetConfigByChain(chainName string) []byte {
switch chainName {
- case params.SokolChainName:
+ case networkname.SokolChainName:
return Sokol
- case params.KovanChainName:
+ case networkname.KovanChainName:
return Kovan
default:
return Sokol
diff --git a/eth/backend.go b/eth/backend.go
index a6ca8aa7a6..9a64a6a864 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -34,9 +34,10 @@ import (
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/direct"
"github.com/ledgerwatch/erigon-lib/etl"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
"github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil"
"github.com/ledgerwatch/erigon-lib/gointerfaces/remote"
- "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry"
+ proto_sentry "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry"
txpool_proto "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool"
prototypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types"
"github.com/ledgerwatch/erigon-lib/kv"
@@ -44,8 +45,9 @@ import (
"github.com/ledgerwatch/erigon-lib/kv/remotedbserver"
txpool2 "github.com/ledgerwatch/erigon-lib/txpool"
"github.com/ledgerwatch/erigon-lib/txpool/txpooluitl"
+ "github.com/ledgerwatch/erigon/cmd/downloader/downloadergrpc"
"github.com/ledgerwatch/erigon/cmd/rpcdaemon/interfaces"
- "github.com/ledgerwatch/erigon/cmd/sentry/download"
+ "github.com/ledgerwatch/erigon/cmd/sentry/sentry"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/debug"
"github.com/ledgerwatch/erigon/consensus"
@@ -68,6 +70,7 @@ import (
"github.com/ledgerwatch/erigon/rpc"
"github.com/ledgerwatch/erigon/turbo/shards"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
stages2 "github.com/ledgerwatch/erigon/turbo/stages"
"github.com/ledgerwatch/log/v3"
"google.golang.org/grpc"
@@ -103,12 +106,15 @@ type Ethereum struct {
minedBlocks chan *types.Block
// downloader fields
- downloadCtx context.Context
- downloadCancel context.CancelFunc
- downloadServer *download.ControlServerImpl
- sentryServers []*download.SentryServerImpl
- sentries []direct.SentryClient
- stagedSync *stagedsync.Sync
+ sentryCtx context.Context
+ sentryCancel context.CancelFunc
+ sentryControlServer *sentry.ControlServerImpl
+ sentryServers []*sentry.SentryServerImpl
+ sentries []direct.SentryClient
+
+ stagedSync *stagedsync.Sync
+
+ downloaderClient proto_downloader.DownloaderClient
notifications *stagedsync.Notifications
@@ -172,8 +178,8 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
ctx, ctxCancel := context.WithCancel(context.Background())
kvRPC := remotedbserver.NewKvServer(ctx, chainKv)
backend := &Ethereum{
- downloadCtx: ctx,
- downloadCancel: ctxCancel,
+ sentryCtx: ctx,
+ sentryCancel: ctxCancel,
config: config,
logger: logger,
chainDB: chainKv,
@@ -243,7 +249,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
if len(stack.Config().P2P.SentryAddr) > 0 {
for _, addr := range stack.Config().P2P.SentryAddr {
- sentryClient, err := download.GrpcSentryClient(backend.downloadCtx, addr)
+ sentryClient, err := sentry.GrpcClient(backend.sentryCtx, addr)
if err != nil {
return nil, err
}
@@ -267,7 +273,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
cfg66 := stack.Config().P2P
cfg66.NodeDatabase = path.Join(stack.Config().DataDir, "nodes", "eth66")
- server66 := download.NewSentryServer(backend.downloadCtx, d66, readNodeInfo, &cfg66, eth.ETH66)
+ server66 := sentry.NewSentryServer(backend.sentryCtx, d66, readNodeInfo, &cfg66, eth.ETH66)
backend.sentryServers = append(backend.sentryServers, server66)
backend.sentries = []direct.SentryClient{direct.NewSentryClientDirect(eth.ETH66, server66)}
@@ -279,7 +285,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
for {
select {
- case <-backend.downloadCtx.Done():
+ case <-backend.sentryCtx.Done():
return
case <-logEvery.C:
logItems = logItems[:0]
@@ -291,7 +297,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
}
}()
}
- backend.downloadServer, err = download.NewControlServer(chainKv, stack.Config().NodeName(), chainConfig, genesis.Hash(), backend.engine, backend.config.NetworkID, backend.sentries, config.BlockDownloaderWindow)
+ backend.sentryControlServer, err = sentry.NewControlServer(chainKv, stack.Config().NodeName(), chainConfig, genesis.Hash(), backend.engine, backend.config.NetworkID, backend.sentries, config.BlockDownloaderWindow)
if err != nil {
return nil, err
}
@@ -340,17 +346,23 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
var blockReader interfaces.FullBlockReader
if config.Snapshot.Enabled {
- allSnapshots := snapshotsync.NewAllSnapshots(config.Snapshot.Dir, params.KnownSnapshots(chainConfig.ChainName))
+ allSnapshots := snapshotsync.NewAllSnapshots(config.Snapshot.Dir, snapshothashes.KnownConfig(chainConfig.ChainName))
if err != nil {
return nil, err
}
blockReader = snapshotsync.NewBlockReaderWithSnapshots(allSnapshots)
+
+ // connect to Downloader
+ backend.downloaderClient, err = downloadergrpc.NewClient(ctx, stack.Config().DownloaderAddr)
+ if err != nil {
+ return nil, err
+ }
} else {
blockReader = snapshotsync.NewBlockReader()
}
mining := stagedsync.New(
- stagedsync.MiningStages(backend.downloadCtx,
+ stagedsync.MiningStages(backend.sentryCtx,
stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miner, *backend.chainConfig, backend.engine, backend.txPool2, backend.txPool2DB, tmpdir),
stagedsync.StageMiningExecCfg(backend.chainDB, miner, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir),
stagedsync.StageHashStateCfg(backend.chainDB, tmpdir),
@@ -391,7 +403,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
if !config.TxPool.Disable {
backend.txPool2Fetch.ConnectCore()
backend.txPool2Fetch.ConnectSentries()
- go txpool2.MainLoop(backend.downloadCtx,
+ go txpool2.MainLoop(backend.sentryCtx,
backend.txPool2DB, backend.chainDB,
backend.txPool2, backend.newTxs2, backend.txPool2Send, backend.txPool2GrpcServer.NewSlotsStreams,
func() {
@@ -407,15 +419,15 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
select {
case b := <-backend.minedBlocks:
//p2p
- //backend.downloadServer.BroadcastNewBlock(context.Background(), b, b.Difficulty())
+ //backend.sentryControlServer.BroadcastNewBlock(context.Background(), b, b.Difficulty())
//rpcdaemon
if err := miningRPC.(*privateapi.MiningServer).BroadcastMinedBlock(b); err != nil {
log.Error("txpool rpc mined block broadcast", "err", err)
}
- if err := backend.downloadServer.Hd.AddMinedHeader(b.Header()); err != nil {
+ if err := backend.sentryControlServer.Hd.AddMinedHeader(b.Header()); err != nil {
log.Error("add mined block to header downloader", "err", err)
}
- if err := backend.downloadServer.Bd.AddMinedBlock(b); err != nil {
+ if err := backend.sentryControlServer.Bd.AddMinedBlock(b); err != nil {
log.Error("add mined block to body downloader", "err", err)
}
@@ -433,7 +445,11 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
return nil, err
}
- backend.stagedSync, err = stages2.NewStagedSync(backend.downloadCtx, backend.logger, backend.chainDB, stack.Config().P2P, *config, chainConfig.TerminalTotalDifficulty, backend.downloadServer, tmpdir, backend.notifications.Accumulator, backend.reverseDownloadCh, backend.statusCh, &backend.waitingForPOSHeaders)
+ backend.stagedSync, err = stages2.NewStagedSync(backend.sentryCtx, backend.logger, backend.chainDB,
+ stack.Config().P2P, *config, chainConfig.TerminalTotalDifficulty,
+ backend.sentryControlServer, tmpdir, backend.notifications.Accumulator,
+ backend.reverseDownloadCh, backend.statusCh, &backend.waitingForPOSHeaders,
+ backend.downloaderClient)
if err != nil {
return nil, err
}
@@ -552,7 +568,7 @@ func (s *Ethereum) StartMining(ctx context.Context, db kv.RwDB, mining *stagedsy
defer skipCycleEvery.Stop()
for range skipCycleEvery.C {
select {
- case s.downloadServer.Hd.SkipCycleHack <- struct{}{}:
+ case s.sentryControlServer.Hd.SkipCycleHack <- struct{}{}:
default:
}
}
@@ -610,7 +626,7 @@ func (s *Ethereum) NetPeerCount() (uint64, error) {
log.Trace("sentry", "peer count", sentryPc)
for _, sc := range s.sentries {
ctx := context.Background()
- reply, err := sc.PeerCount(ctx, &sentry.PeerCountRequest{})
+ reply, err := sc.PeerCount(ctx, &proto_sentry.PeerCountRequest{})
if err != nil {
log.Warn("sentry", "err", err)
return 0, nil
@@ -659,17 +675,17 @@ func (s *Ethereum) Protocols() []p2p.Protocol {
func (s *Ethereum) Start() error {
for i := range s.sentries {
go func(i int) {
- download.RecvMessageLoop(s.downloadCtx, s.sentries[i], s.downloadServer, nil)
+ sentry.RecvMessageLoop(s.sentryCtx, s.sentries[i], s.sentryControlServer, nil)
}(i)
go func(i int) {
- download.RecvUploadMessageLoop(s.downloadCtx, s.sentries[i], s.downloadServer, nil)
+ sentry.RecvUploadMessageLoop(s.sentryCtx, s.sentries[i], s.sentryControlServer, nil)
}(i)
go func(i int) {
- download.RecvUploadHeadersMessageLoop(s.downloadCtx, s.sentries[i], s.downloadServer, nil)
+ sentry.RecvUploadHeadersMessageLoop(s.sentryCtx, s.sentries[i], s.sentryControlServer, nil)
}(i)
}
- go stages2.StageLoop(s.downloadCtx, s.chainDB, s.stagedSync, s.downloadServer.Hd, s.notifications, s.downloadServer.UpdateHead, s.waitForStageLoopStop, s.config.SyncLoopThrottle)
+ go stages2.StageLoop(s.sentryCtx, s.chainDB, s.stagedSync, s.sentryControlServer.Hd, s.notifications, s.sentryControlServer.UpdateHead, s.waitForStageLoopStop, s.config.SyncLoopThrottle)
return nil
}
@@ -678,7 +694,7 @@ func (s *Ethereum) Start() error {
// Ethereum protocol.
func (s *Ethereum) Stop() error {
// Stop all the peer-related stuff first.
- s.downloadCancel()
+ s.sentryCancel()
if s.privateAPI != nil {
shutdownDone := make(chan bool)
go func() {
diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go
index 02aaa207ca..3e6df2d4f6 100644
--- a/eth/ethconfig/config.go
+++ b/eth/ethconfig/config.go
@@ -31,6 +31,7 @@ import (
"github.com/ledgerwatch/erigon/consensus/aura/consensusconfig"
"github.com/ledgerwatch/erigon/consensus/serenity"
"github.com/ledgerwatch/erigon/ethdb/prune"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/consensus"
@@ -119,7 +120,7 @@ func init() {
type Snapshot struct {
Enabled bool
Dir string
- ChainSnapshotConfig *params.SnapshotsConfig
+ ChainSnapshotConfig *snapshothashes.Config
}
// Config contains configuration options for ETH protocol.
diff --git a/eth/stagedsync/stage_headers.go b/eth/stagedsync/stage_headers.go
index 7adcf17c74..5f25739dbd 100644
--- a/eth/stagedsync/stage_headers.go
+++ b/eth/stagedsync/stage_headers.go
@@ -12,7 +12,9 @@ import (
"github.com/holiman/uint256"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/etl"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
"github.com/ledgerwatch/erigon-lib/kv"
+ "github.com/ledgerwatch/erigon/cmd/downloader/downloadergrpc"
"github.com/ledgerwatch/erigon/cmd/rpcdaemon/interfaces"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/dbutils"
@@ -24,8 +26,8 @@ import (
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/rlp"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/erigon/turbo/stages/headerdownload"
-
"github.com/ledgerwatch/log/v3"
)
@@ -42,8 +44,10 @@ type HeadersCfg struct {
tmpdir string
reverseDownloadCh chan types.Header
waitingPosHeaders *bool
- snapshots *snapshotsync.AllSnapshots
- blockReader interfaces.FullBlockReader
+
+ snapshots *snapshotsync.AllSnapshots
+ snapshotDownloader proto_downloader.DownloaderClient
+ blockReader interfaces.FullBlockReader
}
func StageHeadersCfg(
@@ -59,23 +63,25 @@ func StageHeadersCfg(
reverseDownloadCh chan types.Header,
waitingPosHeaders *bool,
snapshots *snapshotsync.AllSnapshots,
+ snapshotDownloader proto_downloader.DownloaderClient,
blockReader interfaces.FullBlockReader,
tmpdir string,
) HeadersCfg {
return HeadersCfg{
- db: db,
- hd: headerDownload,
- statusCh: statusCh,
- chainConfig: chainConfig,
- headerReqSend: headerReqSend,
- announceNewHashes: announceNewHashes,
- penalize: penalize,
- batchSize: batchSize,
- noP2PDiscovery: noP2PDiscovery,
- reverseDownloadCh: reverseDownloadCh,
- waitingPosHeaders: waitingPosHeaders,
- snapshots: snapshots,
- blockReader: blockReader,
+ db: db,
+ hd: headerDownload,
+ statusCh: statusCh,
+ chainConfig: chainConfig,
+ headerReqSend: headerReqSend,
+ announceNewHashes: announceNewHashes,
+ penalize: penalize,
+ batchSize: batchSize,
+ noP2PDiscovery: noP2PDiscovery,
+ reverseDownloadCh: reverseDownloadCh,
+ waitingPosHeaders: waitingPosHeaders,
+ snapshots: snapshots,
+ snapshotDownloader: snapshotDownloader,
+ blockReader: blockReader,
}
}
@@ -291,123 +297,11 @@ func HeadersPOW(
test bool, // Set to true in tests, allows the stage to fail rather than wait indefinitely
useExternalTx bool,
) error {
- var headerProgress uint64
-
- if cfg.snapshots != nil {
- if !cfg.snapshots.AllSegmentsAvailable() {
- // wait for Downloader service to download all expected snapshots
- logEvery := time.NewTicker(logInterval)
- defer logEvery.Stop()
- for {
- headers, bodies, txs, err := cfg.snapshots.SegmentsAvailability()
- if err != nil {
- return err
- }
- expect := cfg.snapshots.ChainSnapshotConfig().ExpectBlocks
- if headers >= expect && bodies >= expect && txs >= expect {
- if err := cfg.snapshots.ReopenSegments(); err != nil {
- return err
- }
- if expect > cfg.snapshots.BlocksAvailable() {
- return fmt.Errorf("not enough snapshots available: %d > %d", expect, cfg.snapshots.BlocksAvailable())
- }
- cfg.snapshots.SetAllSegmentsAvailable(true)
-
- break
- }
- log.Info(fmt.Sprintf("[%s] Waiting for snapshots up to block %d...", s.LogPrefix(), expect), "headers", headers, "bodies", bodies, "txs", txs)
- time.Sleep(10 * time.Second)
-
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-logEvery.C:
- log.Info(fmt.Sprintf("[%s] Waiting for snapshots up to block %d...", s.LogPrefix(), expect), "headers", headers, "bodies", bodies, "txs", txs)
- default:
- }
- }
- }
-
- if !cfg.snapshots.AllIdxAvailable() {
- if !cfg.snapshots.AllSegmentsAvailable() {
- return fmt.Errorf("not all snapshot segments are available")
- }
-
- // wait for Downloader service to download all expected snapshots
- logEvery := time.NewTicker(logInterval)
- defer logEvery.Stop()
- headers, bodies, txs, err := cfg.snapshots.IdxAvailability()
- if err != nil {
- return err
- }
- expect := cfg.snapshots.ChainSnapshotConfig().ExpectBlocks
- if headers < expect || bodies < expect || txs < expect {
- chainID, _ := uint256.FromBig(cfg.chainConfig.ChainID)
- if err := cfg.snapshots.BuildIndices(ctx, *chainID); err != nil {
- return err
- }
- }
-
- if err := cfg.snapshots.ReopenIndices(); err != nil {
- return err
- }
- if expect > cfg.snapshots.IndicesAvailable() {
- return fmt.Errorf("not enough snapshots available: %d > %d", expect, cfg.snapshots.BlocksAvailable())
- }
- cfg.snapshots.SetAllIdxAvailable(true)
- }
-
- c, _ := tx.Cursor(kv.HeaderTD)
- count, _ := c.Count()
- if count == 0 || count == 1 { // genesis does write 1 record
- logEvery := time.NewTicker(logInterval)
- defer logEvery.Stop()
-
- tx.ClearBucket(kv.HeaderTD)
- var lastHeader *types.Header
- //total difficulty write
- td := big.NewInt(0)
- if err := snapshotsync.ForEachHeader(cfg.snapshots, func(header *types.Header) error {
- td.Add(td, header.Difficulty)
- /*
- if header.Eip3675 {
- return nil
- }
- if td.Cmp(cfg.terminalTotalDifficulty) > 0 {
- return rawdb.MarkTransition(tx, blockNum)
- }
- */
- // TODO: append
- rawdb.WriteTd(tx, header.Hash(), header.Number.Uint64(), td)
- lastHeader = header
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-logEvery.C:
- log.Info(fmt.Sprintf("[%s] Writing total difficulty index for snapshots", s.LogPrefix()), "block_num", header.Number.Uint64())
- default:
- }
- return nil
- }); err != nil {
- return err
- }
- tx.ClearBucket(kv.HeaderCanonical)
- if err := fixCanonicalChain(s.LogPrefix(), logEvery, lastHeader.Number.Uint64(), lastHeader.Hash(), tx, cfg.blockReader); err != nil {
- return err
- }
- }
-
- if s.BlockNumber < cfg.snapshots.BlocksAvailable() {
- if err := cfg.hd.AddHeaderFromSnapshot(cfg.snapshots.BlocksAvailable(), cfg.blockReader); err != nil {
- return err
- }
- if err := s.Update(tx, cfg.snapshots.BlocksAvailable()); err != nil {
- return err
- }
- s.BlockNumber = cfg.snapshots.BlocksAvailable()
- }
+ if err := DownloadAndIndexSnapshotsIfNeed(s, ctx, tx, cfg); err != nil {
+ return err
}
+ var headerProgress uint64
var err error
if !useExternalTx {
@@ -452,6 +346,9 @@ func HeadersPOW(
if err != nil {
return err
}
+ if localTd == nil {
+ return fmt.Errorf("localTD is nil: %d, %x", headerProgress, hash)
+ }
headerInserter := headerdownload.NewHeaderInserter(logPrefix, localTd, headerProgress)
cfg.hd.SetHeaderReader(&chainReader{config: &cfg.chainConfig, tx: tx, blockReader: cfg.blockReader})
@@ -812,3 +709,201 @@ func HeadersPrune(p *PruneState, tx kv.RwTx, cfg HeadersCfg, ctx context.Context
}
return nil
}
+
+func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.RwTx, cfg HeadersCfg) error {
+ if cfg.snapshots == nil {
+ return nil
+ }
+
+ // TODO: save AllSegmentsAvailable flag to DB? (to allow Erigon start without Downloader)
+ if !cfg.snapshots.AllSegmentsAvailable() {
+ if err := WaitForDownloader(ctx, tx, cfg); err != nil {
+ return err
+ }
+
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+
+ // Open segments
+ for {
+ headers, bodies, txs, err := cfg.snapshots.SegmentsAvailability()
+ if err != nil {
+ return err
+ }
+ expect := cfg.snapshots.ChainSnapshotConfig().ExpectBlocks
+ if headers >= expect && bodies >= expect && txs >= expect {
+ if err := cfg.snapshots.ReopenSegments(); err != nil {
+ return err
+ }
+ if expect > cfg.snapshots.BlocksAvailable() {
+ return fmt.Errorf("not enough snapshots available: %d > %d", expect, cfg.snapshots.BlocksAvailable())
+ }
+ cfg.snapshots.SetAllSegmentsAvailable(true)
+
+ break
+ }
+ log.Info(fmt.Sprintf("[%s] Waiting for snapshots up to block %d...", s.LogPrefix(), expect), "headers", headers, "bodies", bodies, "txs", txs)
+ time.Sleep(10 * time.Second)
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Waiting for snapshots up to block %d...", s.LogPrefix(), expect), "headers", headers, "bodies", bodies, "txs", txs)
+ default:
+ }
+ }
+ }
+
+ // Create .idx files
+ if !cfg.snapshots.AllIdxAvailable() {
+ if !cfg.snapshots.AllSegmentsAvailable() {
+ return fmt.Errorf("not all snapshot segments are available")
+ }
+
+ // wait for Downloader service to download all expected snapshots
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+ headers, bodies, txs, err := cfg.snapshots.IdxAvailability()
+ if err != nil {
+ return err
+ }
+ expect := cfg.snapshots.ChainSnapshotConfig().ExpectBlocks
+ if headers < expect || bodies < expect || txs < expect {
+ chainID, _ := uint256.FromBig(cfg.chainConfig.ChainID)
+ if err := cfg.snapshots.BuildIndices(ctx, *chainID); err != nil {
+ return err
+ }
+ }
+
+ if err := cfg.snapshots.ReopenIndices(); err != nil {
+ return err
+ }
+ if expect > cfg.snapshots.IndicesAvailable() {
+ return fmt.Errorf("not enough snapshots available: %d > %d", expect, cfg.snapshots.BlocksAvailable())
+ }
+ cfg.snapshots.SetAllIdxAvailable(true)
+ }
+
+ // Fill kv.HeaderTD table from snapshots
+ c, _ := tx.Cursor(kv.HeaderTD)
+ count, _ := c.Count()
+ if count == 0 || count == 1 { // genesis does write 1 record
+ logEvery := time.NewTicker(logInterval)
+ defer logEvery.Stop()
+
+ tx.ClearBucket(kv.HeaderTD)
+ var lastHeader *types.Header
+ //total difficulty write
+ td := big.NewInt(0)
+ if err := snapshotsync.ForEachHeader(cfg.snapshots, func(header *types.Header) error {
+ td.Add(td, header.Difficulty)
+ /*
+ if header.Eip3675 {
+ return nil
+ }
+
+ if td.Cmp(cfg.terminalTotalDifficulty) > 0 {
+ return rawdb.MarkTransition(tx, blockNum)
+ }
+ */
+ // TODO: append
+ rawdb.WriteTd(tx, header.Hash(), header.Number.Uint64(), td)
+ lastHeader = header
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-logEvery.C:
+ log.Info(fmt.Sprintf("[%s] Writing total difficulty index for snapshots", s.LogPrefix()), "block_num", header.Number.Uint64())
+ default:
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ // Fill kv.HeaderCanonical table from snapshots
+ tx.ClearBucket(kv.HeaderCanonical)
+ if err := fixCanonicalChain(s.LogPrefix(), logEvery, lastHeader.Number.Uint64(), lastHeader.Hash(), tx, cfg.blockReader); err != nil {
+ return err
+ }
+
+ sn, ok := cfg.snapshots.Blocks(cfg.snapshots.BlocksAvailable())
+ if !ok {
+ return fmt.Errorf("snapshot not found for block: %d", cfg.snapshots.BlocksAvailable())
+ }
+
+ // ResetSequence - allow set arbitrary value to sequence (for example to decrement it to exact value)
+ lastTxnID := sn.Transactions.Idx.BaseDataID() + uint64(sn.Transactions.Segment.Count())
+ if err := rawdb.ResetSequence(tx, kv.EthTx, lastTxnID+1); err != nil {
+ return err
+ }
+ }
+
+ // Add last headers from snapshots to HeaderDownloader (as persistent links)
+ if s.BlockNumber < cfg.snapshots.BlocksAvailable() {
+ if err := cfg.hd.AddHeaderFromSnapshot(cfg.snapshots.BlocksAvailable(), cfg.blockReader); err != nil {
+ return err
+ }
+ if err := s.Update(tx, cfg.snapshots.BlocksAvailable()); err != nil {
+ return err
+ }
+ s.BlockNumber = cfg.snapshots.BlocksAvailable()
+ }
+
+ return nil
+}
+
+// WaitForDownloader - wait for Downloader service to download all expected snapshots
+// for MVP we sync with Downloader only once, in future will send new snapshots also
+func WaitForDownloader(ctx context.Context, tx kv.RwTx, cfg HeadersCfg) error {
+ const readyKey = "snapshots_ready"
+ v, err := tx.GetOne(kv.DatabaseInfo, []byte(readyKey))
+ if err != nil {
+ return err
+ }
+ if len(v) == 1 && v[0] == 1 {
+ return nil
+ }
+ snapshotsCfg := snapshothashes.KnownConfig(cfg.chainConfig.ChainName)
+
+ // send all hashes to the Downloader service
+ preverified := snapshotsCfg.Preverified
+ req := &proto_downloader.DownloadRequest{Items: make([]*proto_downloader.DownloadItem, len(preverified))}
+ i := 0
+ for filePath, infoHashStr := range preverified {
+ req.Items[i] = &proto_downloader.DownloadItem{
+ TorrentHash: downloadergrpc.String2Proto(infoHashStr),
+ Path: filePath,
+ }
+ i++
+ }
+ for {
+ if _, err := cfg.snapshotDownloader.Download(ctx, req); err != nil {
+ log.Error("[Snapshots] Can't call downloader", "err", err)
+ time.Sleep(10 * time.Second)
+ continue
+ }
+ break
+ }
+
+ // Print download progress until all segments are available
+ for {
+ if reply, err := cfg.snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}); err != nil {
+ log.Warn("Error while waiting for snapshots progress", "err", err)
+ } else if int(reply.Torrents) < len(snapshotsCfg.Preverified) {
+ log.Warn("Downloader has not enough snapshots (yet)")
+ } else if reply.Completed {
+ break
+ } else {
+ readiness := int32(100 * (float64(reply.BytesCompleted) / float64(reply.BytesTotal)))
+ log.Info("[Snapshots] download", "progress", fmt.Sprintf("%d%%", readiness))
+ }
+ time.Sleep(10 * time.Second)
+ }
+
+ if err := tx.Put(kv.DatabaseInfo, []byte(readyKey), []byte{1}); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/go.mod b/go.mod
index 6ce78993fc..9dbbdb79c6 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
github.com/RoaringBitmap/roaring v0.9.4
github.com/VictoriaMetrics/fastcache v1.7.0
github.com/VictoriaMetrics/metrics v1.18.1
+ github.com/anacrolix/go-libutp v1.0.4
github.com/anacrolix/log v0.10.0
github.com/anacrolix/torrent v1.38.0
github.com/btcsuite/btcd v0.21.0-beta
@@ -15,6 +16,7 @@ require (
github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498
+ github.com/dustin/go-humanize v1.0.0
github.com/edsrzf/mmap-go v1.0.0
github.com/emicklei/dot v0.16.0
github.com/fatih/color v1.12.0
@@ -35,7 +37,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
- github.com/ledgerwatch/erigon-lib v0.0.0-20211206140018-b06f3cec6bfb
+ github.com/ledgerwatch/erigon-lib v0.0.0-20211213041304-7fb44e022d47
github.com/ledgerwatch/log/v3 v3.4.0
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/logrusorgru/aurora/v3 v3.0.0
diff --git a/go.sum b/go.sum
index 8307befc2b..22d599ed2e 100644
--- a/go.sum
+++ b/go.sum
@@ -617,8 +617,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20211206140018-b06f3cec6bfb h1:bN+YFj+BSV2sMFl+TR3tPS82D7aq+2OHYRk44wwwtYk=
-github.com/ledgerwatch/erigon-lib v0.0.0-20211206140018-b06f3cec6bfb/go.mod h1:lyGP3i0x4CeabdKZ4beycD5xZfHWZwJsAX+70OfGj4Y=
+github.com/ledgerwatch/erigon-lib v0.0.0-20211213041304-7fb44e022d47 h1:Y4OJ9Z1RNNJ9GceKcFC8JVruvgrAFg+6NJr9O1qoRnU=
+github.com/ledgerwatch/erigon-lib v0.0.0-20211213041304-7fb44e022d47/go.mod h1:lyGP3i0x4CeabdKZ4beycD5xZfHWZwJsAX+70OfGj4Y=
github.com/ledgerwatch/log/v3 v3.4.0 h1:SEIOcv5a2zkG3PmoT5jeTU9m/0nEUv0BJS5bzsjwKCI=
github.com/ledgerwatch/log/v3 v3.4.0/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
diff --git a/node/config.go b/node/config.go
index d1dd53d37e..42f29409a7 100644
--- a/node/config.go
+++ b/node/config.go
@@ -69,6 +69,8 @@ type Config struct {
// Configuration of peer-to-peer networking.
P2P p2p.Config
+ DownloaderAddr string
+
// IPCPath is the requested location to place the IPC endpoint. If the path is
// a simple file name, it is placed inside the data directory (or on the root
// pipe path on Windows), whereas if it's a resolvable path name (absolute or
diff --git a/params/config.go b/params/config.go
index 5c9dff72b4..e24152259f 100644
--- a/params/config.go
+++ b/params/config.go
@@ -23,18 +23,7 @@ import (
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/paths"
-)
-
-const (
- MainnetChainName = "mainnet"
- RopstenChainName = "ropsten"
- RinkebyChainName = "rinkeby"
- GoerliChainName = "goerli"
- DevChainName = "dev"
- ErigonMineName = "erigonmine"
- SokolChainName = "sokol"
- KovanChainName = "kovan"
- FermionChainName = "fermion"
+ "github.com/ledgerwatch/erigon/params/networkname"
)
type ConsensusType string
@@ -70,7 +59,7 @@ var (
var (
// MainnetChainConfig is the chain parameters to run a node on the main network.
MainnetChainConfig = &ChainConfig{
- ChainName: MainnetChainName,
+ ChainName: networkname.MainnetChainName,
ChainID: big.NewInt(1),
Consensus: EtHashConsensus,
HomesteadBlock: big.NewInt(1_150_000),
@@ -94,7 +83,7 @@ var (
// RopstenChainConfig contains the chain parameters to run a node on the Ropsten test network.
RopstenChainConfig = &ChainConfig{
- ChainName: RopstenChainName,
+ ChainName: networkname.RopstenChainName,
ChainID: big.NewInt(3),
Consensus: EtHashConsensus,
HomesteadBlock: big.NewInt(0),
@@ -117,7 +106,7 @@ var (
// RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network.
RinkebyChainConfig = &ChainConfig{
- ChainName: RinkebyChainName,
+ ChainName: networkname.RinkebyChainName,
ChainID: big.NewInt(4),
Consensus: CliqueConsensus,
HomesteadBlock: big.NewInt(1),
@@ -143,7 +132,7 @@ var (
// GoerliChainConfig contains the chain parameters to run a node on the Görli test network.
GoerliChainConfig = &ChainConfig{
- ChainName: GoerliChainName,
+ ChainName: networkname.GoerliChainName,
ChainID: big.NewInt(5),
Consensus: CliqueConsensus,
HomesteadBlock: big.NewInt(0),
@@ -167,7 +156,7 @@ var (
// MainnetChainConfig is the chain parameters to run a PoW dev net to test Erigon mining
ErigonChainConfig = &ChainConfig{
- ChainName: ErigonMineName,
+ ChainName: networkname.ErigonMineName,
ChainID: new(big.Int).SetBytes([]byte("erigon-mine")),
Consensus: EtHashConsensus,
HomesteadBlock: big.NewInt(0),
@@ -188,7 +177,7 @@ var (
}
SokolChainConfig = &ChainConfig{
- ChainName: SokolChainName,
+ ChainName: networkname.SokolChainName,
ChainID: big.NewInt(77),
Consensus: AuRaConsensus,
HomesteadBlock: big.NewInt(0),
@@ -252,7 +241,7 @@ var (
}
KovanChainConfig = &ChainConfig{
- ChainName: KovanChainName,
+ ChainName: networkname.KovanChainName,
ChainID: big.NewInt(42),
Consensus: AuRaConsensus,
HomesteadBlock: big.NewInt(0),
@@ -273,7 +262,7 @@ var (
}
FermionChainConfig = &ChainConfig{
- ChainName: FermionChainName,
+ ChainName: networkname.FermionChainName,
ChainID: big.NewInt(1212120),
Consensus: CliqueConsensus,
HomesteadBlock: big.NewInt(0),
diff --git a/params/networkname/network_name.go b/params/networkname/network_name.go
new file mode 100644
index 0000000000..9766c6add2
--- /dev/null
+++ b/params/networkname/network_name.go
@@ -0,0 +1,13 @@
+package networkname
+
+const (
+ MainnetChainName = "mainnet"
+ RopstenChainName = "ropsten"
+ RinkebyChainName = "rinkeby"
+ GoerliChainName = "goerli"
+ DevChainName = "dev"
+ ErigonMineName = "erigonmine"
+ SokolChainName = "sokol"
+ KovanChainName = "kovan"
+ FermionChainName = "fermion"
+)
diff --git a/params/snapshots.go b/params/snapshots.go
deleted file mode 100644
index 3edca1a112..0000000000
--- a/params/snapshots.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package params
-
-var (
- MainnetChainSnapshotConfig = &SnapshotsConfig{}
- GoerliChainSnapshotConfig = &SnapshotsConfig{
- ExpectBlocks: 5_900_000 - 1,
- }
-)
-
-type SnapshotsConfig struct {
- ExpectBlocks uint64
-}
-
-func KnownSnapshots(networkName string) *SnapshotsConfig {
- switch networkName {
- case MainnetChainName:
- return MainnetChainSnapshotConfig
- case GoerliChainName:
- return GoerliChainSnapshotConfig
- default:
- return nil
- }
-}
diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go
index da2a041a45..2bf010d8f0 100644
--- a/turbo/cli/default_flags.go
+++ b/turbo/cli/default_flags.go
@@ -84,5 +84,6 @@ var DefaultFlags = []cli.Flag{
utils.MinerNoVerfiyFlag,
utils.MinerSigningKeyFileFlag,
utils.SentryAddrFlag,
+ utils.DownloaderAddrFlag,
HealthCheckFlag,
}
diff --git a/turbo/node/node.go b/turbo/node/node.go
index 58ea3eefe9..084a0f05bb 100644
--- a/turbo/node/node.go
+++ b/turbo/node/node.go
@@ -8,6 +8,7 @@ import (
"github.com/ledgerwatch/erigon/eth/ethconfig"
"github.com/ledgerwatch/erigon/node"
"github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/erigon/params/networkname"
erigoncli "github.com/ledgerwatch/erigon/turbo/cli"
"github.com/ledgerwatch/log/v3"
@@ -52,19 +53,19 @@ func NewNodConfigUrfave(ctx *cli.Context) *node.Config {
// If we're running a known preset, log it for convenience.
chain := ctx.GlobalString(utils.ChainFlag.Name)
switch chain {
- case params.RopstenChainName:
+ case networkname.RopstenChainName:
log.Info("Starting Erigon on Ropsten testnet...")
- case params.RinkebyChainName:
+ case networkname.RinkebyChainName:
log.Info("Starting Erigon on Rinkeby testnet...")
- case params.GoerliChainName:
+ case networkname.GoerliChainName:
log.Info("Starting Erigon on Görli testnet...")
- case params.DevChainName:
+ case networkname.DevChainName:
log.Info("Starting Erigon in ephemeral dev mode...")
- case "", params.MainnetChainName:
+ case "", networkname.MainnetChainName:
if !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) {
log.Info("Starting Erigon on Ethereum mainnet...")
}
diff --git a/turbo/snapshotsync/block_reader.go b/turbo/snapshotsync/block_reader.go
index 8efe49d15c..5ad1676eb8 100644
--- a/turbo/snapshotsync/block_reader.go
+++ b/turbo/snapshotsync/block_reader.go
@@ -28,6 +28,23 @@ func (back *BlockReader) Header(ctx context.Context, tx kv.Getter, hash common.H
return h, nil
}
+func (back *BlockReader) Body(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (body *types.Body, err error) {
+ body, _, _ = rawdb.ReadBody(tx, hash, blockHeight)
+ return body, nil
+}
+
+func (back *BlockReader) BodyRlp(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (bodyRlp rlp.RawValue, err error) {
+ body, err := back.Body(ctx, tx, hash, blockHeight)
+ if err != nil {
+ return nil, err
+ }
+ bodyRlp, err = rlp.EncodeToBytes(body)
+ if err != nil {
+ return nil, err
+ }
+ return bodyRlp, nil
+}
+
func (back *BlockReader) HeaderByNumber(ctx context.Context, tx kv.Getter, blockHeight uint64) (*types.Header, error) {
h := rawdb.ReadHeaderByNumber(tx, blockHeight)
return h, nil
@@ -88,6 +105,28 @@ func (back *RemoteBlockReader) Header(ctx context.Context, tx kv.Tx, hash common
}
return block.Header(), nil
}
+func (back *RemoteBlockReader) Body(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (body *types.Body, err error) {
+ block, _, err := back.BlockWithSenders(ctx, tx, hash, blockHeight)
+ if err != nil {
+ return nil, err
+ }
+ if block == nil {
+ return nil, nil
+ }
+ return block.Body(), nil
+}
+
+func (back *RemoteBlockReader) BodyRlp(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (bodyRlp rlp.RawValue, err error) {
+ body, err := back.Body(ctx, tx, hash, blockHeight)
+ if err != nil {
+ return nil, err
+ }
+ bodyRlp, err = rlp.EncodeToBytes(body)
+ if err != nil {
+ return nil, err
+ }
+ return bodyRlp, nil
+}
// BlockReaderWithSnapshots can read blocks from db and snapshots
type BlockReaderWithSnapshots struct {
@@ -106,6 +145,7 @@ func (back *BlockReaderWithSnapshots) HeaderByNumber(ctx context.Context, tx kv.
}
return back.headerFromSnapshot(blockHeight, sn)
}
+
func (back *BlockReaderWithSnapshots) Header(ctx context.Context, tx kv.Getter, hash common.Hash, blockHeight uint64) (*types.Header, error) {
sn, ok := back.sn.Blocks(blockHeight)
if !ok {
@@ -126,6 +166,50 @@ func (back *BlockReaderWithSnapshots) ReadHeaderByNumber(ctx context.Context, tx
return back.headerFromSnapshot(blockHeight, sn)
}
+func (back *BlockReaderWithSnapshots) Body(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (body *types.Body, err error) {
+ sn, ok := back.sn.Blocks(blockHeight)
+ if !ok {
+ canonicalHash, err := rawdb.ReadCanonicalHash(tx, blockHeight)
+ if err != nil {
+ return nil, fmt.Errorf("requested non-canonical hash %x. canonical=%x", hash, canonicalHash)
+ }
+ body, baseTxID, txsAmount := rawdb.ReadBody(tx, hash, blockHeight)
+ if body == nil {
+ return nil, fmt.Errorf("body not found for block %d,%x", blockHeight, hash)
+ }
+ if canonicalHash == hash {
+ body.Transactions, err = rawdb.CanonicalTransactions(tx, baseTxID, txsAmount)
+ if err != nil {
+ return nil, err
+ }
+ return body, nil
+ }
+ body.Transactions, err = rawdb.NonCanonicalTransactions(tx, baseTxID, txsAmount)
+ if err != nil {
+ return nil, err
+ }
+ return body, nil
+ }
+
+ body, _, _, _, err = back.bodyFromSnapshot(blockHeight, sn)
+ if err != nil {
+ return nil, err
+ }
+ return body, nil
+}
+
+func (back *BlockReaderWithSnapshots) BodyRlp(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (bodyRlp rlp.RawValue, err error) {
+ body, err := back.Body(ctx, tx, hash, blockHeight)
+ if err != nil {
+ return nil, err
+ }
+ bodyRlp, err = rlp.EncodeToBytes(body)
+ if err != nil {
+ return nil, err
+ }
+ return bodyRlp, nil
+}
+
func (back *BlockReaderWithSnapshots) BlockWithSenders(ctx context.Context, tx kv.Tx, hash common.Hash, blockHeight uint64) (block *types.Block, senders []common.Address, err error) {
sn, ok := back.sn.Blocks(blockHeight)
if !ok {
@@ -213,3 +297,47 @@ func (back *BlockReaderWithSnapshots) headerFromSnapshot(blockHeight uint64, sn
}
return h, nil
}
+
+func (back *BlockReaderWithSnapshots) bodyFromSnapshot(blockHeight uint64, sn *BlocksSnapshot) (*types.Body, []common.Address, uint64, uint32, error) {
+ buf := make([]byte, 16)
+
+ bodyOffset := sn.Bodies.Idx.Lookup2(blockHeight - sn.Bodies.Idx.BaseDataID())
+
+ gg := sn.Bodies.Segment.MakeGetter()
+ gg.Reset(bodyOffset)
+ buf, _ = gg.Next(buf[:0])
+ b := &types.BodyForStorage{}
+ reader := bytes.NewReader(buf)
+ if err := rlp.Decode(reader, b); err != nil {
+ return nil, nil, 0, 0, err
+ }
+
+ if b.BaseTxId < sn.Transactions.Idx.BaseDataID() {
+ return nil, nil, 0, 0, fmt.Errorf(".idx file has wrong baseDataID? %d<%d, %s", b.BaseTxId, sn.Transactions.Idx.BaseDataID(), sn.Transactions.File)
+ }
+
+ txs := make([]types.Transaction, b.TxAmount)
+ senders := make([]common.Address, b.TxAmount)
+ if b.TxAmount > 0 {
+ txnOffset := sn.Transactions.Idx.Lookup2(b.BaseTxId - sn.Transactions.Idx.BaseDataID()) // need subtract baseID of indexFile
+ gg = sn.Transactions.Segment.MakeGetter()
+ gg.Reset(txnOffset)
+ stream := rlp.NewStream(reader, 0)
+ for i := uint32(0); i < b.TxAmount; i++ {
+ buf, _ = gg.Next(buf[:0])
+ senders[i].SetBytes(buf[1 : 1+20])
+ txRlp := buf[1+20:]
+ reader.Reset(txRlp)
+ stream.Reset(reader, 0)
+ var err error
+ txs[i], err = types.DecodeTransaction(stream)
+ if err != nil {
+ return nil, nil, 0, 0, err
+ }
+ }
+ }
+
+ body := new(types.Body)
+ body.Uncles = b.Uncles
+ return body, senders, b.BaseTxId, b.TxAmount, nil
+}
diff --git a/turbo/snapshotsync/block_snapshots.go b/turbo/snapshotsync/block_snapshots.go
index 4f7c941110..eb4a18217b 100644
--- a/turbo/snapshotsync/block_snapshots.go
+++ b/turbo/snapshotsync/block_snapshots.go
@@ -30,8 +30,8 @@ import (
"github.com/ledgerwatch/erigon/common/dbutils"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/core/types"
- "github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/rlp"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/log/v3"
)
@@ -43,6 +43,8 @@ const (
Transactions SnapshotType = "transactions"
)
+var AllSnapshotTypes = []SnapshotType{Headers, Bodies, Transactions}
+
var (
ErrInvalidCompressedFileName = fmt.Errorf("invalid compressed file name")
)
@@ -90,7 +92,7 @@ type AllSnapshots struct {
segmentsAvailable uint64
idxAvailable uint64
blocks []*BlocksSnapshot
- cfg *params.SnapshotsConfig
+ cfg *snapshothashes.Config
}
// NewAllSnapshots - opens all snapshots. But to simplify everything:
@@ -98,20 +100,20 @@ type AllSnapshots struct {
// - all snapshots of given blocks range must exist - to make this blocks range available
// - gaps are not allowed
// - segment have [from:to) semantic
-func NewAllSnapshots(dir string, cfg *params.SnapshotsConfig) *AllSnapshots {
+func NewAllSnapshots(dir string, cfg *snapshothashes.Config) *AllSnapshots {
if err := os.MkdirAll(dir, 0755); err != nil {
panic(err)
}
return &AllSnapshots{dir: dir, cfg: cfg}
}
-func (s *AllSnapshots) ChainSnapshotConfig() *params.SnapshotsConfig { return s.cfg }
-func (s *AllSnapshots) AllSegmentsAvailable() bool { return s.allSegmentsAvailable }
-func (s *AllSnapshots) SetAllSegmentsAvailable(v bool) { s.allSegmentsAvailable = v }
-func (s *AllSnapshots) BlocksAvailable() uint64 { return s.segmentsAvailable }
-func (s *AllSnapshots) AllIdxAvailable() bool { return s.allIdxAvailable }
-func (s *AllSnapshots) SetAllIdxAvailable(v bool) { s.allIdxAvailable = v }
-func (s *AllSnapshots) IndicesAvailable() uint64 { return s.idxAvailable }
+func (s *AllSnapshots) ChainSnapshotConfig() *snapshothashes.Config { return s.cfg }
+func (s *AllSnapshots) AllSegmentsAvailable() bool { return s.allSegmentsAvailable }
+func (s *AllSnapshots) SetAllSegmentsAvailable(v bool) { s.allSegmentsAvailable = v }
+func (s *AllSnapshots) BlocksAvailable() uint64 { return s.segmentsAvailable }
+func (s *AllSnapshots) AllIdxAvailable() bool { return s.allIdxAvailable }
+func (s *AllSnapshots) SetAllIdxAvailable(v bool) { s.allIdxAvailable = v }
+func (s *AllSnapshots) IndicesAvailable() uint64 { return s.idxAvailable }
func (s *AllSnapshots) SegmentsAvailability() (headers, bodies, txs uint64, err error) {
if headers, err = latestSegment(s.dir, Headers); err != nil {
@@ -137,37 +139,51 @@ func (s *AllSnapshots) IdxAvailability() (headers, bodies, txs uint64, err error
}
return
}
+
func (s *AllSnapshots) ReopenIndices() error {
+ return s.ReopenSomeIndices(AllSnapshotTypes...)
+}
+
+func (s *AllSnapshots) ReopenSomeIndices(types ...SnapshotType) error {
for _, bs := range s.blocks {
- if bs.Headers.Idx != nil {
- bs.Headers.Idx.Close()
- bs.Headers.Idx = nil
- }
- idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Headers.From, bs.Headers.To, Headers)))
- if err != nil {
- return err
- }
- bs.Headers.Idx = idx
+ for _, snapshotType := range types {
+ switch snapshotType {
+ case Headers:
+ if bs.Headers.Idx != nil {
+ bs.Headers.Idx.Close()
+ bs.Headers.Idx = nil
+ }
+ idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Headers.From, bs.Headers.To, Headers)))
+ if err != nil {
+ return err
+ }
+ bs.Headers.Idx = idx
+ case Bodies:
+ if bs.Bodies.Idx != nil {
+ bs.Bodies.Idx.Close()
+ bs.Bodies.Idx = nil
+ }
+ idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Bodies.From, bs.Bodies.To, Bodies)))
+ if err != nil {
+ return err
+ }
+ bs.Bodies.Idx = idx
- if bs.Bodies.Idx != nil {
- bs.Bodies.Idx.Close()
- bs.Bodies.Idx = nil
- }
- idx, err = recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Bodies.From, bs.Bodies.To, Bodies)))
- if err != nil {
- return err
+ case Transactions:
+ if bs.Transactions.Idx != nil {
+ bs.Transactions.Idx.Close()
+ bs.Transactions.Idx = nil
+ }
+ idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Transactions.From, bs.Transactions.To, Transactions)))
+ if err != nil {
+ return err
+ }
+ bs.Transactions.Idx = idx
+ default:
+ panic(fmt.Sprintf("unknown snapshot type: %s", snapshotType))
+ }
}
- bs.Bodies.Idx = idx
- if bs.Transactions.Idx != nil {
- bs.Transactions.Idx.Close()
- bs.Transactions.Idx = nil
- }
- idx, err = recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Transactions.From, bs.Transactions.To, Transactions)))
- if err != nil {
- return err
- }
- bs.Transactions.Idx = idx
s.idxAvailable = bs.Transactions.To - 1
}
return nil
@@ -274,7 +290,6 @@ func (s *AllSnapshots) Blocks(blockNumber uint64) (snapshot *BlocksSnapshot, fou
}
func (s *AllSnapshots) BuildIndices(ctx context.Context, chainID uint256.Int) error {
- fmt.Printf("build!\n")
for _, sn := range s.blocks {
f := path.Join(s.dir, SegmentFileName(sn.Headers.From, sn.Headers.To, Headers))
if err := HeadersHashIdx(f, sn.Headers.From); err != nil {
@@ -288,31 +303,33 @@ func (s *AllSnapshots) BuildIndices(ctx context.Context, chainID uint256.Int) er
}
// hack to read first block body - to get baseTxId from there
- _ = s.ReopenIndices()
+ if err := s.ReopenSomeIndices(Headers, Bodies); err != nil {
+ return err
+ }
+
for _, sn := range s.blocks {
gg := sn.Bodies.Segment.MakeGetter()
buf, _ := gg.Next(nil)
- b := &types.BodyForStorage{}
- if err := rlp.DecodeBytes(buf, b); err != nil {
+ firstBody := &types.BodyForStorage{}
+ if err := rlp.DecodeBytes(buf, firstBody); err != nil {
return err
}
var expectedTxsAmount uint64
{
- off := sn.Bodies.Idx.Lookup2(sn.To - 1)
+ off := sn.Bodies.Idx.Lookup2(sn.To - 1 - sn.From)
gg.Reset(off)
- buf, _ = gg.Next(nil)
- bodyForStorage := new(types.BodyForStorage)
- err := rlp.DecodeBytes(buf, bodyForStorage)
+ buf, _ = gg.Next(buf[:0])
+ lastBody := new(types.BodyForStorage)
+ err := rlp.DecodeBytes(buf, lastBody)
if err != nil {
- panic(err)
+ return err
}
- expectedTxsAmount = bodyForStorage.BaseTxId + uint64(bodyForStorage.TxAmount) - b.BaseTxId
+ expectedTxsAmount = lastBody.BaseTxId + uint64(lastBody.TxAmount) - firstBody.BaseTxId
}
f := path.Join(s.dir, SegmentFileName(sn.Transactions.From, sn.Transactions.To, Transactions))
- fmt.Printf("create: %s\n", f)
- if err := TransactionsHashIdx(chainID, b.BaseTxId, f, expectedTxsAmount); err != nil {
+ if err := TransactionsHashIdx(chainID, firstBody.BaseTxId, f, expectedTxsAmount); err != nil {
return err
}
}
diff --git a/turbo/snapshotsync/block_snapshots_test.go b/turbo/snapshotsync/block_snapshots_test.go
index 84f8e4632b..778fe2b958 100644
--- a/turbo/snapshotsync/block_snapshots_test.go
+++ b/turbo/snapshotsync/block_snapshots_test.go
@@ -6,13 +6,14 @@ import (
"github.com/ledgerwatch/erigon-lib/compress"
"github.com/ledgerwatch/erigon-lib/recsplit"
- "github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/erigon/params/networkname"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/stretchr/testify/require"
)
func TestOpenAllSnapshot(t *testing.T) {
dir, require := t.TempDir(), require.New(t)
- cfg := params.KnownSnapshots(params.MainnetChainName)
+ cfg := snapshothashes.KnownConfig(networkname.MainnetChainName)
createFile := func(from, to uint64, name SnapshotType) {
c, err := compress.NewCompressor("test", path.Join(dir, SegmentFileName(from, to, name)), dir, 100)
require.NoError(err)
diff --git a/turbo/snapshotsync/bodies_snapshot.go b/turbo/snapshotsync/bodies_snapshot.go
deleted file mode 100644
index adcd09ff16..0000000000
--- a/turbo/snapshotsync/bodies_snapshot.go
+++ /dev/null
@@ -1 +0,0 @@
-package snapshotsync
diff --git a/turbo/snapshotsync/build_infobytes.go b/turbo/snapshotsync/build_infobytes.go
deleted file mode 100644
index 839970f34b..0000000000
--- a/turbo/snapshotsync/build_infobytes.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package snapshotsync
-
-import (
- "fmt"
- "io"
- "os"
- "path/filepath"
- "strings"
-
- "github.com/anacrolix/torrent/metainfo"
-)
-
-func BuildInfoBytesForSnapshot(root string, fileName string) (metainfo.Info, error) {
-
- path := filepath.Join(root, fileName)
- fi, err := os.Stat(path)
- if err != nil {
- return metainfo.Info{}, err
- }
- relPath, err := filepath.Rel(root, path)
- if err != nil {
- return metainfo.Info{}, fmt.Errorf("error getting relative path: %s", err)
- }
-
- info := metainfo.Info{
- Name: filepath.Base(root),
- PieceLength: DefaultChunkSize,
- Length: fi.Size(),
- Files: []metainfo.FileInfo{
- {
- Length: fi.Size(),
- Path: []string{relPath},
- PathUTF8: nil,
- },
- },
- }
-
- err = info.GeneratePieces(func(fi metainfo.FileInfo) (io.ReadCloser, error) {
- return os.Open(filepath.Join(root, strings.Join(fi.Path, string(filepath.Separator))))
- })
- if err != nil {
- err = fmt.Errorf("error generating pieces: %s", err)
- return metainfo.Info{}, err
- }
- return info, nil
-}
diff --git a/turbo/snapshotsync/client.go b/turbo/snapshotsync/client.go
index eb3e4a7d88..707640cfb1 100644
--- a/turbo/snapshotsync/client.go
+++ b/turbo/snapshotsync/client.go
@@ -1,14 +1,14 @@
package snapshotsync
import (
- "github.com/ledgerwatch/erigon-lib/gointerfaces/snapshotsync"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
"google.golang.org/grpc"
)
//go:generate ls ./../../interfaces/snapshot_downloader
//go:generate protoc --go_out=. --go-grpc_out=. --proto_path=./../../interfaces/snapshot_downloader "external_downloader.proto" -I=. -I=./../../build/include/google
-func NewClient(addr string) (snapshotsync.DownloaderClient, func() error, error) {
+func NewClient(addr string) (proto_downloader.DownloaderClient, func() error, error) {
opts := []grpc.DialOption{
grpc.WithInsecure(),
}
@@ -18,5 +18,5 @@ func NewClient(addr string) (snapshotsync.DownloaderClient, func() error, error)
return nil, nil, err
}
- return snapshotsync.NewDownloaderClient(conn), conn.Close, nil
+ return proto_downloader.NewDownloaderClient(conn), conn.Close, nil
}
diff --git a/turbo/snapshotsync/const.go b/turbo/snapshotsync/const.go
index 699f44c34f..1056653790 100644
--- a/turbo/snapshotsync/const.go
+++ b/turbo/snapshotsync/const.go
@@ -2,367 +2,8 @@ package snapshotsync
import (
"errors"
-
- "github.com/anacrolix/torrent/metainfo"
- "github.com/ledgerwatch/erigon-lib/gointerfaces/snapshotsync"
- "github.com/ledgerwatch/erigon/params"
-)
-
-const (
- DefaultChunkSize = 1024 * 1024
- MdbxFilename = "mdbx.dat"
- EpochSize = 500_000
-
- //todo It'll be changed after enabling new snapshot generation mechanism
- HeadersSnapshotHash = "0000000000000000000000000000000000000000"
- BlocksSnapshotHash = "0000000000000000000000000000000000000000"
- StateSnapshotHash = "0000000000000000000000000000000000000000"
-
- SnapshotInfoHashPrefix = "ih"
- SnapshotInfoBytesPrefix = "ib"
)
var (
- TorrentHashes = map[uint64]map[snapshotsync.SnapshotType]metainfo.Hash{
- params.MainnetChainConfig.ChainID.Uint64(): {
- snapshotsync.SnapshotType_headers: metainfo.NewHashFromHex(HeadersSnapshotHash),
- snapshotsync.SnapshotType_bodies: metainfo.NewHashFromHex(BlocksSnapshotHash),
- snapshotsync.SnapshotType_state: metainfo.NewHashFromHex(StateSnapshotHash),
- },
- }
ErrInvalidSnapshot = errors.New("this snapshot for this chainID not supported ")
)
-
-func GetAvailableSnapshotTypes(chainID uint64) []snapshotsync.SnapshotType {
- v := TorrentHashes[chainID]
- res := make([]snapshotsync.SnapshotType, 0, len(v))
- for i := range v {
- res = append(res, i)
- }
- return res
-}
-
-var Trackers = [][]string{{
- "http://35.189.110.210:80/announce",
-}, {
- "udp://tracker.openbittorrent.com:80",
- "udp://tracker.openbittorrent.com:80",
- "udp://tracker.publicbt.com:80",
- "udp://coppersurfer.tk:6969/announce",
- "udp://open.demonii.com:1337",
- "http://bttracker.crunchbanglinux.org:6969/announce",
- "udp://wambo.club:1337/announce",
- "udp://tracker.dutchtracking.com:6969/announce",
- "udp://tc.animereactor.ru:8082/announce",
- "udp://tracker.justseed.it:1337/announce",
- "udp://tracker.leechers-paradise.org:6969/announce",
- "udp://tracker.opentrackr.org:1337/announce",
- "https://open.kickasstracker.com:443/announce",
- "udp://tracker.coppersurfer.tk:6969/announce",
- "udp://open.stealth.si:80/announce",
- "http://87.253.152.137/announce",
- "http://91.216.110.47/announce",
- "http://91.217.91.21:3218/announce",
- "http://91.218.230.81:6969/announce",
- "http://93.92.64.5/announce",
- "http://atrack.pow7.com/announce",
- "http://bt.henbt.com:2710/announce",
- "http://bt.pusacg.org:8080/announce",
- "https://tracker.bt-hash.com:443/announce",
- "udp://tracker.leechers-paradise.org:6969",
- "https://182.176.139.129:6969/announce",
- "udp://zephir.monocul.us:6969/announce",
- "https://tracker.dutchtracking.com:80/announce",
- "https://grifon.info:80/announce",
- "udp://tracker.kicks-ass.net:80/announce",
- "udp://p4p.arenabg.com:1337/announce",
- "udp://tracker.aletorrenty.pl:2710/announce",
- "udp://tracker.sktorrent.net:6969/announce",
- "udp://tracker.internetwarriors.net:1337/announce",
- "https://tracker.parrotsec.org:443/announce",
- "https://tracker.moxing.party:6969/announce",
- "https://tracker.ipv6tracker.ru:80/announce",
- "https://tracker.fastdownload.xyz:443/announce",
- "udp://open.stealth.si:80/announce",
- "https://gwp2-v19.rinet.ru:80/announce",
- "https://tr.kxmp.cf:80/announce",
- "https://explodie.org:6969/announce",
-}, {
- "udp://zephir.monocul.us:6969/announce",
- "udp://tracker.torrent.eu.org:451/announce",
- "udp://tracker.uw0.xyz:6969/announce",
- "udp://tracker.cyberia.is:6969/announce",
- "http://tracker.files.fm:6969/announce",
- "udp://tracker.zum.bi:6969/announce",
- "http://tracker.nyap2p.com:8080/announce",
- "udp://opentracker.i2p.rocks:6969/announce",
- "udp://tracker.zerobytes.xyz:1337/announce",
- "https://tracker.tamersunion.org:443/announce",
- "https://w.wwwww.wtf:443/announce",
- "https://tracker.imgoingto.icu:443/announce",
- "udp://blokas.io:6969/announce",
- "udp://api.bitumconference.ru:6969/announce",
- "udp://discord.heihachi.pw:6969/announce",
- "udp://cutiegirl.ru:6969/announce",
- "udp://fe.dealclub.de:6969/announce",
- "udp://ln.mtahost.co:6969/announce",
- "udp://vibe.community:6969/announce",
- "http://vpn.flying-datacenter.de:6969/announce",
- "udp://eliastre100.fr:6969/announce",
- "udp://wassermann.online:6969/announce",
- "udp://retracker.local.msn-net.ru:6969/announce",
- "udp://chanchan.uchuu.co.uk:6969/announce",
- "udp://kanal-4.de:6969/announce",
- "udp://handrew.me:6969/announce",
- "udp://mail.realliferpg.de:6969/announce",
- "udp://bubu.mapfactor.com:6969/announce",
- "udp://mts.tvbit.co:6969/announce",
- "udp://6ahddutb1ucc3cp.ru:6969/announce",
- "udp://adminion.n-blade.ru:6969/announce",
- "udp://contra.sf.ca.us:6969/announce",
- "udp://61626c.net:6969/announce",
- "udp://benouworldtrip.fr:6969/announce",
- "udp://sd-161673.dedibox.fr:6969/announce",
- "udp://cdn-1.gamecoast.org:6969/announce",
- "udp://cdn-2.gamecoast.org:6969/announce",
- "udp://daveking.com:6969/announce",
- "udp://bms-hosxp.com:6969/announce",
- "udp://teamspeak.value-wolf.org:6969/announce",
- "udp://edu.uifr.ru:6969/announce",
- "udp://adm.category5.tv:6969/announce",
- "udp://code2chicken.nl:6969/announce",
- "udp://t1.leech.ie:1337/announce",
- "udp://forever-tracker.zooki.xyz:6969/announce",
- "udp://free-tracker.zooki.xyz:6969/announce",
- "udp://public.publictracker.xyz:6969/announce",
- "udp://public-tracker.zooki.xyz:6969/announce",
- "udp://vps2.avc.cx:7171/announce",
- "udp://tracker.fileparadise.in:1337/announce",
- "udp://tracker.skynetcloud.site:6969/announce",
- "udp://z.mercax.com:53/announce",
- "https://publictracker.pp.ua:443/announce",
- "udp://us-tracker.publictracker.xyz:6969/announce",
- "udp://open.stealth.si:80/announce",
- "http://tracker1.itzmx.com:8080/announce",
- "http://vps02.net.orel.ru:80/announce",
- "http://tracker.gbitt.info:80/announce",
- "http://tracker.bt4g.com:2095/announce",
- "https://tracker.nitrix.me:443/announce",
- "udp://aaa.army:8866/announce",
- "udp://tracker.vulnix.sh:6969/announce",
- "udp://engplus.ru:6969/announce",
- "udp://movies.zsw.ca:6969/announce",
- "udp://storage.groupees.com:6969/announce",
- "udp://nagios.tks.sumy.ua:80/announce",
- "udp://tracker.v6speed.org:6969/announce",
- "udp://47.ip-51-68-199.eu:6969/announce",
- "udp://aruacfilmes.com.br:6969/announce",
- "https://trakx.herokuapp.com:443/announce",
- "udp://inferno.demonoid.is:3391/announce",
- "udp://publictracker.xyz:6969/announce",
- "http://tracker2.itzmx.com:6961/announce",
- "http://tracker3.itzmx.com:6961/announce",
- "udp://retracker.akado-ural.ru:80/announce",
- "udp://tracker-udp.gbitt.info:80/announce",
- "http://h4.trakx.nibba.trade:80/announce",
- "udp://tracker.army:6969/announce",
- "http://tracker.anonwebz.xyz:8080/announce",
- "udp://tracker.shkinev.me:6969/announce",
- "http://0205.uptm.ch:6969/announce",
- "udp://tracker.zooki.xyz:6969/announce",
- "udp://forever.publictracker.xyz:6969/announce",
- "udp://tracker.moeking.me:6969/announce",
- "udp://ultra.zt.ua:6969/announce",
- "udp://tracker.publictracker.xyz:6969/announce",
- "udp://ipv4.tracker.harry.lu:80/announce",
- "udp://u.wwwww.wtf:1/announce",
- "udp://line-net.ru:6969/announce",
- "udp://dpiui.reedlan.com:6969/announce",
- "udp://tracker.zemoj.com:6969/announce",
- "udp://t3.leech.ie:1337/announce",
- "http://t.nyaatracker.com:80/announce",
- "udp://exodus.desync.com:6969/announce",
- "udp://valakas.rollo.dnsabr.com:2710/announce",
- "udp://tracker.ds.is:6969/announce",
- "udp://tracker.opentrackr.org:1337/announce",
- "udp://tracker0.ufibox.com:6969/announce",
- "https://tracker.hama3.net:443/announce",
- "udp://opentor.org:2710/announce",
- "udp://t2.leech.ie:1337/announce",
- "https://1337.abcvg.info:443/announce",
- "udp://git.vulnix.sh:6969/announce",
- "udp://retracker.lanta-net.ru:2710/announce",
- "udp://tracker.lelux.fi:6969/announce",
- "udp://bt1.archive.org:6969/announce",
- "udp://admin.videoenpoche.info:6969/announce",
- "udp://drumkitx.com:6969/announce",
- "udp://tracker.dler.org:6969/announce",
- "udp://koli.services:6969/announce",
- "udp://tracker.dyne.org:6969/announce",
- "http://torrenttracker.nwc.acsalaska.net:6969/announce",
- "udp://rutorrent.frontline-mod.com:6969/announce",
- "http://rt.tace.ru:80/announce",
- "udp://explodie.org:6969/announce",
-}, {
- "udp://public.popcorn-tracker.org:6969/announce",
- "http://104.28.1.30:8080/announce",
- "http://104.28.16.69/announce",
- "http://107.150.14.110:6969/announce",
- "http://109.121.134.121:1337/announce",
- "http://114.55.113.60:6969/announce",
- "http://125.227.35.196:6969/announce",
- "http://128.199.70.66:5944/announce",
- "http://157.7.202.64:8080/announce",
- "http://158.69.146.212:7777/announce",
- "http://173.254.204.71:1096/announce",
- "http://178.175.143.27/announce",
- "http://178.33.73.26:2710/announce",
- "http://182.176.139.129:6969/announce",
- "http://185.5.97.139:8089/announce",
- "http://188.165.253.109:1337/announce",
- "http://194.106.216.222/announce",
- "http://195.123.209.37:1337/announce",
- "http://210.244.71.25:6969/announce",
- "http://210.244.71.26:6969/announce",
- "http://213.159.215.198:6970/announce",
- "http://213.163.67.56:1337/announce",
- "http://37.19.5.139:6969/announce",
- "http://37.19.5.155:6881/announce",
- "http://46.4.109.148:6969/announce",
- "http://5.79.249.77:6969/announce",
- "http://5.79.83.193:2710/announce",
- "http://51.254.244.161:6969/announce",
- "http://59.36.96.77:6969/announce",
- "http://74.82.52.209:6969/announce",
- "http://80.246.243.18:6969/announce",
- "http://81.200.2.231/announce",
- "http://85.17.19.180/announce",
- "http://87.248.186.252:8080/announce",
- "http://87.253.152.137/announce",
- "http://91.216.110.47/announce",
- "http://91.217.91.21:3218/announce",
- "http://91.218.230.81:6969/announce",
- "http://93.92.64.5/announce",
- "http://atrack.pow7.com/announce",
- "http://bt.henbt.com:2710/announce",
- "http://bt.pusacg.org:8080/announce",
- "http://bt2.careland.com.cn:6969/announce",
- "http://explodie.org:6969/announce",
- "http://mgtracker.org:2710/announce",
- "http://mgtracker.org:6969/announce",
- "http://open.acgtracker.com:1096/announce",
- "http://open.lolicon.eu:7777/announce",
- "http://open.touki.ru/announce.php",
- "http://p4p.arenabg.ch:1337/announce",
- "http://p4p.arenabg.com:1337/announce",
- "http://pow7.com:80/announce",
- "http://retracker.gorcomnet.ru/announce",
- "http://retracker.krs-ix.ru/announce",
- "http://retracker.krs-ix.ru:80/announce",
- "http://secure.pow7.com/announce",
- "http://t1.pow7.com/announce",
- "http://t2.pow7.com/announce",
- "http://thetracker.org:80/announce",
- "http://torrent.gresille.org/announce",
- "http://torrentsmd.com:8080/announce",
- "http://tracker.aletorrenty.pl:2710/announce",
- "http://tracker.baravik.org:6970/announce",
- "http://tracker.bittor.pw:1337/announce",
- "http://tracker.bittorrent.am/announce",
- "http://tracker.calculate.ru:6969/announce",
- "http://tracker.dler.org:6969/announce",
- "http://tracker.dutchtracking.com/announce",
- "http://tracker.dutchtracking.com:80/announce",
- "http://tracker.dutchtracking.nl/announce",
- "http://tracker.dutchtracking.nl:80/announce",
- "http://tracker.edoardocolombo.eu:6969/announce",
- "http://tracker.ex.ua/announce",
- "http://tracker.ex.ua:80/announce",
- "http://tracker.filetracker.pl:8089/announce",
- "http://tracker.flashtorrents.org:6969/announce",
- "http://tracker.grepler.com:6969/announce",
- "http://tracker.internetwarriors.net:1337/announce",
- "http://tracker.kicks-ass.net/announce",
- "http://tracker.kicks-ass.net:80/announce",
- "http://tracker.kuroy.me:5944/announce",
- "http://tracker.mg64.net:6881/announce",
- "http://tracker.opentrackr.org:1337/announce",
- "http://tracker.skyts.net:6969/announce",
- "http://tracker.tfile.me/announce",
- "http://tracker.tiny-vps.com:6969/announce",
- "http://tracker.tvunderground.org.ru:3218/announce",
- "http://tracker.yoshi210.com:6969/announc",
- "http://tracker1.wasabii.com.tw:6969/announce",
- "http://tracker2.itzmx.com:6961/announce",
- "http://tracker2.wasabii.com.tw:6969/announce",
- "http://www.wareztorrent.com/announce",
- "http://www.wareztorrent.com:80/announce",
- "https://104.28.17.69/announce",
- "https://www.wareztorrent.com/announce",
- "udp://107.150.14.110:6969/announce",
- "udp://109.121.134.121:1337/announce",
- "udp://114.55.113.60:6969/announce",
- "udp://128.199.70.66:5944/announce",
- "udp://151.80.120.114:2710/announce",
- "udp://168.235.67.63:6969/announce",
- "udp://178.33.73.26:2710/announce",
- "udp://182.176.139.129:6969/announce",
- "udp://185.5.97.139:8089/announce",
- "udp://185.86.149.205:1337/announce",
- "udp://188.165.253.109:1337/announce",
- "udp://191.101.229.236:1337/announce",
- "udp://194.106.216.222:80/announce",
- "udp://195.123.209.37:1337/announce",
- "udp://195.123.209.40:80/announce",
- "udp://208.67.16.113:8000/announce",
- "udp://213.163.67.56:1337/announce",
- "udp://37.19.5.155:2710/announce",
- "udp://46.4.109.148:6969/announce",
- "udp://5.79.249.77:6969/announce",
- "udp://5.79.83.193:6969/announce",
- "udp://51.254.244.161:6969/announce",
- "udp://62.138.0.158:6969/announce",
- "udp://62.212.85.66:2710/announce",
- "udp://74.82.52.209:6969/announce",
- "udp://85.17.19.180:80/announce",
- "udp://89.234.156.205:80/announce",
- "udp://9.rarbg.com:2710/announce",
- "udp://9.rarbg.me:2780/announce",
- "udp://9.rarbg.to:2730/announce",
- "udp://91.218.230.81:6969/announce",
- "udp://94.23.183.33:6969/announce",
- "udp://bt.xxx-tracker.com:2710/announce",
- "udp://eddie4.nl:6969/announce",
- "udp://explodie.org:6969/announce",
- "udp://mgtracker.org:2710/announce",
- "udp://open.stealth.si:80/announce",
- "udp://p4p.arenabg.com:1337/announce",
- "udp://shadowshq.eddie4.nl:6969/announce",
- "udp://shadowshq.yi.org:6969/announce",
- "udp://torrent.gresille.org:80/announce",
- "udp://tracker.aletorrenty.pl:2710/announce",
- "udp://tracker.bittor.pw:1337/announce",
- "udp://tracker.coppersurfer.tk:6969/announce",
- "udp://tracker.eddie4.nl:6969/announce",
- "udp://tracker.ex.ua:80/announce",
- "udp://tracker.filetracker.pl:8089/announce",
- "udp://tracker.flashtorrents.org:6969/announce",
- "udp://tracker.grepler.com:6969/announce",
- "udp://tracker.ilibr.org:80/announce",
- "udp://tracker.internetwarriors.net:1337/announce",
- "udp://tracker.kicks-ass.net:80/announce",
- "udp://tracker.kuroy.me:5944/announce",
- "udp://tracker.leechers-paradise.org:6969/announce",
- "udp://tracker.mg64.net:2710/announce",
- "udp://tracker.mg64.net:6969/announce",
- "udp://tracker.opentrackr.org:1337/announce",
- "udp://tracker.piratepublic.com:1337/announce",
- "udp://tracker.sktorrent.net:6969/announce",
- "udp://tracker.skyts.net:6969/announce",
- "udp://tracker.tiny-vps.com:6969/announce",
- "udp://tracker.yoshi210.com:6969/announce",
- "udp://tracker2.indowebster.com:6969/announce",
- "udp://tracker4.piratux.com:6969/announce",
- "udp://zer0day.ch:1337/announce",
- "udp://zer0day.to:1337/announce",
-}}
diff --git a/turbo/snapshotsync/downloader.go b/turbo/snapshotsync/downloader.go
deleted file mode 100644
index fecfe84c63..0000000000
--- a/turbo/snapshotsync/downloader.go
+++ /dev/null
@@ -1,382 +0,0 @@
-package snapshotsync
-
-import (
- "bytes"
- "context"
- "encoding/binary"
- "errors"
- "fmt"
- "path/filepath"
- "time"
-
- lg "github.com/anacrolix/log"
- "github.com/anacrolix/torrent/bencode"
- "github.com/ledgerwatch/erigon-lib/gointerfaces/snapshotsync"
- "github.com/ledgerwatch/erigon-lib/kv"
- "github.com/ledgerwatch/log/v3"
-
- "github.com/anacrolix/torrent"
- "github.com/anacrolix/torrent/metainfo"
- "github.com/ledgerwatch/erigon/common"
- "github.com/ledgerwatch/erigon/common/debug"
- "github.com/ledgerwatch/erigon/ethdb"
-)
-
-type Client struct {
- Cli *torrent.Client
- snapshotsDir string
- trackers [][]string
-}
-
-func New(snapshotsDir string, seeding bool, peerID string) (*Client, error) {
- torrentConfig := DefaultTorrentConfig()
- torrentConfig.Seed = seeding
- torrentConfig.DataDir = snapshotsDir
- torrentConfig.UpnpID = torrentConfig.UpnpID + "leecher"
- torrentConfig.PeerID = peerID
-
- torrentClient, err := torrent.NewClient(torrentConfig)
- if err != nil {
- log.Error("Fail to start torrnet client", "err", err)
- return nil, fmt.Errorf("fail to start: %w", err)
- }
-
- return &Client{
- Cli: torrentClient,
- snapshotsDir: snapshotsDir,
- trackers: Trackers,
- }, nil
-}
-
-func DefaultTorrentConfig() *torrent.ClientConfig {
- torrentConfig := torrent.NewDefaultClientConfig()
- torrentConfig.ListenPort = 0
- torrentConfig.NoDHT = true
- torrentConfig.DisableTrackers = false
- torrentConfig.Debug = false
- torrentConfig.Logger = NewAdapterLogger()
- torrentConfig.Logger = torrentConfig.Logger.FilterLevel(lg.Info)
- return torrentConfig
-}
-
-func (cli *Client) Torrents() []metainfo.Hash {
- t := cli.Cli.Torrents()
- hashes := make([]metainfo.Hash, 0, len(t))
- for _, v := range t {
- hashes = append(hashes, v.InfoHash())
- }
- return hashes
-}
-func (cli *Client) Load(tx kv.Tx) error {
- log.Info("Load added torrents")
- return tx.ForEach(kv.SnapshotInfo, []byte{}, func(k, infoHashBytes []byte) error {
- if !bytes.HasPrefix(k[8:], []byte(SnapshotInfoHashPrefix)) {
- return nil
- }
- networkID, snapshotName := ParseInfoHashKey(k)
- infoHash := metainfo.Hash{}
- copy(infoHash[:], infoHashBytes)
- infoBytes, err := tx.GetOne(kv.SnapshotInfo, MakeInfoBytesKey(snapshotName, networkID))
- if err != nil {
- return err
- }
-
- log.Info("Add torrent", "snapshot", snapshotName, "hash", infoHash.String(), "infobytes", len(infoBytes) > 0)
- _, err = cli.AddTorrentSpec(snapshotName, infoHash, infoBytes)
- if err != nil {
- return err
- }
-
- return nil
- })
-}
-
-func (cli *Client) SavePeerID(db kv.Putter) error {
- return db.Put(kv.BittorrentInfo, []byte(kv.BittorrentPeerID), cli.PeerID())
-}
-
-func (cli *Client) Close() {
- cli.Cli.Close()
-}
-
-func (cli *Client) PeerID() []byte {
- peerID := cli.Cli.PeerID()
- return peerID[:]
-}
-func (cli *Client) AddTorrentSpec(snapshotName string, snapshotHash metainfo.Hash, infoBytes []byte) (*torrent.Torrent, error) {
- t, ok := cli.Cli.Torrent(snapshotHash)
- if ok {
- return t, nil
- }
- t, _, err := cli.Cli.AddTorrentSpec(&torrent.TorrentSpec{
- Trackers: Trackers,
- InfoHash: snapshotHash,
- DisplayName: snapshotName,
- InfoBytes: infoBytes,
- })
- return t, err
-}
-
-type torrentSpecFromDb struct {
- exists bool
- snapshotType snapshotsync.SnapshotType
- networkID uint64
- infoHash torrent.InfoHash
- infoBytes []byte
-}
-
-func (cli *Client) AddTorrent(ctx context.Context, spec *torrentSpecFromDb) (*torrentSpecFromDb, error) { //nolint: interfacer
- newTorrent := false
- if !spec.exists {
- log.Info("Init new torrent", "snapshot", spec.snapshotType.String())
- newTorrent = true
- var ok bool
- spec.infoHash, ok = TorrentHashes[spec.networkID][spec.snapshotType]
- if !ok {
- return nil, fmt.Errorf("%w type %v, networkID %v", ErrInvalidSnapshot, spec.snapshotType, spec.networkID)
- }
- }
- log.Info("Added torrent spec", "snapshot", spec.snapshotType.String(), "hash", spec.infoHash.String())
- t, err := cli.AddTorrentSpec(spec.snapshotType.String(), spec.infoHash, spec.infoBytes)
- if err != nil {
- return nil, fmt.Errorf("error on add snapshot: %w", err)
- }
- log.Info("Getting infobytes", "snapshot", spec.snapshotType.String())
- spec.infoBytes, err = cli.GetInfoBytes(context.Background(), spec.infoHash)
- if err != nil {
- log.Warn("Init failure", "snapshot", spec.snapshotType.String(), "err", ctx.Err())
- return nil, fmt.Errorf("error on get info bytes: %w", err)
- }
- t.AllowDataDownload()
- t.DownloadAll()
- log.Info("Got infobytes", "snapshot", spec.snapshotType.String(), "file", t.Files()[0].Path())
-
- if newTorrent {
- return spec, nil
- }
- return nil, nil
-}
-
-func (cli *Client) GetInfoBytes(ctx context.Context, snapshotHash metainfo.Hash) ([]byte, error) {
- t, ok := cli.Cli.Torrent(snapshotHash)
- if !ok {
- return nil, errors.New("torrent not added")
- }
- for {
- select {
- case <-ctx.Done():
- return nil, fmt.Errorf("add torrent timeout: %w", ctx.Err())
- case <-t.GotInfo():
- return common.CopyBytes(t.Metainfo().InfoBytes), nil
- default:
- log.Info("Searching infobytes", "seeders", t.Stats().ConnectedSeeders, "active peers", t.Stats().ActivePeers)
- time.Sleep(time.Second * 60)
- }
- }
-}
-
-func (cli *Client) Download() {
- log.Info("Start snapshot downloading")
- torrents := cli.Cli.Torrents()
- for i := range torrents {
- t := torrents[i]
- go func(t *torrent.Torrent) {
- defer debug.LogPanic()
- t.AllowDataDownload()
- t.DownloadAll()
-
- tt := time.Now()
- prev := t.BytesCompleted()
- dwn:
- for {
- if t.Info().TotalLength()-t.BytesCompleted() == 0 {
- log.Info("Dowloaded", "snapshot", t.Name(), "t", time.Since(tt))
- break dwn
- } else {
- stats := t.Stats()
- log.Info("Downloading snapshot",
- "snapshot", t.Name(),
- "%", int(100*(float64(t.BytesCompleted())/float64(t.Info().TotalLength()))),
- "mb", t.BytesCompleted()/1024/1024,
- "diff(kb)", (t.BytesCompleted()-prev)/1024,
- "seeders", stats.ConnectedSeeders,
- "active", stats.ActivePeers,
- "total", stats.TotalPeers)
- prev = t.BytesCompleted()
- time.Sleep(time.Second * 10)
-
- }
-
- }
- }(t)
- }
- cli.Cli.WaitAll()
-
- for _, t := range cli.Cli.Torrents() {
- log.Info("Snapshot seeding", "name", t.Name(), "seeding", t.Seeding())
- }
-}
-
-func (cli *Client) GetSnapshots(tx kv.Tx, networkID uint64) (map[snapshotsync.SnapshotType]*snapshotsync.SnapshotsInfo, error) {
- mp := make(map[snapshotsync.SnapshotType]*snapshotsync.SnapshotsInfo)
- networkIDBytes := make([]byte, 8)
- binary.BigEndian.PutUint64(networkIDBytes, networkID)
- err := tx.ForPrefix(kv.SnapshotInfo, append(networkIDBytes, []byte(SnapshotInfoHashPrefix)...), func(k, v []byte) error {
- var hash metainfo.Hash
- if len(v) != metainfo.HashSize {
- return nil
- }
- copy(hash[:], v)
- t, ok := cli.Cli.Torrent(hash)
- if !ok {
- return nil
- }
-
- var gotInfo bool
- readiness := int32(0)
- select {
- case <-t.GotInfo():
- gotInfo = true
- readiness = int32(100 * (float64(t.BytesCompleted()) / float64(t.Info().TotalLength())))
- default:
- }
-
- _, tpStr := ParseInfoHashKey(k)
- tp, ok := snapshotsync.SnapshotType_value[tpStr]
- if !ok {
- return fmt.Errorf("incorrect type: %v", tpStr)
- }
-
- val := &snapshotsync.SnapshotsInfo{
- Type: snapshotsync.SnapshotType(tp),
- GotInfoByte: gotInfo,
- Readiness: readiness,
- SnapshotBlock: SnapshotBlock,
- Dbpath: filepath.Join(cli.snapshotsDir, t.Files()[0].Path()),
- }
- mp[snapshotsync.SnapshotType(tp)] = val
- return nil
- })
- if err != nil {
- return nil, err
- }
-
- return mp, nil
-}
-
-func (cli *Client) SeedSnapshot(name string, path string) (metainfo.Hash, error) {
- info, err := BuildInfoBytesForSnapshot(path, MdbxFilename)
- if err != nil {
- return [20]byte{}, err
- }
-
- infoBytes, err := bencode.Marshal(info)
- if err != nil {
- return [20]byte{}, err
- }
-
- t, err := cli.AddTorrentSpec(name, metainfo.HashBytes(infoBytes), infoBytes)
- if err != nil {
- return [20]byte{}, err
- }
- return t.InfoHash(), nil
-}
-func (cli *Client) StopSeeding(hash metainfo.Hash) error {
- t, ok := cli.Cli.Torrent(hash)
- if !ok {
- return nil
- }
- ch := t.Closed()
- t.Drop()
- <-ch
- return nil
-}
-
-func getTorrentSpec(db kv.Tx, snapshotName snapshotsync.SnapshotType, networkID uint64) (*torrentSpecFromDb, error) {
- snapshotNameS := snapshotName.String()
- var infoHashBytes, infobytes []byte
- var err error
- b := make([]byte, 8)
- binary.BigEndian.PutUint64(b, networkID)
- infoHashBytes, err = db.GetOne(kv.SnapshotInfo, MakeInfoHashKey(snapshotNameS, networkID))
- if err != nil {
- return nil, err
- }
- var infoHash metainfo.Hash
- if infoHashBytes != nil {
- copy(infoHash[:], infoHashBytes[:metainfo.HashSize])
- }
-
- infobytes, err = db.GetOne(kv.SnapshotInfo, MakeInfoBytesKey(snapshotNameS, networkID))
- if err != nil {
- return nil, err
- }
-
- return &torrentSpecFromDb{
- exists: infoHashBytes != nil,
- snapshotType: snapshotName,
- networkID: networkID,
- infoHash: infoHash,
- infoBytes: infobytes,
- }, nil
-}
-func saveTorrentSpec(db kv.Putter, spec *torrentSpecFromDb) error {
- snapshotNameS := spec.snapshotType.String()
- b := make([]byte, 8)
- binary.BigEndian.PutUint64(b, spec.networkID)
- err := db.Put(kv.SnapshotInfo, MakeInfoHashKey(snapshotNameS, spec.networkID), spec.infoHash.Bytes())
- if err != nil {
- return err
- }
- return db.Put(kv.SnapshotInfo, MakeInfoBytesKey(snapshotNameS, spec.networkID), spec.infoBytes)
-}
-
-func MakeInfoHashKey(snapshotName string, networkID uint64) []byte {
- b := make([]byte, 8)
- binary.BigEndian.PutUint64(b, networkID)
- return append(b, []byte(SnapshotInfoHashPrefix+snapshotName)...)
-}
-
-func MakeInfoBytesKey(snapshotName string, networkID uint64) []byte {
- b := make([]byte, 8)
- binary.BigEndian.PutUint64(b, networkID)
- return append(b, []byte(SnapshotInfoBytesPrefix+snapshotName)...)
-}
-
-// ParseInfoHashKey returns networkID and snapshot name
-func ParseInfoHashKey(k []byte) (uint64, string) {
- return binary.BigEndian.Uint64(k), string(bytes.TrimPrefix(k[8:], []byte(SnapshotInfoHashPrefix)))
-}
-
-func GetInfo() {
-
-}
-
-func SnapshotSeeding(chainDB kv.RwDB, cli *Client, name string, snapshotsDir string) error {
- var snapshotBlock uint64
- var hasSnapshotBlock bool
- if err := chainDB.View(context.Background(), func(tx kv.Tx) error {
- v, err := tx.GetOne(kv.BittorrentInfo, kv.CurrentHeadersSnapshotBlock)
- if err != nil && !errors.Is(err, ethdb.ErrKeyNotFound) {
- return err
- }
- hasSnapshotBlock = len(v) == 8
- if hasSnapshotBlock {
- snapshotBlock = binary.BigEndian.Uint64(v)
- } else {
- log.Warn("Snapshot block unknown", "snapshot", name, "v", common.Bytes2Hex(v))
- }
- return nil
- }); err != nil {
- return err
- }
-
- if hasSnapshotBlock {
- hash, err := cli.SeedSnapshot(name, SnapshotName(snapshotsDir, name, snapshotBlock))
- if err != nil {
- return err
- }
- log.Info("Start seeding", "snapshot", name, "hash", hash.String())
- }
- return nil
-}
diff --git a/turbo/snapshotsync/logger.go b/turbo/snapshotsync/logger.go
deleted file mode 100644
index 793d60efc9..0000000000
--- a/turbo/snapshotsync/logger.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package snapshotsync
-
-import (
- lg "github.com/anacrolix/log"
- "github.com/ledgerwatch/log/v3"
-)
-
-func init() {
- lg.Default = NewAdapterLogger()
-}
-func NewAdapterLogger() lg.Logger {
- return lg.Logger{
- LoggerImpl: lg.LoggerImpl(adapterLogger{}),
- }
-}
-
-type adapterLogger struct{}
-
-func (b adapterLogger) Log(msg lg.Msg) {
- lvl, ok := msg.GetLevel()
- if !ok {
- lvl = lg.Info
- }
-
- switch lvl {
- case lg.Debug:
- log.Info(msg.String())
- case lg.Info:
- log.Info(msg.String())
- case lg.Warning:
- log.Warn(msg.String())
- case lg.Error:
- log.Error(msg.String())
- case lg.Critical:
- log.Error(msg.String())
- default:
- log.Warn("unknown log type", "msg", msg.String())
- }
-}
diff --git a/turbo/snapshotsync/postprocessing.go b/turbo/snapshotsync/postprocessing.go
deleted file mode 100644
index 5f2fb6c26d..0000000000
--- a/turbo/snapshotsync/postprocessing.go
+++ /dev/null
@@ -1,336 +0,0 @@
-package snapshotsync
-
-import (
- "context"
- "encoding/binary"
- "errors"
- "fmt"
- "math/big"
- "os"
-
- "github.com/ledgerwatch/erigon-lib/etl"
- "github.com/ledgerwatch/erigon-lib/gointerfaces/snapshotsync"
- "github.com/ledgerwatch/erigon-lib/kv"
- "github.com/ledgerwatch/erigon/common"
- "github.com/ledgerwatch/erigon/common/dbutils"
- "github.com/ledgerwatch/erigon/core/rawdb"
- "github.com/ledgerwatch/erigon/core/types"
- "github.com/ledgerwatch/erigon/eth/stagedsync/stages"
- "github.com/ledgerwatch/erigon/ethdb"
- "github.com/ledgerwatch/erigon/rlp"
- "github.com/ledgerwatch/log/v3"
-)
-
-const (
- SnapshotBlock = 11_500_000
- HeaderHash11kk = "0x4d5e647b2d8d3a3c8c1561ebb88734bc5fc3c2941016f810cf218738c0ecd99e"
- Header11kk = "f90211a01cb6a590440a9ed02e8762ac35faa04ec30cdbcaff0b276fa1ab5e2339033a6aa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347944bb96091ee9d802ed039c4d1a5f6216f90f81b01a08b2258fc3693f6ed1102f3142839a174b27f215841d2f542b586682898981c6da07d63a1ceded7864e95f09fe65b6bd17fb3f02a3644b1340bb0ab8a7267251e62a04cb5cf79c8a58a4787ec1ed0af51bcce19e6ad701dd40a45086244b933104cf2b901002438522b194b05881a7d976aa8c45ff47193ba8adc6fe2cc85eb68c66503558fa0ba43cebbd2327cfa297a87228511374ed3a2f66f3999426dced224c464840303de108b8604dcafce84d678b589cbe8a74aa2c540668a9a9acfa1eb94c6569918d819063600c000f3c060d649129f8327cad2c7ba1f9495531224b34a1ad8ca0810ab2d2d43a18877484dc33d220c0531024f1dc7448f8a6c016340ae143efd87c5e681d40a34e6be5803ea696038d3ad090048cb267a2ae72e7290da6b385f9874c002302c85e96005aa08031e30ac2a8a9a021bdc2a7a39a1089a08586cefcb937700ff03e4acaa37448c00f4ad02116216437bc52846ebd205869231e574870bf465887ac96883a7d8c083bdfd7483bde3a8845f7befc090505059452d657468706f6f6c2d757331a07a1a8c57afdf3be769e0f6a54e92900374cc207c7cf01b9da6ccca80a8b4006c88d495a5d800490fad"
- Body11kk = "f99a34f99a30f8ac8214268541314cf0008301d4c0946b0359f95796327475ad4f12ae4e1047c3a67fa380b844a9059cbb000000000000000000000000fa1dfa4588664928951778b355b3de1f703ade4a00000000000000000000000000000000000000000000000000000000001e0b8725a0b3855a81e5f5ab38eae3b2c1bcd9ce72e143803ef4e47d9845c177aa156137f1a05993fea2c5b3e4995ca84aac5a11dd87ea1dd9a70f6cfc52f88a4b932edec56df8ac8214278541314cf0008301d4c0946b0359f95796327475ad4f12ae4e1047c3a67fa380b844a9059cbb000000000000000000000000def1b9aaedeeb0dbeba57773fad640aa7f30e735000000000000000000000000000000000000000000000000000000000057f29b26a0dca35e61c625a15c23579b346e032598a2ae33086ef1146330ae59335b28b90aa0162e3d461d1ecd5a57ef92ec582aab7daa6d7a402bb162963f37e1af9b0a8eb7f8ad830109f9853a35294400830138809412d79c345cac7b050a5ff0797b5a607e254c73f580b844a9059cbb000000000000000000000000e7e7a94e68be242a0db0c525dc3acefb755ae271000000000000000000000000000000000000000000000000000000000041696725a06f44e89a311d35682d93f81599bbb23c7a3477682c0ee66436688eca14ead9a5a00465af62a83af5135904c7953fcbb4821cc977a912b9e9055afc978224014a30f86d5785131794b40083021fdb9456710f7e591fa0559003831e041dd67789d2af4a881bc16d674ec800008025a08286e05045a922f5cb264de212d0fcd9bae59dfdc98a9a9d30b8cfc90baaa273a05945cb96da53d2ac56063d5a7273892e946f74ab371771ba75a725c223626820f88b825259851298879d29830249f0940000000071e801062eb0544403f66176bba42dc080a400000019000000000000000000000000000000000000000000000000000000000000079d25a0d704c1f7f9480b5f8f7e74c9dc5cf77391f4eb8abd901e5d5331af8272f4820ba026c7b701f8a24c6ce84d86c02fcdfd116b9a590bccb4848e012b6baefded1e99f8678211cc851229298c0083030d4094e5fd3dc48656dfed32601468426fe57595d8938080801ba0d1e94509c4ac498235452be50481fc49fb4037f37d2686a13c52280fcec15815a064fa69a8fb3bc49599be90dcd293f72a2575da8e991285c2a4923aa6042c1c45f86d82f06285120b5c270082627094ad978dc888e1a2590244f8bf9e716eb18b3b68d787354a6ba7a180008026a0ed667359fcecb1d62418dfd5535fa5ecc0abf7dcf314b1a59222f8e87a1af8a0a00755cf05e3dd4e9f95d9f9240cf5e66d8320fe3e64f7ed2eccf879c2db546ca7f8708302e5a28510c388d000830186a094dfda58c58d9d2e862d386706efdf6153e147ae8b8821db0ba827b770008026a07c7bbca5548e94b3da64473eb35b287d802a00433b5e67815da4765803b076a0a00c4c1bd0ba655f8c82cc4880bd4a1821de03bba379644685e6d8ceb3252d22def8708302e5a38510c388d000830186a0941afd9659259df405dd3f507ef77887aa0cb751b98843966d18b91b80008025a0c345c9fc5217f79b2e9375916020162f10f0505298ce5c074b15ad14495c8e81a00292440b70d3a6455902013ba251caa8578a22b76d59b14a668d839e2947f608f8ab818f851087ee06008301d4c0946bc1f3a1ae56231dbb64d3e82e070857eae8604580b844a9059cbb00000000000000000000000024434ebf296c2f9cd59b14412aae5c4ca1d5aad2000000000000000000000000000000000000000000000a9e566513a4910d400025a0db1575b0bb6f9ff489eceda148be18c2914680dd61d93e94b272d25919536feba0539db0b71bfd18077a63b7cec6624459bfb2cc1b5ec157bd540d7e323b833bf7f8aa1d851087ee06008301d4c0946bc1f3a1ae56231dbb64d3e82e070857eae8604580b844a9059cbb00000000000000000000000024434ebf296c2f9cd59b14412aae5c4ca1d5aad2000000000000000000000000000000000000000000002a4c2b736cdf768d400026a06abb41e737220b9f918e351d4e1106c7f4ab1c76e9f71dcf1313e34056c855e4a02e5737341566322cf7049561d9c62bebbb2d9fe681a9672a98b76d27b3525cc4f86c44851087ee060082520894efb2e870b14d7e555a31b392541acf002dae6ae98808a9fd000b5d10008026a0a3a8ce710f12d38d704b748aac579095fa76a2d4335fddc507c2b320ddadb341a048b3889aeb79855224b84c89aca69ef15716a965b1a3d4a416a2b932e53617d3f8aa3d851087ee06008301d4c0946bc1f3a1ae56231dbb64d3e82e070857eae8604580b844a9059cbb00000000000000000000000024434ebf296c2f9cd59b14412aae5c4ca1d5aad200000000000000000000000000000000000000000000386992ac084d52b9a40026a0a30a1900def92ba2d630ce8851016f200a6f0ab7cbb3726e856b80c8c7c2cdb9a046b2ebc2062888b195266b27c24b832d16c38a681097661e7c85dc918c659f74f8aa61851087ee06008301d4c0946bc1f3a1ae56231dbb64d3e82e070857eae8604580b844a9059cbb00000000000000000000000024434ebf296c2f9cd59b14412aae5c4ca1d5aad200000000000000000000000000000000000000000000d3998a945d035620e20025a00aef9d9d942121f0ec78f855f5b41a8f5012e9ad245adf3b19bd25a5e60d40cca07119f902bc2bf5ef3a30788ee545e4dbedbc52054c970409fc6a742281802831f8aa37851087ee06008301d4c0946bc1f3a1ae56231dbb64d3e82e070857eae8604580b844a9059cbb00000000000000000000000024434ebf296c2f9cd59b14412aae5c4ca1d5aad200000000000000000000000000000000000000000000135d111acd67564fac0026a0f1d8a696da6547e491e1fb89c642493f3ae1b80c1f2b7f48857940117b10e83da040b4b8557111d4fbfa45e8711e1856dacf6d5aacdfd83975d91f8507f6d2491af88a82238285104fe73d7882ea60940000000071e801062eb0544403f66176bba42dc080a400000039000000000000000000000000000000000000000000000000000000000000fdeb25a08fb55bfd8459411b7bb69bc388a2f1fb4ccf2e28a52739ba6e7246b8dfdd282ca06924a9f7387c9405b0ebe379a69a3fab0e5606254213b67646470d0d9ae36fdbf8ac82b747851010b872008302bf209410086399dd8c1e3de736724af52587a2044c9fa280b844a9059cbb000000000000000000000000444705e7ccdc6bf014f3e956e5c4eda78b59f79b0000000000000000000000000000000000000000000013ad784a9942cb78980026a0b2074cb4a3d2954203fb6dfdd75dbe2b4e581452c8111691b2a8903d64c741baa014edfd2a96f05c667482fb098da4df0d323ceaa03c1120fa8c3b11b1e6752cd9f86c03850ee6b28000825208946cbdaad85cf5f084c25d349972178e12e0f3811b8801cd3fc4be2751008026a08cce1042c48eac631c16b870576577bc6837ea9fc8aa8be4fe6085115fc1072fa01e5e0c39bb23d8a844d3fbb15a2b803db00d19f16db962c51aca9efac4735be2f8aa11850ee6b280008301117094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000f2b293e9c1cbef2316ba8a4eccfecc8ba7af123c0000000000000000000000000000000000000000000000000000000013fae61025a0fbdaa8990cf1562bc687fba0919bee234fc4943ba536959b5d53199b08e645eda03610f59eb4a9f1b264a41da1f4b75e6ddcfdddc25e51d319e8aeae9f1866e2c6f86f820109850ee6b28000830111709429bd07c50a2a2a843b309264361c9a91950ec650880a47e9a2ff200c008025a098d7a79510ec8088e40332b6c17eb825c85b0f7f5169b38f32b4176804ea798ba02103aa7c8c18724bd2266f56f6eb2982e59ce66fc7e815b96258f1733000680df8aa1d850ee6b280008301117094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000f2b293e9c1cbef2316ba8a4eccfecc8ba7af123c000000000000000000000000000000000000000000000000000000001141445026a0304b4214474adb7c012af51f66a557bd5e4dd5ca6384181b75718b695cc0ca58a028e921c3e0bde2b373d3327575493f56f2c38e938db5a3af79d9cf73ee5bf41df8aa07850ee6b280008301117094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000f2b293e9c1cbef2316ba8a4eccfecc8ba7af123c0000000000000000000000000000000000000000000000000000000015bb0ca026a05817ff739574af40bb6a2a1e3521c81d9feeac9e1581ea1862e1d1310e992f25a04062cbda0d6c9cfd1ec650ab2f566ad443a32ea185460ba7b848e0c707222bd5f8aa04850ee6b280008301117094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000f2b293e9c1cbef2316ba8a4eccfecc8ba7af123c000000000000000000000000000000000000000000000000000000002b181ac025a01db94ba187623de31c790f335151b554f6edc8c3d3ae31c4bd1ff87f76b494c1a02f2d366e315e0243f2c00127f5bb00e3306e6c56fd93d415588381e2636912d3f8ab820ee7850ee6b2800082a0f994dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb00000000000000000000000030c6ce4559c9e40f79a21be12d8ac6a7cab5fbf400000000000000000000000000000000000000000000000000000000115987401ca0760fc54c11cf12f3a7cccb5340b4a625a926dbde65d482400de8eb9c2b21eb26a0725adba5a62c12842b6167cff941048910ac3bb0aa3502924d713b5648819d15f86c09850ed81dd19882520894f47b8ff418b81d929b1e2ab079ca96f9bae00ada888ef0f36da28600008025a0f3c3e987f3474dd3b8d298f9b127ac5dcd0d7728ce040da6cf148170b61b9008a06cd26c1e56d27009833ecebee4dcc7b69791476a64cddb9f660b39e9a79bf19bf9015103850e6f7cec008302a5e6947a250d5630b4cf539739df2c5dacb4c659f2488d87943afad18a21f4b8e4fb3bdb4100000000000000000000000000000000000000000000000038e62046fb1a00000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000c279a165c11340ff6262bd74ce7b328ca7d738d5000000000000000000000000000000000000000000000000000000005f7bf4210000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000bd301be09eb78df47019aa833d29edc5d815d83825a02f07ebc6959fac30dcd3d89d9a102f919086a2641afb6be871b021a956fc9cb7a0195f7d7e0b59d32190c2fd10c7a448d9a2ea40b7c72762a99c4613b690089073f901ac818c850e6f7cec0083048968947a250d5630b4cf539739df2c5dacb4c659f2488d80b901448803dbee0000000000000000000000000000000000000000000000003782dace9d90000000000000000000000000000000000000000000000000000001d690b82191f53800000000000000000000000000000000000000000000000000000000000000a00000000000000000000000006795e7f4219a48e083157db6b52cf70002eced5f000000000000000000000000000000000000000000000000000000005f7bf45c00000000000000000000000000000000000000000000000000000000000000040000000000000000000000005befbb272290dd5b8521d4a938f6c4757742c430000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000001712aad2c773ee04bdc9114b32163c058321cd8526a064c6e05031df9eb138b78b0e4d6578d17d2b17fda9f05d8ac921ed707390fa5ca06a826a830e4e740cca349d89f1afce04cd48cdf527780401ce28d235497734c0f8ab820c6b850e6f7cec0082715e94a1d0e215a23d7030842fc67ce582a6afa3ccab8380b844095ea7b30000000000000000000000009748d39b0b0949c0d5214094045fb7e2222a26ca000000000000000000000000000000000000000000000000000000000000000025a038e6bd86c77f10475337e7cd32c88043465d9cdfde5d1fdce4e778ae55383b3ea0767b31439035cc8499be4c407d7c1db7f38755b7cc8cbbc09065b689de0716bdf9016b34850e6f7cec00830290bc947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe500000000000000000000000000000000000000000000000007492cb7eb148000000000000000000000000000000000000000000000000000075567ee0995c7e400000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000ddaaef5d33f009bef641ee9a8054d0fa7c1e8d0e000000000000000000000000000000000000000000000000000000005f7bf45c000000000000000000000000000000000000000000000000000000000000000200000000000000000000000028cb7e841ee97947a86b06fa4090c8451f64c0be000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc226a02daaf93f114bbe4209f1c082ead858afdc91ed29f2d5fa7866457e14a114d99ca022e100caeb5c78e445a014875c10b9bdbeb2828e1d7f2b35b5f0558f3d230981f9016d820274850e6f7cec008302878c947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe50000000000000000000000000000000000000000000006d3c17ae6ef8aafbcc90000000000000000000000000000000000000000000000000e718f3360c35d6800000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000e53699169e2ce5c49dbb85d86de834ad4a4a9d6a000000000000000000000000000000000000000000000000000000005f7bf45c000000000000000000000000000000000000000000000000000000000000000200000000000000000000000087c00817abe35ed4c093e59043fae488238d2f74000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc225a0747216c39ceee5fcfcf7981317eba8108952819d6d0cf1fc4301e2882d3d54d7a068555b7b379ae6a66f31f64616860db5be895ee5fb5e6e1045d6217769a80d00f9026e830efef8850e6f7cec0083017c4094baf075545c3a56ecbaf219e4a1b69bc2b94b0b7580b9020464887334000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000a7d8bd0000000000000000000000000000000000000000000000000000000000000180000000000000000000000000000000000000000000000000000000000000000331005a675e1d000118e351dc07f3000000000000000000000000000000000000526b143b1737372ff3262608cdf5000000000000000000000000000000000000e37003f2f90000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003d600e2aeaa0800fc011eb527f90f000000000000000000000000000000000000bfa0eaf4ebd0c9d80c09da01490d00000000000000000000000000000000000017a1fd0b0900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000021ba0e8b0c6d27b12a2cd8506f694a874547161c2ef5fb3134fec6415010124624e32a018c3b32b8c7fbd563c33025b712f4b311cfa10ae2b7d4e6a8df76721b0087b97f8cc822bf2850e6f7cec00830249f0940000000071e801062eb0544403f66176bba42dc080b86400000022000086fef14c27c78deaeb4349fd959caa11fc5b5d75000000000000000000000000000000000025948db5a78975a28a00000000000000000000000000000001000000000000000000000000fca59cd816ab1ead66534d82bc21e7515ce441cf25a049cbfb1f56c0a1555daf81f30121a791d149d1f4b5c9bfb278a070255d5bad65a0669c9a99b922a620ef5295e6f0b88bdc2619eb3ea9043d9ba784967e7c796721f9015482025b850e6f7cec008302aa06947a250d5630b4cf539739df2c5dacb4c659f2488d88011c640541017538b8e4fb3bdb41000000000000000000000000000000000000000000000001158e460913d0000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000003c8e49a61e91e6fdf7a56bfc4d6e347dd863ac27000000000000000000000000000000000000000000000000000000005f7bf45b0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000002a7f709ee001069771ceb6d42e85035f7d18e73626a02068b8d08937830226e99f3751ac9d21c381177b46f3cd7db65402655d4d8a0fa01a67a20e628a491743bee70f0dd3a573fca70c754d4b5b117052df173160b32df9015278850e6f7cec008302ad2f947a250d5630b4cf539739df2c5dacb4c659f2488d8801859bf17ded6942b8e4fb3bdb410000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000934831793baa281838c1e09e665a4f1406bc7560000000000000000000000000000000000000000000000000000000005f7bf45c0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000de201daec04ba73166d9917fdf08e1728e270f0625a0b9eee4c06f560fa62718be16a004561f8822346db521eff8b2454e8768bc0f07a07e4161e5898bab99feb09b44cf6737fb7c5f1e3f48138d599a22fc00ed867c0ff8ac820758850e6f7cec00830102cc94990f341946a3fdb507ae7e52d17851b87168017c80b844095ea7b30000000000000000000000002c6a9c15e7f13de8ae55302bd29608c9eed78f080000000000000000000000000000000000000000000000008ac7230489e8000026a06feeea2d9505a20d7b2d10e1936d5bf9117d4b88e632c786322c3217dd916ae5a02e618e1517a44cb023837b64f1e902f0ba4a9a6a7d2d9806175a83f5d9ec826cf8ab81a6850e6f7cec00830381f9944d23c0537d7ee782cfaf9bbecdd60840e3d0ffe880b844441a3e7000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000045be7ad5bf65491d325a0a2b643fdec2ee570d216e2fce4f6030e7f7d162157808fc0e07c22da253e96b1a0752e21f128a8a64f7b79beadb3c65714505e9df52cf99068c7acf83a21e2d928f86b02850e6f7cec00825208947264c9f085858c607733daac1eef2625237fb333870b4fcc9ecd62008026a053b3489fc1005069296c19ee693c6e0ad78367ba818b930558b006beed0fa479a003adaa5da3158347c5141ca120dc4f2623e0fb0a90fab1129093fa665893e11ff8aa04850e6f7cec0083013d8c943fef5a74bcd2b8d2d44885b144df7f9d149b4b4580b844a9059cbb00000000000000000000000044fa1f6a271c6eb84227304e92813142758e133300000000000000000000000000000000000000000000001226788055013fc00026a0e2e6d5433b941dd559ff46677069c3689ac3dd5e28d12ee0287cb6ff3cb48bc5a010ae409467d096bc0c07fb1c4b2a3d9afca1e69d4842f329ccf987966aaceccef86c18850e6f7cec0082520894dfc105318661fc8241eb4d50275ad49bfd99903d88014a6701dc1c80008025a0c6301d39576a4f65741d63644701dcf63fb0dd8b8faaf5dbec353f3f64197d7ba0011557cc13411b8664590e2bc18517cbec0683bf4da982cad2f85cf3eed68fa4f8ab8201b4850e6f7cec0082914d9432ce7e48debdccbfe0cd037cc89526e4382cb81b80b844095ea7b300000000000000000000000073282a63f0e3d7e9604575420f777361eca3c86a0000000000000000000000000000000000000000000000001e3c5e6d667f9c2e25a09d6afbad3bdf0216cba142d92e752fd8c22956ba61fa642d2665e507e7a1192fa047403cec0c3d6bf90fa1c933907d80ec4668a605f2a18dfbc87255c8dda92ea8f8d240850e6f7cec00830454ff9473282a63f0e3d7e9604575420f777361eca3c86a880de0b6b3a7640000b864aed351470000000000000000000000000000000000000000000000000000000000000cfc0000000000000000000000000000000000000000000000000de0b6b3a7640000000000000000000000000000000000000000000000000000000000000000000025a042d718cb1b80c7615c6ed21a5cf00b8b174f70eb1004005e0dfffc4fb2a4a090a07ceb1139767232185f78e822b78649d0d4123861205f8236b8e6268750048180f8aa80850e6f7cec008301117094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb00000000000000000000000040c266a51ced55caf390cb6e00b5b38c5736a79200000000000000000000000000000000000000000000000000000000b06d6f9026a0493eb28fb2ad6ed89ddcf5f2cc30f6432d503c69dd91d5ff0c990c9f4f188886a045bc2d33815b1765150e63ef106e5b4c5ad877b25c1419a122304b089572f15ff86d820229850e6f7cec0082520894bd7de91133e32ced22820aeb44a864b2ec25c7188711c37937e080008025a07d0a9d5cdb3f59f1bfb3df400acd0aa3112ab5948018e49024571d3254f7aee9a02f0a90d0eb973cf22f86fc9c4d15a1e6862f2e9cf2817d63a12380866938f93cf9013201850e6f7cec008302fbca947a250d5630b4cf539739df2c5dacb4c659f2488d888684fd111d7566bbb8c4f305d71900000000000000000000000080fb784b7ed66730e8b1dbd9820afd29931aab0300000000000000000000000000000000000000000000016cabe0ead394bc000000000000000000000000000000000000000000000000016ad91964884878800000000000000000000000000000000000000000000000000085d8cdb8b5a19d41000000000000000000000000e62d82f95e7fb1848b1a01a5972d4f7a0507a55a000000000000000000000000000000000000000000000000000000005f7bf43925a0305fb0ab4a1a1b73a4dbfb4ad525ca66b9fa64e737361343cf0881dfdf489024a0749ceb292f716bbf44ad3e8c5100da848a456086e45fb273d77accf232b8fd74f8a977850e6f7cec0082ad46946b175474e89094c44da98b954eedeac495271d0f80b844095ea7b30000000000000000000000007a250d5630b4cf539739df2c5dacb4c659f2488dffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff25a096609f8ab69bbb36e2862e2d6cef3fd7cf38535243737431aac84dcb7ee0507ba00dc3a059327e9248c684828dbce2b482843dd2b066058ba81cb3bb4a4fd5af16f8a93e850e6f7cec0082beb29487047986e8e4961c11d2edcd94285e3a1331d97b80b844095ea7b30000000000000000000000007a250d5630b4cf539739df2c5dacb4c659f2488dffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff25a0d2e07ce83d0c82fa351da9765e8a9ba3cb6c48694e40f39cfd5adaf10a114aa9a03281be0d89b856c3b5615af37ea71e8a8860b9b626d71d7bccf082b99fc2861bf9016c819c850e6f7cec008301f961947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010438ed173900000000000000000000000000000000000000000000001080bca471a87251120000000000000000000000000000000000000000000000000fd6f74af9fc2ee000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000f13f7bf69a5e57ea3367222c65dd3380096d3fbf000000000000000000000000000000000000000000000000000000005f7bf45800000000000000000000000000000000000000000000000000000000000000020000000000000000000000002a7f709ee001069771ceb6d42e85035f7d18e736000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc225a0e901cfdeae3ac58d894863e2c810c32c3dd9d3c93c2a5c205a1dd1f860d51d8aa07454fb37616bb4897a4ff1aae0690c678b4039fc88cbb24b942767444027c3cff86c09850e6f7cec00825208947fd97df8b1bd5b29e97ccf09197fa86196b6a79e880de0b6b3a76400008025a09f2c471e5f9372d9d1d67ff2ad43a2a290cef47624e71c0d41e28a6e4b7ce7aaa05601a15d106d6d9ebbfcaa8d755c88d7ab4144f689d394b712a30ddeb4794d78f8cc82525a850e6f7cebff830249f0940000000071e801062eb0544403f66176bba42dc080b86400000022000086fef14c27c78deaeb4349fd959caa11fc5b5d75000000000000000000000000000000000025948db5a78975a28a00000000000000000000000000000001000000000000000000000000fca59cd816ab1ead66534d82bc21e7515ce441cf26a0dae920ba1886fd25c3749ebe20b67e39de3fce33227c1205a9fbd389652f8674a069100b5700683f1f58729cf795837c93a1ffaf8095368a82c477e287b8530b8cf87083032eb4850df847580083015f90940ea6bf3bf79c21703f34d021c2b7dcf76c44fa888804820fbd9230bc008026a0128468f13235769729c71c3929fb11127d400455ff314782c1508c3c6c273163a025cdb5a37aa93a72b1848940336010df7e27d89396a3e6d56dfd44dd373d4538f8ac824268850dd4841200830119ed94dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000008d8478f898907e6251d98601f32a0c1a94d315580000000000000000000000000000000000000000000000000000000001a35a541ba092cf05068392bebbe1e7fb04c7afc1e8781b5dfa4aa8d09bce894e5d9353406ba00dd9f708bd64cebc57b26596c553d05620084aebab5c219bbf5db14ee071a0ddf901323f850dbcac94198302ea14947a250d5630b4cf539739df2c5dacb4c659f2488d882f24f966aa302b2eb8c4f305d719000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec70000000000000000000000000000000000000000000000000000000047820d4100000000000000000000000000000000000000000000000000000000472685810000000000000000000000000000000000000000000000002ee8a11d3104686700000000000000000000000015fdaab0c0e130c7f0e72d7efab5e96f103f143c000000000000000000000000000000000000000000000000000000005f7bf02025a01199f616db8a6af136eff302e4dea17713ab7a1f82f4ca3967dcdd7124626098a01f83463019dadf0b2f4d6d60808be65f59a02169c0a267adf8adcb2eb644bed6f86f83010d82850dbcac8e00830186a0945dee759699e83cbc165fa0bf12f80a9ad11c6a0087470de4df820000801ba072ca22aa17cf7a155c6be34f2591ff378832d8db9331009272d5edbfb3b38453a010fd777ad7ccfee732efc0b9309f917495eabf8c59d6425c5719af0cdd42654cf9018c8192850d92f367e98302d223947a250d5630b4cf539739df2c5dacb4c659f2488d80b901248803dbee0000000000000000000000000000000000000000000000d8d726b7177a800000000000000000000000000000000000000000000000000000000000004186672900000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000834d399905c86b993354777ab858e08f6d5b996c000000000000000000000000000000000000000000000000000000005f7bf45c0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000000ff6ffcfda92c53f615a4a75d982f399c989366b26a0b0f7ae019099233e1a7a4fee1bcc0dc0e21cbe61e49f110cf582169f2808dfeda0663acd0f4b34b6d4471748bc1f1bd3bc39d615d725592816d5a57866b6170a07f8ad8314be13850d4576fa00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb00000000000000000000000009a6cc6e8a82f6f4e10972300d0b1094072007dc000000000000000000000000000000000000000000000000000000002625a00026a02f21d1dfe66cacbd2d5558809ec77b7c2fa772bf02012f823bfe860f112fa672a056a28ff19d10debc6eba7b0ec39a5c61e88deb0e0258380421bb48b1f270c3baf86d81a0850d4576fa0082520894104ff25f04d2ebcc602495fc5b4f4d422eb7975088016345785d8a00008025a08c8ec39beb6ff5136625b1d07d364bbcdb8f408ffbc65f01951a7b6491601508a0425709fcd2cdd064dd1196044283e0b8667635dd72bd606015301f95c3baf0c0f8ad83147438850d4576fa00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000eb845a45e0dc707164b55f04e6a15e967705ff2500000000000000000000000000000000000000000000000000000000743b3c6526a01f72ff693b4ccf22e1e091643d9b1434e90297e8164560c6047844e0e18f1c0ca02886161bdc412246a540b082900dbddfb7448f1227405b9b52f9b67a858f1e01f8ad83147439850d4576fa00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000003e8d29f9ef0b3b4bdc997c7e644d1e44eac5a185000000000000000000000000000000000000000000000000000000003d648d8026a00748de85240f9a86b2584f9c905f7239a9cb6e136a7e495f19f3c009a62912d9a04cc16a5a7497b42d45368e01d0d6348edad9f8dfc62b8f80c6d23a30fa96115df870830115f1850d4576fa0083015f909433f543df44521a4777f9b65756922b4a51b189598801bea209329540008025a09500efe32941004db60b82da2478379add6ed9cd9592e48ebd63dac7f68dd42ba05977faec04d591e3bb8c6be36e7cffad810e8fdc614f9d0c0a877ee4fec8822cf870830115f2850d4576fa0083015f9094b44e83a8d9aaddc4864f5105822dd5ff234741f7880b4d876ab52f80008025a037d89eb746412bb23c7bcb9090736bc1ec8494c4094227676065f48089d1f4f1a005d6a3a8def1ae1c671ab50f172cf0ea4c744a8a40a8d926a693578e16044949f8ad83128a50850d4576fa00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000000556c4e74997b8c2aa9fab24bcfa7671714124ac000000000000000000000000000000000000000000000000000000001e0a6e0026a039497e16a9268eae2309470719234e0d0585d8c98c61e4ba97a2a10f5a586e1ca070f0a295cda26024665e80966761bcf4c9d489ca8a8211c83763edbd6abfd615f8ac821801850c92a69c00830141da94dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb00000000000000000000000097870b2951805078c826dec929a0a4e39c1e3ec40000000000000000000000000000000000000000000000000000000005b8d80025a0e40cbd82ebe816bd45396bc46035bbef4062358f71a6f808b6a13c775644b0a1a028b30761ba0479db9a2dcc965b9d6868eff9975f24dd9129aa6bc4f5b480e00df86e820e41850c92a69c00830186a0949469805c2450c797e94db7b684437883af5b26df87d529ae9e8600008025a0766d5f402b9cb8b770574d56a0a18e31a2858d953c02634035540e67f816e67ea0660cf0f8727caa2fff314e153021f34f08940f71af0300051eb9be94334cdeb6f8ac82013c850bdfd63e008301075094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000ea68673a2aa2d7845ebda48f68fece2233a3b90e00000000000000000000000000000000000000000000000000000000042c1d8026a03ddea161a13234d724e4d468935d97c898c1ac1d25498e32ccb37173ec12a9baa022f1e426f03dd02b67cea7068830628ea16df0a853e6a33a772d1b1a5b5215adf8b9829ae2850bdfd63e008304f58894000000000000006f6502b7f2bbac8c30a3f67e9a80b8510000d9c58600140051000000000a68000000000000c12d099be31567add4e4e4d0d45691c3f58f566337134075f5b5a0a94ac891c7b5ec5db5cfcf392cc04744ab87a4c37afd91680ef280b96ee21a026e26a091f853150fbed3d6ef89aa0eefa97ce96374ce267ce44e48b8e355aaae52450ea06d2ac99c554c8d922523541dbe7e961c87cd58621f90bc6d95b72f3bf103044af9029182122a850bdfd63e00830bc315947ee8ab2a8d890c000acc87bf6e22e2ad383e23ce80b90228865a6b4f00000000000000000000000063faee0a2cbcedc0102575c6ff17da38b1565c2200000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001847ecc63970000000000000000000000000000000000000000000000001c9eb028e55c1af0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000c12d099be31567add4e4e4d0d45691c3f58f5663000000000000000000000000164a1229f4826c9dd70ee3d9f4f3d7b68a172153000000000000000000000000c04744ab87a4c37afd91680ef280b96ee21a026e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000078d65bce7dd600000000000000000000000000c12d099be31567add4e4e4d0d45691c3f58f5663000000000000000000000000c04744ab87a4c37afd91680ef280b96ee21a026e000000000000000000000000000000000000000000003b3558c3d643af5e8324000000000000000000000000000000000000000000000000000000000000000126a0aa58ab93b2fc330ed5fd4d5e9e5165877fc517bdc944e03f9b107de1d558fd9aa010272681e6e57c3aa74f719f526873c81e01d01222157448e913f62ab9dcd06df8f6827823850bdfd63e00830a4cb894000000000000006f6502b7f2bbac8c30a3f67e9a80b88e0000bb6bee0051008e0000000014d1000000000000c12d099be31567add4e4e4d0d45691c3f58f566300000000000000000000000000000000000000000000000000000000000000000000000000000000003b0116363e435d9e4ef24eca6282a21b7cc662df164a1229f4826c9dd70ee3d9f4f3d7b68a172153c04744ab87a4c37afd91680ef280b96ee21a026e26a0c4e7826ef9776170a23747941d8e2031e1d632cf4bd0860d00c5ecf14f0fac4aa03a4ab912aa4b0bafeb1251e62a686c05d01856725c2bc9fcb9d514b16e8c3efcf86c01850bdfd63e00825208947a3b3a48da48dfd7171064c43efca3eb88dd83bd880a804988f69b5c008025a07ab7ce5668879c2a8b2271e9899006d1f955755bcca0e84c7aacc87f7384d8f5a001c09a314305066a7a99b0757c3fd34306fd740c3fe676ce9b779989027002def9021f83017e4a850bdfd63e008304f58894e33c8e3a0d14a81f0dd7e174830089e82f65fc8580b901b5000b060d0713abadabced27fd1eaeaeaaf0fd72adb0120000000000000000000007fc95945eaa14e7a2954052a4c9bfbaa79d170ae000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000c12d099be31567add4e4e4d0d45691c3f58f56630000000000000000000000000000000000000000000003643f0f256d9376e9c40000000000000000000000000000000000000000000003520346458fd696cc0e00000000000000000000000000000000000000000000000018a893fdda355e00012000000000000000000000c04744ab87a4c37afd91680ef280b96ee21a026e000000000000000000000000c12d099be31567add4e4e4d0d45691c3f58f5663000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000e33c8e3a0d14a81f0dd7e174830089e82f65fc850000000000000000000000000000000000000000000000000000f539c0ca6be80000000000000000000000000000000000000000000000000000e92936feea3900000000000000000000000000000000000000000000017cd98b0be009d000001ca05422af299139186ec365f14c427205bc76e7b41d7d6eaf14183a282d41fe90b2a00cc85b86c76ce5d6f9cdcad085624d82476baf34d6c38afb99c8db0ee02a906ef8d48260ee850bdfd63e0083061a8094000000000000006f6502b7f2bbac8c30a3f67e9a80b86c00000bb276002f006c000000001bc1000000000000c12d099be31567add4e4e4d0d45691c3f58f5663017fc95945eaa14e7a2954052a4c9bfbaa79d170ae00071afd498d0000008ac7230489e80000022b1c8c1227a00000c04744ab87a4c37afd91680ef280b96ee21a026e25a00e84a9a46e23d619837c594d3192cea4d9fa388066a4d5b100f79a17a9b45928a06f82d747d441daa4de0c573baf56c442c53c615021b5e9e23d25a3f985d5cc10f9015233850bdfd63e0083026b7d947a250d5630b4cf539739df2c5dacb4c659f2488d88243ea5ce72ec5d47b8e4fb3bdb4100000000000000000000000000000000000000000000021e19e0c9bab24000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000c281e970225cf3bce07d9024b4423b1fd3df0e29000000000000000000000000000000000000000000000000000000005f7bf4390000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000c12d099be31567add4e4e4d0d45691c3f58f566325a053005935343bd6b8355c3559ffcd855b30936a0386fbaa36b04aecb6b6fd51c8a0670215bcb30cb573ab8b4c424323039b00fccd611437dc99708c4a93769590f0f8f8821662850bdfd63e00831050879478a55b9b3bbeffb36a43d9905f654d2769dc55e880b890c89e4361bdb09816eaff92a339048a9119bfdb7f0b848b42628642fe0abf20dcc2c82f28fe4f3e406928c22938f493207e57277a8b483b3b122dcdc4000000005016c281e970225cf3bce07d9024b4423b1fd3df0e29470002b67d020000000c427fc95945eaa14e7a2954052a4c9bfbaa79d170ae000000c12d099be31567add4e4e4d0d45691c3f58f5663550000011ba074a4ce83493116995c8eb5746c1068cdd6422514ad83ef16f9e113c69bdb2b93a03002e2ca1e6e7ad945af6eb93d7a4b1d66c8e24efcee9923a565a4619f5640e6f8ab821a47850bcdf49b0082c7bc94a0b86991c6218b36c1d19d4a2e9eb0ce3606eb4880b844a9059cbb000000000000000000000000948045ebbc5fe5ded30f2ace84f99ce1b9fc30a900000000000000000000000000000000000000000000000000000000001bd4e325a083ebfa18687b06c006e7db126cc80855081a2b9954aa2ef84614c467276a1e08a01be36d428ed259ef55ef0cf93515fba796518b62c2a101bb3dcda2fbef6e20a6f901ed820182850ba43b740183061a8094a57bd00134b2850b2a1c55860c9e9ea100fdd6cf80b901841cff79cd000000000000000000000000728258295b1ccbc61d72c4e86c5b44345ac790ce000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001042fdc7315000000000000000000000000c2adda861f89bbb333c90c492cb837741916a2250000000000000000000000009f8f72aa9304c8b593d555f12ef6589cc3a579a200000000000000000000000056178a0d5f301baf6cf3e1cd53d9863437345bf90000000000000000000000000000000000000000000000228a17fe49a0620000000000000000000000000000000000000000000000000f69c471cb506f64bd8000000000000000000000000000000000000000000000000015c808740b32a90f000000000000000000000000000000000000000000000000000000005f7bf0340000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000001ca05df7b3acaeb55dec2fb125f4261814983a0f4fb24696bb3c02ecd4bdb8664ab2a0634cd17f4bd2eebd956dcd1c60c882483dd145f36738802d1540d6ca0bd18e6ef9015482016c850ba43b740083026981947a250d5630b4cf539739df2c5dacb4c659f2488d8801028b9db6d1e430b8e4fb3bdb410000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000277d46fcbe71144c6489d182a8aa00da61b57b07000000000000000000000000000000000000000000000000000000005f7c43e90000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000014c4d8e57751e083bcd1edcca5b435720166f5e925a00985497b640cb6f70739b068292050a427b6e0c908a9d29422582a9214a6003ea0415cb1f10d3b6f2a201ef340dbc8ca8c67171e7422cd315bb9fb574d36111317f8a901850ba43b740082ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000eba638932d82ebc1161eb9a992572ecafc197f7e00000000000000000000000000000000000000000000000000000001b9df78601ca0f0f487cfbb3896e83daab39082d361b5df9c6b7af1b24c227268f23c886c6ff7a00db21e461dea4897cb626c4aa60df3979aba5ab4e89971b962ecc59729403e1cf9015258850b2d05e00083037033947a250d5630b4cf539739df2c5dacb4c659f2488d88027f7d0bdb920000b8e47ff36ab5000000000000000000000000000000000000000000000229a587e3e68685925200000000000000000000000000000000000000000000000000000000000000800000000000000000000000007562be2022d31a75f9887b7b932256c704f0c8e7000000000000000000000000000000000000000000000000000000005f7bf45b0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000009b06d48e0529ecf05905ff52dd426ebec0ea301125a057e78766ae798e30b06e2ffea600bb64950ee8498790fae4be4232a58a7dab76a00bcf0307bd910dc4eb8a0f59c08c38941c951ca5c176e394c20e2802f0ec0569f8a928850b2d05e00082f17594dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000005f053acac0d67f0e6d1cc7a5284727ddc108a56400000000000000000000000000000000000000000000000000000000389fd98026a0af5ebed76724a1a2ec0480f7a928c758f105039ece0c9dd358fd36ff89148d96a058f4978b9b948fc259616bbc48bff2d42b201af89d3920f730c9bda89bd95ad9f8aa3c850b2d05e000830320b394de4ee8057785a7e8e800db58f9784845a5c2cbd680b844a9059cbb00000000000000000000000038e3c92a543ed6c87eaed33bcba04b9d302912200000000000000000000000000000000000000000000004d12beac3ee39f0000026a0df7786f7994c5546becc6741ef476cb6d1a4b07cdfc95ce9d4c54973b86460baa07a32b0a60ca94b0902b6a2640ca23c65daecc540d4c60662f8e1488373659117f8ab819c850b2d05e0008301494794dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000002ebd488b3b362228c503afc7cdf1698f37722fc200000000000000000000000000000000000000000000000000000000000f424026a0ac0838acef8e84034d475c2e2ab9b194ca0a29ad1550a69aabc497e857373d96a03ba2143fd0ab21a2ca741b874ccd2db3b6dcdf36756b7b7e678b90157e068f4ef86e8319ac74850b2d05e000825208941c6cc4f378266a1ea89edf1f92fa529869571a18870e90eda39440008026a0f1885607a6a384eeca3f2a221cbae0a7d1be32189a26e3f6024f1013b6458a66a02eea3bde50f2a790547d02e53f21a721131cc0d6823a7a0f104a1d2e027bcb9bf86e8319ac75850b2d05e0008252089498aae5c4097f803a0616b537a4f62bb25ad837ec870e90eda39440008026a08da329c004df7420e2065f1344a77dc6e2819b5fc5db5812c362af58596989e5a054e3f72cfedd01491d81ac512663d1bc6915757f32d12ac65385214dfa6b8390f86e8319ac76850b2d05e000825208945b879b10efcc5400c4b5931be5e577c8c97593d1870e90eda39440008026a02260ca75dc848b82e2e24e00de3dd0bea9c664adfde6805a791c77c16dca03f5a0225743a8e187c5b54148cb099920874a99a9e761b2870e46a3f9bf4d60fe9e4af86e8319ac77850b2d05e00082520894ec0ae94c1c03959118a66fbb43fc39bc2c1bb63b870e90eda39440008025a08b750dcdc29374b2cb70f517ca646f5725b820a0acfe10b9e3a216a652b2f54ea0765bc5c251ee433c85ab35bd130bba145df8511fccce743adafc46722d4da752f86e8319ac78850b2d05e00082520894558d7cdf026486652f3d94b2c860c9fdb9b05fae870e90eda39440008026a0bad87f9413c9d85e5fd708e81d62f68d8c34addcfb8e269594573c6f9a9427a2a0672011774c1598a0099b20bb3426768a96ef8962de389ecb94d6eb7e0d5f0f81f904b375850b09429fca83053d2b94dc6c91b569c98f9f6f74d90f9beff99fdaf4248b88745c7039d837c000b90444288b8133b1d717c9fedc8e8f40fb977a4a887e7f3b9721a403109441c45dac7dbd42d537000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000032464a3bc150000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000745c7039d837c00000000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000d0e6cbd9da5d6214818a3ca2c4be074174faea8100000000000000000000000041f8d14c9475444f30a80431c68cf24dc9a8369a000000000000000000000000b9e29984fe50602e7a619662ebed4f90d93824c7000000000000000000000000dc6c91b569c98f9f6f74d90f9beff99fdaf4248b00000000000000000000000000000000000000000000000000000000b0de47ff000000000000000000000000000000000000000000000000745c7039d837c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005f7bf6caa989bdc492903b542331fb370f3307fd1b2b2bbabf59c552db34fd50dcf47be5000000000000000000000000000000000000000000000000000000000000018000000000000000000000000000000000000000000000000000000000000001e00000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000581c0d6b99cda91d02c52cfe70cdee39e3908e10733ea0c7bbea96e4bd0cf1eb3c9a179df6e905e92a575c6d947671bfe84aa761eca6ffec506e310223bd236d6854da8d98e2a1a211130e940c71f2004885e8c7810800160400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000561c4cce9f6578c835fb47c7b5d6fb1b4d4f5f815dda45bd98d920c70577d1f063fb3a80eacb0dddc9730cef6735e23abe58b2747d3173eb0f75ce1ca86d3472bb01da8d98e2a1a211130e940c71f2004885e8c78108040000000000000000000025a014fad5808e4a9ae55ab8dd2c1807fa7ec2c1d4c15bea056922b2e791b6264941a047f50bb736001dec7314f4ea345412053d147442a4204236d5f27d3d69ccae6cf8ac820244850af16b1bb383030d4094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000ba6f0cb6688748e1dd48d1bd5924f930bd164ed2000000000000000000000000000000000000000000000000000000000cc6438025a0b5374ec5198978e5e9e362dbe3dce2a7271bbb466ae3aeeb4fbc7fe845089986a02110277cc3dcaf906ef487fe923f0c36347415a31425449d0edc92c9777d5782f86c1d850ab5d04c0082520894c0631a2e7e8b96f7a22e117560b39c9e908a6da98806f05b59d3b200008025a0196275ae3606075a8f23e705b9c8438c0fe9aba2cdac79ca0fbc52b456826ae5a003071f97a20eb4e372adfe0261bc123f31dbe1fdf431749281a8e4626e6f9ddef90d8b04850ab5d04c00830357e8943b5d2b254224954547a33cbf753bcaa5eb4b27bd80b90d249ec9b36b0000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000001c5d932241d20c7a72a42bc6a023437347bb80940c45e7fc8c4c5e6f1cc02c591b1c7b04f95c1be906d5354ad4d3458e97a04a9ba7a45b1303756728eaa0d45b7f000000000000000000000000000000000000000000000000000000000000003200000000000000000000000007bae6e785f8c9f5aee6ba49cf72d19d5e412707000000000000000000000000000000000000000000000000ac7b5695e7e7f530000000000000000000000000c31bf9a4969a3ad0f2b9823317b329125bd3e923000000000000000000000000000000000000000000000000abc29874867d4bf8000000000000000000000000d1e6fc718660618d73d458a77bd0339f6c2f4212000000000000000000000000000000000000000000000000a98966b8c60716b000000000000000000000000066b064b5fac00eeb32cd6bce64bbf5fd98581b08000000000000000000000000000000000000000000000000a8fc4c2e8afd20f000000000000000000000000061f1d048323de4aa996290f63a1e2a29636a3d06000000000000000000000000000000000000000000000000a8d7cb23466da910000000000000000000000000ad1b4d6d80aea57c966d9751a5fe2c60a0469f60000000000000000000000000000000000000000000000000a7a6751221a3ac90000000000000000000000000bdc7ac031671bd67190b09a8c3cb1e7872b6c6a1000000000000000000000000000000000000000000000000a7883a2f27b3c020000000000000000000000000088b99442b893a861db16e5c8c6a01a66d0c3ada000000000000000000000000000000000000000000000000a6e441d4b6183960000000000000000000000000dcf54138acc74e9b99570af10328e89d97d5512f000000000000000000000000000000000000000000000000a6610325fe2ec410000000000000000000000000d65e87f756d0f21faceb8d32eacc509c1284f25d000000000000000000000000000000000000000000000000a62aa70b1f6f1f6000000000000000000000000037a33fcd214405e74ed93addfc7f0787326c4f40000000000000000000000000000000000000000000000000a445a47aeb3c1b1800000000000000000000000068799b81c7bfca8c661197645755ac33da8ef4ff000000000000000000000000000000000000000000000000a3fa48f31df4cd6800000000000000000000000090496a8e1925ad4a1931ca2be0e50480abc24e8e000000000000000000000000000000000000000000000000a2f5c981c11bb8e000000000000000000000000097cf2bcd41c323940022bf8906da9011b8271ab5000000000000000000000000000000000000000000000000a289933188fd0790000000000000000000000000945fe27749f564173237fddf47749b676a14111f000000000000000000000000000000000000000000000000a1088c0a7dd222c0000000000000000000000000b21763ce87472dc76fc3b485fca9d57df5d47bfd000000000000000000000000000000000000000000000000a0b781dfa364764000000000000000000000000067513103973cf85cb20f2f8ebe61cfc1990d3d3f000000000000000000000000000000000000000000000000a0820178a791d2700000000000000000000000003b2e5507929f9b794f39acba3f0311cc636390140000000000000000000000000000000000000000000000009f93541fdff84f4000000000000000000000000080c5cae1fc0b6a548ea16fe40171d01555f4f57e0000000000000000000000000000000000000000000000009f8a6ff0dc0416b00000000000000000000000003d16451a4b73e778bfec126025ba79716a17e32d0000000000000000000000000000000000000000000000009ed512db2aea289c00000000000000000000000033debb5ee65549ffa71116957da6db17a9d8fe570000000000000000000000000000000000000000000000009e9a91a4b0b0683000000000000000000000000024d0db7e182fcd29ddea14c04bcb182c89cbb0c00000000000000000000000000000000000000000000000009dcaa8e3be581688000000000000000000000000952ab11461fc27f65175b0469eee9f0f5af3a9ef0000000000000000000000000000000000000000000000009d3748d8c22dc31000000000000000000000000061af6c7925cb106ea04fb3148affba6220bda5c20000000000000000000000000000000000000000000000009c6cf18620f1e0180000000000000000000000002db3c0f42022fdc8dfe70036fee85e48a24b88af0000000000000000000000000000000000000000000000009bd0fa53e03389700000000000000000000000002fe50c88f228dacfc24100de0c5167aa7a539dc600000000000000000000000000000000000000000000000099d85913e41be54000000000000000000000000033d66941465ac776c38096cb1bc496c673ae739000000000000000000000000000000000000000000000000098e8c47b582093f00000000000000000000000009e75d5cf5d6a1970e47bc3d37f3c21a878c321e900000000000000000000000000000000000000000000000097c535c877dce088000000000000000000000000852bab46dd2dd7a585d23a79a8600c2a65fcf2e000000000000000000000000000000000000000000000000097811829ba6749d0000000000000000000000000bb9297f03e92ecefe1a45bda0e585fc6f9fab41600000000000000000000000000000000000000000000000097811829ba6749d0000000000000000000000000dabc521955476811ed6d39d7d54224c37bd8acb000000000000000000000000000000000000000000000000097811829ba6749d0000000000000000000000000eb5831daff60bc27a3af7ded164ce159dcac035100000000000000000000000000000000000000000000000097811829ba6749d0000000000000000000000000f4317d52acdaf924329c8a15c37a9273e2a3797b00000000000000000000000000000000000000000000000097811829ba6749d00000000000000000000000006c31b772cc57cac99c85d2d0b0df8e54b07a7a5500000000000000000000000000000000000000000000000096b154951fcf8310000000000000000000000000ea94c9a3bcc2bf5a3c08e3de1ccc64c6b604971000000000000000000000000000000000000000000000000095585cb6c4282a10000000000000000000000000e5ae7e62ee71f675f2fe0ff5cc83c215a44e76d100000000000000000000000000000000000000000000000094ca53acb0c381500000000000000000000000000db5e7924d87d2ec27876882e7af911a58050dd1000000000000000000000000000000000000000000000000932042d59dfabc70000000000000000000000000c2d3ad9240a85cbdbf2e77223185542996f192eb00000000000000000000000000000000000000000000000091ad5c281f8ac6d00000000000000000000000001fd252771bc2dfd38ec774efeeb3c6ab7e8e535b000000000000000000000000000000000000000000000000915a77a49303dab80000000000000000000000000b946efae53975b97a0d1d02f75fabf55d0d6a960000000000000000000000000000000000000000000000009082d22b97189770000000000000000000000000e07e487d5a5e1098bbb4d259dac5ef83ae273f4e0000000000000000000000000000000000000000000000009074a6a89f402e980000000000000000000000001fc022c33332bb9e45f3ff8129a51f03347f842f0000000000000000000000000000000000000000000000008fb33fc665fa01a00000000000000000000000006726b6dc15eaef7a530cd9e27a5027fff66a46420000000000000000000000000000000000000000000000008ec37e27521c85500000000000000000000000002fec5d8835b39f314187dc1da933b273f32f18c30000000000000000000000000000000000000000000000008e2577f5cf5c0b700000000000000000000000001f4d088464a0175c8ddb90bc7a510b1d5a0da1a60000000000000000000000000000000000000000000000008c7de56bd4e5a0580000000000000000000000000c780749e6d0be3c64c130450b20c40b843fbec40000000000000000000000000000000000000000000000008c14be8f739d98a8000000000000000000000000291502048a3396486cded59288df83fb9cc9d1cf0000000000000000000000000000000000000000000000008c0d91180c2789f00000000000000000000000009e199d8a3a39c9892b1c3ae348a382662dcbaa120000000000000000000000000000000000000000000000008bd9a02684ae1cc0000000000000000000000000c13ec844eb19d6a72ddd5f2779484ba35279a8170000000000000000000000000000000000000000000000008bd9a02684ae1cc0000000000000000000000000970c7ab1ca32547eaacdfcc89f6ed924600da8670000000000000000000000000000000000000000000000008bd9a02684ae1cc025a0d7be5b752c62db2acc078d5ec60fdf525d8d1866399827772c7b97c8b2996b7ea0390cf16c61b37128ad6e8e56a36190e11de890f326ba7014b82fe655f08cb37ef86e8305985c850a567242d6825208941d86c8a33c7b3b6c7f2fc580e7a2b076c49cac75870118a797e62f6e8026a029ce1c2f5f6e95826e3484b579a32b14088966c802120a7f64051d64188683f8a044a9a1c7f2d05971808a1699b2d21e1240dd5e99ddb716cd34be9ebc23e17ff3f86e8305985d850a567242d68252089499ab81b3e74800ac05958cf8e8ebb9d3a3f994648702c8cc350d8c048026a0f06dc5296371c0503dfd1a8c7c1372f23bce4ded60a992e348df49116241b12aa0659b3d14f8a3a9a26bbf8571e9520c8f319977a368c665ced3708daa18074703f901ab43850a3e9ab8008304fba5947a250d5630b4cf539739df2c5dacb4c659f2488d80b90144ded9382a000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec700000000000000000000000000000000000000000000000000001e4bc4bcbad7000000000000000000000000000000000000000000000000000000002ddd72f10000000000000000000000000000000000000000000000001e3cf46b06880d66000000000000000000000000ce156d5d62a8f82326da8d808d0f3f76360036d0000000000000000000000000000000000000000000000000000000005f7bf4390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001bfd21d58146585fa30ccb92362c65c6e09d6b9ee2bdc77e87f02dae12152a13f72e3c0ba57b03f89e25d5ba8bbf4f945ac5b8f1693b7739f3d802f640876b682326a062147cdc091cd1f38684508540739bcd6fe25c07e3bcec27f88edae81e0c4753a0569a6a8f75bbac18ad6a97ee205aebafa30f8c31d53d0f582d6630c0eee3babbf9012c820539850a02ffee00830493e09451b4b27a7bd296fd34ca7c469f49d5bcd7fe513780b8c4942d468b0000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000010000000000000000000000007c5e7fe5fbef2a35b02e1bce0d39e0076ec389a30000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000c9c653445750800025a013e64eaee87f711f381bb4a6c240ebe85489843050872b076872b512f7c809eda02864404d277f8dd792b0871b2873e086725661906c5cb606e93a1a4291261328f86b82040c850a02ffee00830493e0943388a0117f0c5c7f493afdf46ea03fe7445dab6980843d18b91225a0125032c84dd6e8d83c39daaeb652d36590cc3fbc36924b2cdedd080a70f5d819a03cb2f9762d738d5140831726d20d039e83a0b12ed93e28d0f5563737d1a043b0f8a901850a02ffee0082e74994dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000006c8d7c88371e673807372d7d3eb84101905be6cf000000000000000000000000000000000000000000000000000000004510cee21ba03e58bb7a78dcfa7ed656e565fe79ab2ff5787e6790d18abcd8db3151f4bdff32a024087d8b67f5a2c7f4e82d73444b7fe59d29128fa3e69a407da7ded484f1633af8d2278509fd0a0d00830124bc94006a7fab93971acdcfd5e465c29d714ea0d530b5880de0b6b3a7640000b864ad65d76d00000000000000000000000000000000000000000000000081c4d9b0b73ea666000000000000000000000000000000000000000000000000000000005f7bf45c0000000000000000000000001764dd87455540b164f45edd8d1efabeba26d52425a03359ecd73c9462378f22b31f6d3cc9cc0f4689d2748baf03a15bce87d66f826ba00ce062c7e1e797ecdee2f97625959230607aad8f182d44c367e855e26647fdfff86c028509fd0a0d00825208949187a1b5022ad5e801c474589c3d683e067467c188058d15e1762800008025a0151e70852cd730f3a3069bc8088f0caa6dec4c16defb7cf78776b0a55477641fa0709edc84332329640ba64a75b28e36c2bd9472cf251f10ae679da3cb0a8665b9f86b038509fd0a0d008252089469ae0b74d23a741a25a6e997de6418f374a0cf4d870c8304402804008026a02d86bfefc7eea31f1899d12e71d8ad9399c4a74ccca7fe43218a485858dc6bf3a00c93da0de2cfeda1909b651cafc831f28ab568b18a195bc5c4538ed09094f372f901548202848509fd0a0d008302a6b4947a250d5630b4cf539739df2c5dacb4c659f2488d8820b00eea5820b8beb8e4fb3bdb410000000000000000000000000000000000000000000000001bc16d674ec800000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000373d0ed6a697846661c3b0a0b9b94577c564c79b000000000000000000000000000000000000000000000000000000005f7bf4690000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000c34ef872a751a10e2a80243ef826ec0942fe3f1426a0b9c2f746418b791458f6a3437cd5c73c486373a6917c2e6def8d938539aaa492a03e9771211cee9602321727096da87c4b78b086a5ea0d08fe6b5164217333eac4f8ab821b878509fd0a0d0082ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000003c707feb77ee94b42ea0544c6982865b651ac9660000000000000000000000000000000000000000000000000000000005f5e1001ca0d26fff2eaa7936817c0d64c73dd405a2671e6db3f9d5d38adf58312f035cf734a0418044c1fe119a29b5b280dc352c96a70d1ebc1a31e67dfbd4d9039f063f10c3f901527c8509fd0a0d00830243a1947a250d5630b4cf539739df2c5dacb4c659f2488d880d2f13f7789f0000b8e47ff36ab5000000000000000000000000000000000000000000000000073b44eb80b69e320000000000000000000000000000000000000000000000000000000000000080000000000000000000000000579c90f77ae0c44ea7ba329c41a980d36348d8a7000000000000000000000000000000000000000000000000000000005f7bf45c0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000005befbb272290dd5b8521d4a938f6c4757742c43026a0c9e13f207c1877b89b510db359d494bc9a46cbb54ecd7227eb28c8f101decfe2a05ada94e4ecce058759c82d417fb1135c93d878f3e2143f73e88a2741feb1a853f8aa018509fd0a0d00830132b294c05d14442a510de4d3d71a3d316585aa0ce32b5080b844a9059cbb0000000000000000000000006a43c606609fd2f10b5e04bd199c311f2381a26a00000000000000000000000000000000000000000000007062f534e5517b000026a0f41544dcc58b2f6c501f4d339e8019c1c05cdd8a95eea59a11a67564e06c16f0a070ab365c8e8138326e309989eba25093c4731748d7449f1903a403118f2ff62bf9016b558509c765240083029c35947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe5000000000000000000000000000000000000000000000000660fb7013f9c7b1a0000000000000000000000000000000000000000000000000cd39e0af9d7081f00000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000f68b6c658087cc45a32afdb8c38509f6eaa84b0c000000000000000000000000000000000000000000000000000000005f7bf45c000000000000000000000000000000000000000000000000000000000000000200000000000000000000000038acefad338b870373fb8c810fe705569e1c7225000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc225a05479d4365dc72b124109ddf48047e17ed0bd6b111dd72343dc57f3f0b23df5fda0544dd5bde89240cee4ecadcaaf71ece54053a68646fe98203b812cb5d9e6ca87f9018c81968509c765240083030526947a250d5630b4cf539739df2c5dacb4c659f2488d80b901248803dbee000000000000000000000000000000000000000000000091fce1efdfdef40000000000000000000000000000000000000000000000000000000000003185259f00000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000b15dc08b5d8a0c596c1f7970732ba7e5074c9ebb000000000000000000000000000000000000000000000000000000005f7bf0b50000000000000000000000000000000000000000000000000000000000000003000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000009e78b8274e1d6a76a0dbbf90418894df27cbceb526a02fbcf47980497d578369380634d9107bce7e5e00e3db4335a46513a054878971a04abc8b37dec7aa082b6fcaa84577838fb08fccbceaff700be48102b5d6c17153f9015482019b8509c7652400830283bf947a250d5630b4cf539739df2c5dacb4c659f2488d880de0b6b3a7640000b8e47ff36ab500000000000000000000000000000000000000000000000274287b3e75d1707c0000000000000000000000000000000000000000000000000000000000000080000000000000000000000000f3c31bb059036a9cb0fe4f516d88aca7a8c0ae0f000000000000000000000000000000000000000000000000000000005f7bf0240000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000000fdc5313333533cc0c00c22792bff7383d3055f226a0c263437091b1b6470cefea0b3e958b1df35b51bb14211dfb217a0ab42cd8686aa02c97da3139e224a50a66d0d0256a463e6fbe5c3e0eda7d5c4f1a156b07b02bfbf86c148509c765240082520894cecea3a4cc75bdb5681e0c1e9b9ef09a8793bba7880780d514a93f0000801ba0ddd15b761f33f327617511150ef88572812500f1b1720d124b118e39b8ffe000a051cec8a670a822bd6faabae5cdad84d770f97dce67a654bcfd5b65caf3aa167cf86b028509c7652400825208945cfe24f2912ebb0fbbde9d70e6c495e69f58a58f8725ec578c01e8a18025a0e755e96f5d27937fbe5a171c2601df09076a1cdd21f1717a1042bf8196b19af5a03d784c0743047be9f39af71cb3d0eddb9c1e89d4082cba5eb39cd6ed78a0a4daf86b0c8509c7652400825208940adca8d84b20b30a8a7d4ed207ccad825c0cd39f870aa87bee5380008026a09129654f51f4fa187ebf9b7af4b95bfa25a1135f51e1fd247ef9c0dd3046b527a0464e5c2b13f0e3108a67a3201676a00115770a915b2cd8a42e887cbce681b3e5f90131038509c7652400830308d3947a250d5630b4cf539739df2c5dacb4c659f2488d87071afd498d0000b8c4f305d719000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec700000000000000000000000000000000000000000000000000000000000ac6eb00000000000000000000000000000000000000000000000000000000000ab91f000000000000000000000000000000000000000000000000000711e4fb1a60000000000000000000000000006c015175b6fa02f93ce0fe99e13554fa6a104c59000000000000000000000000000000000000000000000000000000005f7bf45c26a0654606132d677aa2b799c7ff664581d1fd7e10534c7138fbcc871772e275ae46a0250615fdc003986fcda12488d162770f98973e583ab86b8c07132cd0fed93272f9016b028509c7652400830290c1947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe5000000000000000000000000000000000000000000000000013fbe85edc900000000000000000000000000000000000000000000000000000180b9cdf65f766d00000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000e9bc563691fa5adca0dab7ed8e7c1341277ed4c2000000000000000000000000000000000000000000000000000000005f7bf45c00000000000000000000000000000000000000000000000000000000000000020000000000000000000000009dfc4b433d359024eb3e810d77d60fbe8b0d9b82000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc226a05f5fc078448ccc652abf2e867870bacd9c4f78b10fb16dcc5597170d9d04f7a0a07ec8ad3754705ea33324ff8991dcaa36b8ea34722f506dabc7ed6733f3847da0f86c238509c765240082ae3494f22953567a8807848c36ee02576694649a37f645880bcbce7f1b1500008025a099c7734ad747ff41d40e3ca37f28c1e7c1cbf11bf495fcfc7db4161f76901aeda02c08079a11a875b06bd38b9db97266613f83b2c173462185e91295500b8c86d1f8aa3685098bca5a0083015f9094514910771af9ca656af840dff83e8264ecf986ca80b844a9059cbb0000000000000000000000005c985e89dde482efe97ea9f1950ad149eb73829b00000000000000000000000000000000000000000000004f78d640cc92a8000026a036462ef4f7dc79093050f26be02535f20064fdedcc456c27c45b4e4bcd3b186da03ac78299efb26baf2e54276886e2d726c6459b7d67e614f63f549c9cc6b29cc4f8aa0585098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000fdb16996831753d5331ff813c29a93c76834a0ad000000000000000000000000000000000000000000000000000000009fa9def025a02576ec6be227f2ccebe0fe03d1c89e0c6e85a4da440c13006662ef3b79f246cda01b207e62214fedca6621c32f8f9b50034bb76b73a9a196dde34c737c7591976af8aa0385098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000adb2b42f6bd96f5c65920b9ac88619dce4166f94000000000000000000000000000000000000000000000000000000007733b2c026a07c2de322142a5037fe2b0e849903883af9f7801b64b12049f2eb31fc6b4fc679a04287a9e2e1bf0238aac81b0bca1c6f2e147cd43bf9ef219f4eddd424504e3ed0f8aa0385098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000fdb16996831753d5331ff813c29a93c76834a0ad0000000000000000000000000000000000000000000000000000000077326f5025a0bc3b627348c07ceaecde8c20ae369ed4044435eadb98fd730d39327b03110ac0a078cff03080223c0fc2d61652c62d4b77fd4e2662338e1fd6df25738d482d45c8f8aa2185098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000fdb16996831753d5331ff813c29a93c76834a0ad000000000000000000000000000000000000000000000000000000007731273025a009d0269469b831785c4daa4f705547f3379099988f0409427a6b7aebfefc8b07a06004458520a81a83c5603c0ec2717660dfafa2f58c8411d237f44fe1312d2628f8aa1585098bca5a0083015f909432d74896f05204d1b6ae7b0a3cebd7fc0cd8f9c780b844a9059cbb000000000000000000000000e93381fb4c4f14bda253907b18fad305d799241a00000000000000000000000000000000000000000000014fb68f5d03c265400026a01855cad304a3c245f7635998e739ae70924444b10337677c554369b25ee76e95a00ec7988b478d3e0aa4e7a434da8928e413efc073108988091fc817d624d9a2bff8aa6985098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000000a98fb70939162725ae66e626fe4b52cff62c2e5000000000000000000000000000000000000000000000000000000007f60084026a03a371eca9ea58a82b38af8032e73870382e18009cf39d8eb290ed342b0825bfca03f3fa1cce7e07ff0909a1914d6747c33123634dd6a5940f2a3093e020ce9f8d2f8aa1585098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb00000000000000000000000046705dfff24256421a05d056c29e81bdc09723b800000000000000000000000000000000000000000000000000000000773021a026a063a1a1a30aded50d8619b08e0fbf2b9defc2e521a0731f2953518b65cb413907a02ddaa13c9793edf51f570429f09f714f2a79c1a9a1ccc343899be81911bcdfd6f8aa2e85098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000adb2b42f6bd96f5c65920b9ac88619dce4166f94000000000000000000000000000000000000000000000000000000007733232c25a0eea5d54d8a2988876099e73b7a7b458a370f94f4b7343119da720bbe130624aca048ae11d7219a72fc9f20e1144624db5f1ec7d2b220b5ef59d6cc22c4feda515df8aa1185098bca5a00830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000006748f50f686bfbca6fe8ad62b22228b87f31ff2b000000000000000000000000000000000000000000000000000000014b48598026a09bd76874b6033a5cab13a2215a7b3a78a571c4411a8225d4855de261a1ece36ea054edbd5d59ed79cdad561ea97df9fc17977495ded7ce4ea4955725a10562e7f1f8a911850979e8bd4482a0f994dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000006b71dcaa3fb9a4901491b748074a314dad9e980b000000000000000000000000000000000000000000000000000000001091f18026a0737ea17db2080698c2af3d7a9f9ca14bfcdd0e21fc0e592271fe050d4209d1f4a035ae008889f7d5d85b8ec48bc3929f681b09305f36eb65a6760918e9b9b04677f9018d8214c1850979e8bd4483010e2a94d141a26d786e9432d345774721fc141c13ec10f380b9012452bf1d92000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000010000000000000000000000003fdc8e278f5bc4576a7fd13d1a01dc081ae0a2b00000000000000000000000000000000000000000000000000000000000000001000000000000000000000000da57b127b937185816683503de31cdb1fa203001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000af90e3681ba01773e74e7b8be1b0a6cdea3dd634dcb724613891e99b06e690a05addee7d8cfaa04d966d9097c1e3b0f2d2dd7a2eac5deea28d8fa328643d0b065ea86aea472fedf8aa07850979e8bd448301d4c094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000c3462da2baac16770caf09c6349241945c0714f700000000000000000000000000000000000000000000000000000000810351d426a0176f3b44e6e9c0c12c89132a64089ba2b9d66aaf7aee5df92747c5a9bd847fcda01d9d2f4f002f006817cbe5da7375dfd5949e682d24bf347c61fe3350c9098764f8ab821b88850979e8bd4482ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000531052427a3fdb4e1eebc8765cdf6e98972b31d4000000000000000000000000000000000000000000000000000000001e8480001ba019040c51840ce978756fb8d0b98a47b05d36e8e78228ecaf6d223f93fae8f3caa01ae2bdf972eae310ef838938165d47379563cdd265a7004ff42836917da8a522f8a906850979e8bd4482a0f994dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000006b71dcaa3fb9a4901491b748074a314dad9e980b0000000000000000000000000000000000000000000000000000000024d5ee8025a06fe92d92cc309c4150846ea5ed9413490306579b6c7ce664478443e6fbb74ac3a046448faf47445d8b533c6423aba3ab72322b3e24ecdea9a0051c6c7983927715f9018b51850973f2d6008303b352947a250d5630b4cf539739df2c5dacb4c659f2488d80b9012438ed17390000000000000000000000000000000000000000000000002636c37f92d4f0c0000000000000000000000000000000000000000000000000054e31d02918ea9000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000bcc414838d46bb02a5651a8714cd647a0c1fca27000000000000000000000000000000000000000000000000000000005f7bf4640000000000000000000000000000000000000000000000000000000000000003000000000000000000000000dfcb3fe1c69bf561a920f623fd152e8630674d0d000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000009464a922c053c519ac0237a9650e303d532a4d225a05d067e854341536417fcf031529d2ab65232d4d7964a8c7dac9907cf6fb738e8a055549b345c8b0a10e9aba2d721142995d9e4a74393107d46ca6f80a186b13dd9f9015267850973f2d6008302a2ad947a250d5630b4cf539739df2c5dacb4c659f2488d8805734bbb01b06a28b8e4fb3bdb410000000000000000000000000000000000000000000000000de0b6b3a764000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000001ce0c96cf5aad45a8824c22152ec5055f83ddaa1000000000000000000000000000000000000000000000000000000005f7bf45d0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000000c7be37b00c1d3127c09da66fadf3d6eb600dece26a07033f3a1433bd9455f9808c6ea10268aa3f0399b351c9fa96487fba14ab00e6da028578d7fcfa99a356d83787d38cec328f981bd2f3a50a5141f22298f8704edc0f8ca028509502f95b383019f539427b4bc90fbe56f02ef50f2e2f79d7813aa8941a780b86423b872dd000000000000000000000000946f2bf0f01580f3063b94115d48221a92916cf7000000000000000000000000f629527029499db6a6154a1cb20eb210735da33d000000000000000000000000000000000000000000000000000000000000927225a04852d05e9c6c106918bc5312720148113e6a48ae4403a6fd436b08ead64d41e8a05c1cec000313c180f5d29294c27b0d98b3969b7644800366f23ca91b73d31d78f8ca268509502f90008306b5e094bd277e47d0ecdd5db6c57eda717dd8f5a329edec80b864fc1d230e000000000000000000000000000000000000000000000000000000000000898a0000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000025a0542d85a2796e173a90b813bce70913b51600f7d180a64195ef04d7e2c7b8f90aa0123ed49099ae6c7a368fba99cdb6b6fdd1622e66b9ad8e663ca7d9065e51bcf8f8aa808509502f9000830186a094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000adb2b42f6bd96f5c65920b9ac88619dce4166f9400000000000000000000000000000000000000000000000000000000e10f4ba026a05574388857f5e60d3ec72ceb629d81b98c336f0907ef37f4359a46dc8213dc16a02e749cdc7f6b0a7b240235be95128f0a603abf4e7ad95e1c3df043eef640ef6ef8aa168509502f9000830186a094df574c24545e5ffecb9a659c229253d4111d87e180b844a9059cbb0000000000000000000000001062a747393198f70f71ec65a582423dba7e5ab3000000000000000000000000000000000000000000000000000000743058279026a063f627fcf6ac293cac6f84b5f8807aef92682368e99b244878d162998e5c250fa07a20fbb570af828a69f7e62d06cfe913fa3739f42cb43c2cc16e036efbaa0709f86c808509502f900082520894f85a34c8c414dc13ea15590cd383198ef0c5c63d880e397b561dbfa0008025a0228e8f924ee18c31d102873dada7c8564021e1bf7b3f717cfa230f59ddc42132a045bb47ed08ed4df51d1325f8670ffb91ee4c8891036886caf72b474afa6d94e1f8a90685094312a10082ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb00000000000000000000000030950d97d1854f587b73f8346c2a808ad73b73ac0000000000000000000000000000000000000000000000000000000051a88a801ba0a3bb6f585af27c6fd0053936542418d3583b5f7d2887a1ba353160f4edb008d2a01ccc663badc590398149dc6757d2351d8276cbd71c8771549cfd902ceda762adf8aa80850938580c008303d09094dac17f958d2ee523a2206206994597c13d831ec780b844095ea7b300000000000000000000000041f8d14c9475444f30a80431c68cf24dc9a8369affffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff26a02a8ef1097bb4bc7036343ad59ce010d590ba8c3ad659b160e50287bc817b6d89a06d4e37d3f219cb4f9cf06328259d09a5441e0bf2aec1eacd999ae3142fca4a25f8a90c850938580c0082a0f994dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000006b71dcaa3fb9a4901491b748074a314dad9e980b0000000000000000000000000000000000000000000000000000000005208fa026a04b59e7f4ce7a662e3d133a26fb7beea4c8dcd2df71f162dcfade0a0b28119d4ba06da4670c1575a071084fcebdfb6c73dffbb7d6bde8364d73c1c798aa8b0090f3f86b4b85091494cbb3825208948ac4bf2465dbeec007b7f9809979f22aedeb7806870a71ea17a3c0008026a05a5a5f216393d82283357f4bc9a9b569b467cebdd08da509cc1880301bbefd01a060d0c4d61ad1455b6847965f5f521e62affa0fc86f97eb6395344bb746549344f8890385091494c60083018808941f9840a85d5af5bf1d1762f925bdaddc4201f98480a45c19a95c000000000000000000000000050ed33cc70e3d7952967acb55901b4b44d3cee226a0def74c07de86a0fe72d478fb412e3430b28a2658fe6b10eac18311f6f5a3c4b4a06246e2c183d3dc928cf860bf2c82dc557788a4f8866c179297bfbe1d6479f06df8691985091494c6008311935f9401285d34fe29262a61788775df716acc7ddf897c8084d1058e5926a0ef28dfb04a55901b9a67b00c794502813cfb85b019c9804a004fec560b3c17c3a056254016ed435f806ba9b92be8b3a290a9617b79bac068f223b6abac5220af69f8698085091494c600830878a6949765fea9752505a685c1bce137ae5b2efe8ddf6280844e71d92d25a084331b1cd673c4634c16fa25c6d381b36fcca12b8fdfa34ceeee2f4e7fc68d29a05c0c6ffae21c5c348d0b03c82c6f0cfcb75a10c0b130809aa791977ce26bf17af8ca0185091494c6008301a3869427b4bc90fbe56f02ef50f2e2f79d7813aa8941a780b86442842e0e000000000000000000000000068323fa0171364b9fbb928dcfc22dfe965785f800000000000000000000000003bbdbc26a0aaf9a8991a28b1cc429c8e0ea2897000000000000000000000000000000000000000000000000000000000000927b25a032507f49426c3b56e42db20d0cea50235528728ab5b98097fc562461373161f8a065b9f9df23f54faf91e8a298f1b40da3f00de262492a62f0f9b85d89cbfb2085f8a90185091494c60082fde8940435316b3ab4b999856085c98c3b1ab21d85cd4d80b844a9059cbb0000000000000000000000007b00bcf548a069906f654627c3e04795357d82990000000000000000000000000000000000000000000001f29fe46ae84af800001ca0a7a4d1458365c4beb07b05a6e7dfa0a9bdf73807c508dbd4cd65aa254070147ba035333336910d89b785f3c33feb9c46a11230002e14e32f90533c348499ee6a35f8ac824b2b85091494c60083011170949dffe202df7f82ba57a7f8d571628805eff7fed980b844a9059cbb00000000000000000000000017849384ca9173290f540e566e369df84e924b9800000000000000000000000000000000000000000000006c6b935b8bbd40000025a069acf936392176ffb46c35851218efbe318218f8250261db49a8c9fd411aef94a049509eb3ddd2865547739a9be6c45958552f9d77f9abbbfea2c5047984e378f9f8aa8085091494c6008301259494725440512cb7b78bf56b334e50e31707418231cb80b844a9059cbb0000000000000000000000007b00bcf548a069906f654627c3e04795357d82990000000000000000000000000000000000000000000821a69b498f0c61f800001ca08b42ff2c6746c6663ac1ce97183287bfe79e1e36906ba0e3a349707bdf378442a04f8af6df29b3a0bbc4d5b2b7b1934388c096c9c5eb90172fb297d9361cdf80d4f8890285091494c6008301cdbe94a39a6e35d98e531058b52c28af4ffdeeafb1749d80a4a694fc3a00000000000000000000000000000000000000000000001d776c9997241a800026a0acc704c9c151bb6988ff16cc1f9c3605525c7e2dfebb1251ae4a21733e0a5447a00fc1abe875a992fa1368766c31d1a301467cf565387d9b3772dcd50b96dbad02f86d0185091494c60082520894d43ffc3367ea4fee1a32342c8b92ddc558e5e826890108721d3368fdf0008026a0efd4fbbadc6d2714947ce815eeb99f711459db9573d66a14b4837e1a994442cba02da5a5d74fbf3d4b5a8a17acff8d4e8d0055ed488ef13e56c738e95ef8c85efdf8cc82034585091494c6008305d22794b86021cba87337dea87bc055666146a263c9e0cd80b864fc1d230e000000000000000000000000000000000000000000000000000000000000922f0000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000025a0734009bcae96f41f77d600c562a70c1606abcc0751268407b7a4829b92eb8cf7a00fb0a91b8b350cc898d3fbf4ea9249344ee677e334d353f228ebc1ec40bcffe5f8698085091494c600830878a6949765fea9752505a685c1bce137ae5b2efe8ddf6280844e71d92d25a0ee1e0d0c01d19fa2881d2db84d4c54c8719ecee4fe8ada44ab6b51bd1892480ba065d9eed77a7f76565d10552537b29753f8f7182403f0334b0a823bc70fa1da08f8897e85091494c6008302b4c59475ce0e501e2e6776fcaaa514f394a88a772a897080a451c6590a0000000000000000000000000000000000000000000000056bc75e2d6310000025a0b56d9e64fd7baac51c8a1464af551978ce94037ba93cd23499a01a42c7950a5fa06d04bf6de15e8f2ffb9359ce517960422106a3bd9c5a4baa3e865ee3e021e2a6f8cd830226b185091494c60083014c0894bb97e381f1d1e94ffa2a5844f6875e614698100980b86423b872dd0000000000000000000000006dd0ff909dc42abd3b6b52f6e7643d215715e1e7000000000000000000000000c1ab0c708aa638ad492d99383217c033f0eeab950000000000000000000000000000000000000000000000ce661595e03f85f8d925a01831096c49030fd369e15776f7d4bac38ef0f8e5572e3447841e66c5ccd5807da02fd5e62915e24339d30a472a565330b8808dbeb06009e6bed9715893c8b36d2df9018d82017585089d5f37b383035ff3947a250d5630b4cf539739df2c5dacb4c659f2488d80b9012438ed173900000000000000000000000000000000000000000000001650c2523dbb933d49000000000000000000000000000000000000000000000000000000000674219400000000000000000000000000000000000000000000000000000000000000a000000000000000000000000082cd3c7035104d1fbfb55acc62667907791fe423000000000000000000000000000000000000000000000000000000005f7bf4390000000000000000000000000000000000000000000000000000000000000003000000000000000000000000b1f66997a5760428d3a87d68b90bfe0ae64121cc000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec725a0f6147595b4fba2f27e1d595a05c11fab5953335862c6edea856f9e9a77814529a056473fcf8e676c26ba05a1767ae9a514901c8fe6faa1ec473d47879afe7762f3f8ab82015785089d5f37b382ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb00000000000000000000000089191c0f10eebde841ea5d12b1f1098ef73859de000000000000000000000000000000000000000000000000000000174876e80025a0be3553ba3c9a205e4d76afbc0a6dd6f0d8a84f7efc8d5076eb98fc43a76808baa055e94b68d8a1da425f4febc9e0272cb65e1da38e6d58c06489d3b54e79b52b4ff8ab8218ee85089d5f37b3828fe094de7f89c3d1261f07589fa52fea19bc1b14dfc91980b844a9059cbb0000000000000000000000008700e5430a4b57f24a7f84abe38864e51af7a4b8000000000000000000000000000000000000000000000000000000909af338a526a02cfb25a32d955a1d8c6b51f339c8cb0acf6cf3e27d45fbb0a2ecfc12b85602f0a024d77aa56ace50dfe58c7413aa08a5006728e293bb26397e7be096277903938ff901522a85089d5f37b383028908947a250d5630b4cf539739df2c5dacb4c659f2488d880b263a94504c2e58b8e47ff36ab500000000000000000000000000000000000000000000000ae0b305bb98f9a7ad000000000000000000000000000000000000000000000000000000000000008000000000000000000000000071a85ed3f949a63dc13c127fc2f2751c13eed024000000000000000000000000000000000000000000000000000000005f7befe80000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000002a7f709ee001069771ceb6d42e85035f7d18e73626a0e3ed1ca94b4285e9ae128d9ffc907da874634e6784ac29db70570e5a610aacc7a016da61376d706d3c8bbedfad33e229d2f80a0015e40e5345e7415aaf9fab79d9f9016b0a85089d5f37b38302935b947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe5000000000000000000000000000000000000000000000006551c67c8504ff9b600000000000000000000000000000000000000000000000003bc3298899feb5700000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000e42bf6c707ff70adbcb5d3c41a18af9c7b59078d000000000000000000000000000000000000000000000000000000005f7bf436000000000000000000000000000000000000000000000000000000000000000200000000000000000000000005d3606d5c81eb9b7b18530995ec9b29da05faba000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc225a038bae485ad7254f45da6ee9ccfaa7b61e09335ed4d63145c171a6db4b2c58b43a06aaacde283c87486dfaeed8b7c65da2a4968b0da29e5891a74633d1142b9eb7bf8ab81be85089d5f37b383033e3794577af3dce5aaa89510135d7f6e095e33c06b8b1f80b844e2bbb158000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000026a05591dc7d86736fec625daea5eb6e71cf2d41f29b77d6d9e59f030ef0352aba5aa04459cd24f3cbba8eb1f5640ad1f1f36a8402cf438bcccd48b9feb280d80c1e10f901731685089d5f37b38303bbc3947a250d5630b4cf539739df2c5dacb4c659f2488d8802b3a3cb06ced308b901047ff36ab500000000000000000000000000000000000000000000022dbeb06eb8dc19a65e000000000000000000000000000000000000000000000000000000000000008000000000000000000000000032769651c655db12154b17910dd477592c54576d000000000000000000000000000000000000000000000000000000005f7bf4390000000000000000000000000000000000000000000000000000000000000003000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec70000000000000000000000009b06d48e0529ecf05905ff52dd426ebec0ea301125a0a0d302734d13499be466feeda44570a2bdbf364b31e34be62ee16b4dc4b5699ba0687177566a9a2e6bf7d3f2d68cf60e93482447dd3b472b67c17b8f1e00981b78f9016b7885089d5f37b3830295f8947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe500000000000000000000000000000000000000000000000f23c3cdb2637622cb0000000000000000000000000000000000000000000000000f2ec381bef4f22100000000000000000000000000000000000000000000000000000000000000a00000000000000000000000003904ea4b55a09e7c6e4c213cce0db7a9f599a4c3000000000000000000000000000000000000000000000000000000005f7bf43900000000000000000000000000000000000000000000000000000000000000020000000000000000000000002a7f709ee001069771ceb6d42e85035f7d18e736000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc225a0f34f01bed0f338683c5207c1587d1a5ba229779db4c1de9d5482c4a905a2ad3ea068e89a5b8047924cb71cbc8e123e8b05a8c8719a049afb02a3c6972febcd0085f8a90985089d5f37b382ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb000000000000000000000000d3d0278216491edb86bc06f8cfb11d51a9dd808b0000000000000000000000000000000000000000000000000000000005f5e10025a06c7279aba31527aae21db6b895612f99ba1244bc9aef8264bc9fa6a15cb3efaaa073c383316de97f4c2b5b9b51ea7d74d17be0b4dd9529c6ab4e59e93f5fb57346f8aa81dd85089d5f37b382bf2c940ff6ffcfda92c53f615a4a75d982f399c989366b80b844095ea7b30000000000000000000000007a250d5630b4cf539739df2c5dacb4c659f2488dffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff26a0e4ce469a5e40f47e838933ebf2a5af9cc0cb39b1c87dcc90db9ea90207060960a04210af1974ec3fbade5ffdf79e92a4287b68290a52c8301aa7a96f325b29b089f8a97285089d5f37b382ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000002562dac3ecc0df840568597f5fe5db96d7fad37f0000000000000000000000000000000000000000000000000000000011e1a30025a05d6b5a2fb581a0c76a0b6ad86d9f14f9461a924bfa23fd3e6dc29469aecee960a047cb4d29954ef81da7a65933c5e398a3d012b106ad1f49dd78ede22414a59ff0f8ab82011f85089d5f37b382ea6094dac17f958d2ee523a2206206994597c13d831ec780b844a9059cbb0000000000000000000000006742a5be977a95e6bd413ad7dd67e750d25acf77000000000000000000000000000000000000000000000000000000000081b32026a09d70ff51194da072becc6095e121853a1e9500ee6285a37fb3fff0b7b37d8d46a03054b0e28af6cac98ca911dcfe7e47eef7e9546d1c4743d72b5761839c133f24f8ab82010585089d5f37b382ea60945913d0f34615923552ee913dbe809f9f348e706e80b844a9059cbb0000000000000000000000009bcae07a4f32391c337081db539f18c8a41f0b17000000000000000000000000000000000000000000000000016345785d8a000026a0d8364bccfe1cd13452d32320bbbe7007d2a49eb4c44e4192adcf13c5319dba12a0556f74086c4f19473dcd0e1651ed2f1ad41a6e4955ff4489e580637b76cf559bf8cc82027b85089d5f37b3830449df94398ec7346dcd622edc5ae82352f02be94c62d11980b864d2d0e0660000000000000000000000002260fac5e5542a773aa44fbcfedf7c193bc2c5990000000000000000000000000000000000000000000000000000000002f51218000000000000000000000000000000000000000000000000000000000000000026a05f066b907360fdd977a7be7b87f8abb63c026bd6a9b79961b2df31f4b05197b6a03ee77c3bb73623eaf754e19a3de2587917d5bc9fe9fd6d18d01263994a8ae482f9016b2d85089d5f37b383040b67947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe5000000000000000000000000000000000000000000000005fe80d4a5da71c8000000000000000000000000000000000000000000000000000f4d4114fc1b990700000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000d5ef38a6bfe29e37fde0e05c3b5d053d35cde31c000000000000000000000000000000000000000000000000000000005f7bf45c0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000960b236a07cf122663c4303350609a66a7b288c0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc226a077384993315e895c506addb765b6ead9aa40643cfe0e47fa9a7cbe7b68b5c26da023e3d59adf9659ebc24ac7b1c61a8d34870c1473482062dee58001e433dacffbf8ab8202e885089d5f37b38270d094c02aaa39b223fe8d0a0e5c4f27ead9083c756cc280b844095ea7b3000000000000000000000000e2466deb9536a69bf8131ecd0c267ee41dd1cda0000000000000000000000000000000000000000000000000000000000000000026a0122be7ab3dd42a95a6f9f7afd4b53cdd96a54533355678de8e6a079c1f3a507da0184286f4fb49dc18285b01b7b881ef7219783a94e4adba214d5f846cd05bf080f8ab8202e985089d5f37b38270d094c02aaa39b223fe8d0a0e5c4f27ead9083c756cc280b844095ea7b30000000000000000000000006f400810b62df8e13fded51be75ff5393eaa841f000000000000000000000000000000000000000000000000000000000000000026a09cad61d8896acfc385722a5bbb7cc933cf424848b3c1e5ca77d65bf05c6d6fcda074d8318b68ede0035abf2b7be4ff24b8434a14e9265539d0a7a667da017671a1f8ab8202ea85089d5f37b38270d094c02aaa39b223fe8d0a0e5c4f27ead9083c756cc280b844095ea7b30000000000000000000000003e66b66fd1d0b02fda6c811da9e0547970db2f21000000000000000000000000000000000000000000000000000000000000000026a0e6503c732760d18c807a75da58051281d667f8c4198c45ed51ca087614d5785aa01e224cb49e6837da617107107dc0ea4dfad191de3797142648bc1dde0df987faf9016d82018f85089d5f37b3830295f8947a250d5630b4cf539739df2c5dacb4c659f2488d80b9010418cbafe500000000000000000000000000000000000000000000000f068603508f9456ad0000000000000000000000000000000000000000000000000e494da22dc7c0b800000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000ffcee9c64c3cdab271f5c3c7c184b153f1344b2a000000000000000000000000000000000000000000000000000000005f7bf03d00000000000000000000000000000000000000000000000000000000000000020000000000000000000000002a7f709ee001069771ceb6d42e85035f7d18e736000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc226a0197b3da148fcee4ccbf62eb74082ef5b79384afdaf3abd2175173b404d2c6c3ba02fa9b883c18f17f6327cc8de85764d2b59b3dc3e8b5359a225585e719067c0fdf8ab82069a850826299e0082a95994b4e16d0168e52d35cacd2c6185b44281ec28c9dc80b844095ea7b3000000000000000000000000fab1eab539d2cf50162245e59ddea174a301d85d000000000000000000000000000000000000000000000000000000000000000026a04919640cc96cbb20a3fb12871737fbd812a1f8c812f852db7abfbec889c04a20a04c6f5c296e8c2628537c559d76ad3b750cf068b689e6021dc17691bb846d6d5bf86b1b8507ea8ed9b38252089409f72bc14a0901c3d8127dc3b64971187900a66987470de4df8200008025a0ef0162ddd727191632140402fcd1d3d3e05cc0386eb5700c71b12d85a1732176a02958d5b3f7e4c9a555a172c6b4921cc568708ea8138c26bd1445036fbbaef2eff86b078507ea8ed9b38252089437434a338fdcff98e8ae3c07e3ce6bfae701a47d87ce0eb154f900008026a0d38057dbe925e7517a9321248cdce4a91aef0caac2a28b275909ba1e9b365312a01b1f4727a4a644c733b7fc975dd3791d4931d1e210ad8423415a8d3a4ec85b41f86c038507b4e9eb00825208941bb95bd09a9463a9fb0874cc9305744eed1965368815957b01e7cdcf5b801ba06a8e9d014dd495c9d5c98b8e7d71e3d170e2fe4b83752d219f8bf794c8c9f2eda023dd8b8cb5491984b639e0855817d71dca426e1ac25053e2156d5f95042b86f4c0"
-)
-
-/*
-0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347
-0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347
-0x4d5e647b2d8d3a3c8c1561ebb88734bc5fc3c2941016f810cf218738c0ecd99e
- ,BranchPageN,LeafPageN,OverflowN,Entries
-b,4085,326718,0,11635055
-eth_tx,212580,47829931,6132023,969755142
-*/
-
-var (
- HeadersPostProcessingStage = stages.SyncStage("post processing")
- Snapshot11kkTD = []byte{138, 3, 199, 118, 5, 203, 95, 162, 81, 64, 161}
-)
-
-func PostProcessing(db kv.RwDB, downloadedSnapshots map[snapshotsync.SnapshotType]*snapshotsync.SnapshotsInfo) error {
- if _, ok := downloadedSnapshots[snapshotsync.SnapshotType_headers]; ok {
- if err := db.Update(context.Background(), func(tx kv.RwTx) error {
- return GenerateHeaderIndexes(context.Background(), tx)
- }); err != nil {
- return err
- }
- }
- if _, ok := downloadedSnapshots[snapshotsync.SnapshotType_state]; ok {
- if err := db.Update(context.Background(), func(tx kv.RwTx) error {
- return PostProcessState(tx, downloadedSnapshots[snapshotsync.SnapshotType_state])
- }); err != nil {
- return err
- }
- }
-
- if _, ok := downloadedSnapshots[snapshotsync.SnapshotType_bodies]; ok {
- if err := db.Update(context.Background(), func(tx kv.RwTx) error {
- return PostProcessBodies(tx)
- }); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func PostProcessBodies(tx kv.RwTx) error {
- v, err := stages.GetStageProgress(tx, stages.Bodies)
- if err != nil {
- return err
- }
-
- if v > 0 {
- return nil
- }
- err = tx.ClearBucket(kv.TxLookup)
- if err != nil {
- return err
- }
-
- ethTxC, err := tx.Cursor(kv.EthTx)
- if err != nil {
- return err
- }
- k, _, err := ethTxC.Last()
- if err != nil {
- return err
- }
- if len(k) != 8 {
- return errors.New("incorrect transaction id in body snapshot")
- }
- secKey := make([]byte, 8)
- binary.BigEndian.PutUint64(secKey, binary.BigEndian.Uint64(k)+1)
- err = tx.Put(kv.Sequence, []byte(kv.EthTx), secKey)
- if err != nil {
- return err
- }
-
- bodyC, err := tx.Cursor(kv.BlockBody)
- if err != nil {
- return err
- }
- k, body, err := bodyC.Last()
- if err != nil {
- return err
- }
-
- if body == nil {
- return fmt.Errorf("empty body for key %s", common.Bytes2Hex(k))
- }
-
- number := binary.BigEndian.Uint64(k[:8])
- err = stages.SaveStageProgress(tx, stages.Bodies, number)
- if err != nil {
- return err
- }
- return tx.Commit()
-}
-
-func PostProcessState(db kv.RwTx, info *snapshotsync.SnapshotsInfo) error {
- v, err := stages.GetStageProgress(db, stages.Execution)
- if err != nil {
- return err
- }
-
- if v > 0 {
- return nil
- }
- // clear genesis state
- if err = db.ClearBucket(kv.PlainState); err != nil {
- return err
- }
- if err = db.ClearBucket(kv.EthTx); err != nil {
- return err
- }
- err = stages.SaveStageProgress(db, stages.Execution, info.SnapshotBlock)
- if err != nil {
- return err
- }
- err = stages.SaveStageProgress(db, stages.Senders, info.SnapshotBlock)
- if err != nil {
- return err
- }
- return nil
-}
-
-//It'll be enabled later
-func PostProcessNoBlocksSync(db ethdb.Database, blockNum uint64, blockHash common.Hash, blockHeaderBytes, blockBodyBytes []byte) error {
- v, err := stages.GetStageProgress(db, stages.Execution)
- if err != nil {
- return err
- }
-
- if v > 0 {
- return nil
- }
- log.Info("PostProcessNoBlocksSync", "blocknum", blockNum, "hash", blockHash.String())
-
- tx, err := db.(ethdb.HasRwKV).RwKV().BeginRw(context.Background())
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- //add header
- err = tx.Put(kv.Headers, dbutils.HeaderKey(SnapshotBlock, blockHash), blockHeaderBytes)
- if err != nil {
- return err
- }
- //add canonical
- err = tx.Put(kv.HeaderCanonical, dbutils.EncodeBlockNumber(SnapshotBlock), blockHash.Bytes())
- if err != nil {
- return err
- }
- body := new(types.Body)
- err = rlp.DecodeBytes(blockBodyBytes, body)
- if err != nil {
- return err
- }
- err = rawdb.WriteBody(tx, blockHash, SnapshotBlock, body)
- if err != nil {
- return err
- }
-
- err = tx.Put(kv.HeaderNumber, blockHash.Bytes(), dbutils.EncodeBlockNumber(SnapshotBlock))
- if err != nil {
- return err
- }
- b, err := rlp.EncodeToBytes(big.NewInt(0).SetBytes(Snapshot11kkTD))
- if err != nil {
- return err
- }
- err = tx.Put(kv.HeaderTD, dbutils.HeaderKey(SnapshotBlock, blockHash), b)
- if err != nil {
- return err
- }
-
- err = tx.Put(kv.HeadHeaderKey, []byte(kv.HeadHeaderKey), blockHash.Bytes())
- if err != nil {
- return err
- }
-
- err = tx.Put(kv.HeadBlockKey, []byte(kv.HeadBlockKey), blockHash.Bytes())
- if err != nil {
- return err
- }
-
- err = stages.SaveStageProgress(tx, stages.Headers, blockNum)
- if err != nil {
- return err
- }
- err = stages.SaveStageProgress(tx, stages.Bodies, blockNum)
- if err != nil {
- return err
- }
- err = stages.SaveStageProgress(tx, stages.BlockHashes, blockNum)
- if err != nil {
- return err
- }
- err = stages.SaveStageProgress(tx, stages.Senders, blockNum)
- if err != nil {
- return err
- }
- err = stages.SaveStageProgress(tx, stages.Execution, blockNum)
- if err != nil {
- return err
- }
- return tx.Commit()
-}
-
-func generateHeaderHashToNumberIndex(ctx context.Context, tx kv.RwTx) error {
- c, err := tx.Cursor(kv.Headers)
- if err != nil {
- return err
- }
- log.Info("Generate headers hash to number index")
- lastHeader, _, innerErr := c.Last()
- if innerErr != nil {
- return innerErr
- }
- c.Close()
-
- headNumberBytes := lastHeader[:8]
- headHashBytes := lastHeader[8:]
-
- headNumber := big.NewInt(0).SetBytes(headNumberBytes).Uint64()
- headHash := common.BytesToHash(headHashBytes)
-
- return etl.Transform("Torrent post-processing 1", tx, kv.Headers, kv.HeaderNumber, os.TempDir(), func(k []byte, v []byte, next etl.ExtractNextFunc) error {
- return next(k, common.CopyBytes(k[8:]), common.CopyBytes(k[:8]))
- }, etl.IdentityLoadFunc, etl.TransformArgs{
- Quit: ctx.Done(),
- ExtractEndKey: dbutils.HeaderKey(headNumber, headHash),
- })
-}
-
-func generateHeaderTDAndCanonicalIndexes(ctx context.Context, tx kv.RwTx) error {
- var hash common.Hash
- var number uint64
- var err error
-
- h := rawdb.ReadHeaderByNumber(tx, 0)
- td := h.Difficulty
-
- log.Info("Generate TD index & canonical")
- err = etl.Transform("Torrent post-processing 2", tx, kv.Headers, kv.HeaderTD, os.TempDir(), func(k []byte, v []byte, next etl.ExtractNextFunc) error {
- header := &types.Header{}
- innerErr := rlp.DecodeBytes(v, header)
- if innerErr != nil {
- return innerErr
- }
- number = header.Number.Uint64()
- hash = header.Hash()
- td = td.Add(td, header.Difficulty)
- tdBytes, innerErr := rlp.EncodeToBytes(td)
- if innerErr != nil {
- return innerErr
- }
-
- return next(k, dbutils.HeaderKey(header.Number.Uint64(), header.Hash()), tdBytes)
- }, etl.IdentityLoadFunc, etl.TransformArgs{
- Quit: ctx.Done(),
- })
- if err != nil {
- return err
- }
- log.Info("Generate TD index & canonical")
- err = etl.Transform("Torrent post-processing 2", tx, kv.Headers, kv.HeaderCanonical, os.TempDir(), func(k []byte, v []byte, next etl.ExtractNextFunc) error {
- return next(k, common.CopyBytes(k[:8]), common.CopyBytes(k[8:]))
- }, etl.IdentityLoadFunc, etl.TransformArgs{
- Quit: ctx.Done(),
- })
- if err != nil {
- return err
- }
- rawdb.WriteHeadHeaderHash(tx, hash)
- rawdb.WriteHeaderNumber(tx, hash, number)
- err = stages.SaveStageProgress(tx, stages.Headers, number)
- if err != nil {
- return err
- }
- err = stages.SaveStageProgress(tx, stages.BlockHashes, number)
- if err != nil {
- return err
- }
- rawdb.WriteHeadBlockHash(tx, hash)
- log.Info("Last processed block", "num", number, "hash", hash.String())
- return nil
-}
-
-func GenerateHeaderIndexes(ctx context.Context, tx kv.RwTx) error {
- v, err1 := stages.GetStageProgress(tx, HeadersPostProcessingStage)
- if err1 != nil {
- return err1
- }
-
- if v == 0 {
- if err := generateHeaderHashToNumberIndex(ctx, tx); err != nil {
- return err
- }
- if err := generateHeaderTDAndCanonicalIndexes(ctx, tx); err != nil {
- return err
- }
- if err := stages.SaveStageProgress(tx, HeadersPostProcessingStage, 1); err != nil {
- return err1
- }
-
- return nil
- }
- return nil
-}
diff --git a/turbo/snapshotsync/postprocessing_test.go b/turbo/snapshotsync/postprocessing_test.go
deleted file mode 100644
index 2227323030..0000000000
--- a/turbo/snapshotsync/postprocessing_test.go
+++ /dev/null
@@ -1,107 +0,0 @@
-package snapshotsync
-
-import (
- "context"
- "math/big"
- "os"
- "testing"
-
- "github.com/ledgerwatch/erigon-lib/kv"
- mdbx2 "github.com/ledgerwatch/erigon-lib/kv/mdbx"
- "github.com/ledgerwatch/erigon/common/dbutils"
- "github.com/ledgerwatch/erigon/core/rawdb"
- "github.com/ledgerwatch/erigon/core/types"
- "github.com/ledgerwatch/erigon/ethdb/snapshotdb"
- "github.com/ledgerwatch/erigon/rlp"
- "github.com/ledgerwatch/log/v3"
- "github.com/stretchr/testify/require"
- "github.com/torquem-ch/mdbx-go/mdbx"
-)
-
-func TestHeadersGenerateIndex(t *testing.T) {
- snPath := t.TempDir()
- snKV := mdbx2.NewMDBX(log.New()).Path(snPath).MustOpen()
- defer os.RemoveAll(snPath)
- headers := generateHeaders(10)
- err := snKV.Update(context.Background(), func(tx kv.RwTx) error {
- for _, header := range headers {
- headerBytes, innerErr := rlp.EncodeToBytes(header)
- if innerErr != nil {
- panic(innerErr)
- }
- innerErr = tx.Put(kv.Headers, dbutils.HeaderKey(header.Number.Uint64(), header.Hash()), headerBytes)
- if innerErr != nil {
- panic(innerErr)
- }
- }
- return nil
- })
- if err != nil {
- t.Fatal(err)
- }
- snKV.Close()
-
- db := mdbx2.NewMDBX(log.New()).InMem().WithTablessCfg(mdbx2.WithChaindataTables).MustOpen()
- defer db.Close()
- //we need genesis
- if err := db.Update(context.Background(), func(tx kv.RwTx) error {
- return rawdb.WriteCanonicalHash(tx, headers[0].Hash(), headers[0].Number.Uint64())
-
- }); err != nil {
- t.Fatal(err)
- }
-
- snKV = mdbx2.NewMDBX(log.New()).Path(snPath).Flags(func(flags uint) uint { return flags | mdbx.Readonly }).WithTablessCfg(mdbx2.WithChaindataTables).MustOpen()
- defer snKV.Close()
-
- snKV = snapshotdb.NewSnapshotKV().HeadersSnapshot(snKV).DB(db).Open()
- snTx, err := snKV.BeginRw(context.Background())
- require.NoError(t, err)
- defer snTx.Rollback()
-
- err = GenerateHeaderIndexes(context.Background(), snTx)
- if err != nil {
- t.Fatal(err)
- }
- td := big.NewInt(0)
- for i, header := range headers {
- td = td.Add(td, header.Difficulty)
- canonical, err1 := rawdb.ReadCanonicalHash(snTx, header.Number.Uint64())
- if err1 != nil {
- t.Errorf("reading canonical hash for block %d: %v", header.Number.Uint64(), err1)
- }
- if canonical != header.Hash() {
- t.Error(i, "canonical not correct", canonical)
- }
-
- hasHeader := rawdb.HasHeader(snTx, header.Hash(), header.Number.Uint64())
- if !hasHeader {
- t.Error(i, header.Hash(), header.Number.Uint64(), "not exists")
- }
- headerNumber := rawdb.ReadHeaderNumber(snTx, header.Hash())
- if headerNumber == nil {
- t.Error(i, "empty header number")
- } else if *headerNumber != header.Number.Uint64() {
- t.Error(i, header.Hash(), header.Number.Uint64(), "header number incorrect")
- }
- if td == nil {
- t.Error(i, "empty td")
- } else {
- td, err := rawdb.ReadTd(snTx, header.Hash(), header.Number.Uint64())
- if err != nil {
- panic(err)
- }
- if td.Cmp(td) != 0 {
- t.Error(i, header.Hash(), header.Number.Uint64(), "td incorrect")
- }
- }
- }
-}
-
-func generateHeaders(n int) []types.Header {
- headers := make([]types.Header, n)
- for i := uint64(0); i < uint64(n); i++ {
- headers[i] = types.Header{Difficulty: new(big.Int).SetUint64(i), Number: new(big.Int).SetUint64(i)}
- }
- return headers
-}
diff --git a/turbo/snapshotsync/server.go b/turbo/snapshotsync/server.go
deleted file mode 100644
index 857249ed7b..0000000000
--- a/turbo/snapshotsync/server.go
+++ /dev/null
@@ -1,220 +0,0 @@
-package snapshotsync
-
-import (
- "context"
- "errors"
- "fmt"
- "time"
-
- "github.com/anacrolix/torrent"
- "github.com/ledgerwatch/erigon-lib/gointerfaces/snapshotsync"
- "github.com/ledgerwatch/erigon-lib/kv"
- "github.com/ledgerwatch/erigon-lib/kv/mdbx"
- "github.com/ledgerwatch/log/v3"
- "golang.org/x/sync/errgroup"
- "google.golang.org/protobuf/types/known/emptypb"
-)
-
-var (
- ErrNotSupportedNetworkID = errors.New("not supported network id")
- ErrNotSupportedSnapshot = errors.New("not supported snapshot for this network id")
-)
-var (
- _ snapshotsync.DownloaderServer = &SNDownloaderServer{}
-)
-
-func NewServer(dir string, seeding bool) (*SNDownloaderServer, error) {
- db := mdbx.MustOpen(dir + "/db")
- sn := &SNDownloaderServer{
- db: db,
- }
- if err := db.Update(context.Background(), func(tx kv.RwTx) error {
- peerID, err := tx.GetOne(kv.BittorrentInfo, []byte(kv.BittorrentPeerID))
- if err != nil {
- return fmt.Errorf("get peer id: %w", err)
- }
- sn.t, err = New(dir, seeding, string(peerID))
- if err != nil {
- return err
- }
- if len(peerID) == 0 {
- err = sn.t.SavePeerID(tx)
- if err != nil {
- return fmt.Errorf("save peer id: %w", err)
- }
- }
- return nil
- }); err != nil {
- return nil, err
- }
-
- return sn, nil
-}
-
-type SNDownloaderServer struct {
- snapshotsync.UnimplementedDownloaderServer
- t *Client
- db kv.RwDB
-}
-
-func (s *SNDownloaderServer) Download(ctx context.Context, request *snapshotsync.DownloadSnapshotRequest) (*emptypb.Empty, error) {
- ctx, cancel := context.WithTimeout(ctx, time.Minute*10)
- defer cancel()
- eg := errgroup.Group{}
-
- networkId := request.NetworkId
- mode := FromSnapshotTypes(request.Type)
-
- var headerSpec, bodySpec, stateSpec, receiptSpec *torrentSpecFromDb
-
- if err := s.db.View(ctx, func(tx kv.Tx) error {
- var err error
- headerSpec, err = getTorrentSpec(tx, snapshotsync.SnapshotType_headers, networkId)
- if err != nil {
- return err
- }
- bodySpec, err = getTorrentSpec(tx, snapshotsync.SnapshotType_bodies, networkId)
- if err != nil {
- return err
- }
- stateSpec, err = getTorrentSpec(tx, snapshotsync.SnapshotType_state, networkId)
- if err != nil {
- return err
- }
- receiptSpec, err = getTorrentSpec(tx, snapshotsync.SnapshotType_receipts, networkId)
- if err != nil {
- return err
- }
- return nil
- }); err != nil {
- return nil, err
- }
-
- if mode.Headers {
- eg.Go(func() error {
- var newSpec *torrentSpecFromDb
- var err error
- newSpec, err = s.t.AddTorrent(ctx, headerSpec)
- if err != nil {
- return fmt.Errorf("add torrent: %w", err)
- }
- if newSpec != nil {
- log.Info("Save spec", "snapshot", newSpec.snapshotType.String())
- if err = s.db.Update(ctx, func(tx kv.RwTx) error {
- return saveTorrentSpec(tx, newSpec)
- }); err != nil {
- return err
- }
- if err != nil {
- return err
- }
- }
- return nil
- })
- }
-
- if mode.Bodies {
- eg.Go(func() error {
- var newSpec *torrentSpecFromDb
- var err error
- newSpec, err = s.t.AddTorrent(ctx, bodySpec)
- if err != nil {
- return fmt.Errorf("add torrent: %w", err)
- }
- if newSpec != nil {
- log.Info("Save spec", "snapshot", newSpec.snapshotType.String())
- if err = s.db.Update(ctx, func(tx kv.RwTx) error {
- return saveTorrentSpec(tx, newSpec)
- }); err != nil {
- return err
- }
- if err != nil {
- return err
- }
- }
- return nil
- })
- }
-
- if mode.State {
- eg.Go(func() error {
- var newSpec *torrentSpecFromDb
- var err error
- newSpec, err = s.t.AddTorrent(ctx, stateSpec)
- if err != nil {
- return fmt.Errorf("add torrent: %w", err)
- }
- if newSpec != nil {
- log.Info("Save spec", "snapshot", newSpec.snapshotType.String())
- if err = s.db.Update(ctx, func(tx kv.RwTx) error {
- return saveTorrentSpec(tx, newSpec)
- }); err != nil {
- return err
- }
- if err != nil {
- return err
- }
- }
- return nil
- })
- }
-
- if mode.Receipts {
- eg.Go(func() error {
- var newSpec *torrentSpecFromDb
- var err error
- newSpec, err = s.t.AddTorrent(ctx, receiptSpec)
- if err != nil {
- return fmt.Errorf("add torrent: %w", err)
- }
- if newSpec != nil {
- log.Info("Save spec", "snapshot", newSpec.snapshotType.String())
- if err = s.db.Update(ctx, func(tx kv.RwTx) error {
- return saveTorrentSpec(tx, newSpec)
- }); err != nil {
- return err
- }
- if err != nil {
- return err
- }
- }
- return nil
- })
- }
- err := eg.Wait()
- if err != nil {
- return nil, err
- }
- return &emptypb.Empty{}, nil
-}
-func (s *SNDownloaderServer) Load() error {
- return s.db.View(context.Background(), func(tx kv.Tx) error {
- return s.t.Load(tx)
- })
-}
-
-func (s *SNDownloaderServer) Snapshots(ctx context.Context, request *snapshotsync.SnapshotsRequest) (*snapshotsync.SnapshotsInfoReply, error) {
- reply := snapshotsync.SnapshotsInfoReply{}
- tx, err := s.db.BeginRo(ctx)
- if err != nil {
- return nil, err
- }
- defer tx.Rollback()
- resp, err := s.t.GetSnapshots(tx, request.NetworkId)
- if err != nil {
- return nil, err
- }
- for i := range resp {
- reply.Info = append(reply.Info, resp[i])
- }
- return &reply, nil
-}
-
-func (s *SNDownloaderServer) Stats(ctx context.Context) map[string]torrent.TorrentStats {
- stats := map[string]torrent.TorrentStats{}
- torrents := s.t.Cli.Torrents()
- for _, t := range torrents {
- stats[t.Name()] = t.Stats()
- }
- return stats
-}
diff --git a/turbo/snapshotsync/snapshot_mode.go b/turbo/snapshotsync/snapshot_mode.go
index 592b56c38f..f217c87a3c 100644
--- a/turbo/snapshotsync/snapshot_mode.go
+++ b/turbo/snapshotsync/snapshot_mode.go
@@ -1,5 +1,6 @@
package snapshotsync
+/*
import (
"fmt"
@@ -83,3 +84,4 @@ func SnapshotModeFromString(flags string) (SnapshotMode, error) {
}
return mode, nil
}
+*/
diff --git a/turbo/snapshotsync/snapshot_mode_test.go b/turbo/snapshotsync/snapshot_mode_test.go
index 810d4494f4..84d24418eb 100644
--- a/turbo/snapshotsync/snapshot_mode_test.go
+++ b/turbo/snapshotsync/snapshot_mode_test.go
@@ -1,5 +1,6 @@
package snapshotsync
+/*
import (
"reflect"
"testing"
@@ -39,3 +40,4 @@ func TestSnapshotModeFromString(t *testing.T) {
t.Fatal(sm)
}
}
+*/
diff --git a/turbo/snapshotsync/snapshothashes/embed.go b/turbo/snapshotsync/snapshothashes/embed.go
new file mode 100644
index 0000000000..bb0f139f0f
--- /dev/null
+++ b/turbo/snapshotsync/snapshothashes/embed.go
@@ -0,0 +1,49 @@
+package snapshothashes
+
+import (
+ _ "embed"
+ "encoding/json"
+
+ "github.com/ledgerwatch/erigon/params/networkname"
+)
+
+//go:embed erigon-snapshots/mainnet.json
+var mainnet []byte
+var Mainnet = fromJson(mainnet)
+
+//go:embed erigon-snapshots/goerli.json
+var goerli []byte
+var Goerli = fromJson(goerli)
+
+type Preverified map[string]string
+
+func fromJson(in []byte) (out Preverified) {
+ if err := json.Unmarshal(in, &out); err != nil {
+ panic(err)
+ }
+ return out
+}
+
+var (
+ MainnetChainSnapshotConfig = &Config{}
+ GoerliChainSnapshotConfig = &Config{
+ ExpectBlocks: 5_900_000 - 1,
+ Preverified: Goerli,
+ }
+)
+
+type Config struct {
+ ExpectBlocks uint64
+ Preverified Preverified
+}
+
+func KnownConfig(networkName string) *Config {
+ switch networkName {
+ case networkname.MainnetChainName:
+ return MainnetChainSnapshotConfig
+ case networkname.GoerliChainName:
+ return GoerliChainSnapshotConfig
+ default:
+ return nil
+ }
+}
diff --git a/turbo/snapshotsync/snapshothashes/erigon-snapshots b/turbo/snapshotsync/snapshothashes/erigon-snapshots
new file mode 160000
index 0000000000..40713656a9
--- /dev/null
+++ b/turbo/snapshotsync/snapshothashes/erigon-snapshots
@@ -0,0 +1 @@
+Subproject commit 40713656a9f5c60dcb0125cb382d41014d601434
diff --git a/turbo/snapshotsync/wrapdb.go b/turbo/snapshotsync/wrapdb.go
index 5d7b74f5e6..e13442ab0c 100644
--- a/turbo/snapshotsync/wrapdb.go
+++ b/turbo/snapshotsync/wrapdb.go
@@ -1,5 +1,6 @@
package snapshotsync
+/*
import (
"github.com/ledgerwatch/erigon-lib/gointerfaces/snapshotsync"
"github.com/ledgerwatch/erigon-lib/kv"
@@ -56,3 +57,4 @@ func WrapBySnapshotsFromDownloader(db kv.RwDB, snapshots map[snapshotsync.Snapsh
return snKV.Open(), nil
}
+*/
diff --git a/turbo/stages/headerdownload/header_algos.go b/turbo/stages/headerdownload/header_algos.go
index 87ec09d99a..23558b9bd0 100644
--- a/turbo/stages/headerdownload/header_algos.go
+++ b/turbo/stages/headerdownload/header_algos.go
@@ -26,7 +26,7 @@ import (
"github.com/ledgerwatch/erigon/core/types"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/p2p/enode"
- "github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/erigon/params/networkname"
"github.com/ledgerwatch/erigon/rlp"
"github.com/ledgerwatch/log/v3"
)
@@ -439,10 +439,10 @@ func InitPreverifiedHashes(chain string) (map[common.Hash]struct{}, uint64) {
var encodings []string
var height uint64
switch chain {
- case params.MainnetChainName:
+ case networkname.MainnetChainName:
encodings = mainnetPreverifiedHashes
height = mainnetPreverifiedHeight
- case params.RopstenChainName:
+ case networkname.RopstenChainName:
encodings = ropstenPreverifiedHashes
height = ropstenPreverifiedHeight
default:
diff --git a/turbo/stages/mock_sentry.go b/turbo/stages/mock_sentry.go
index d2bad02470..780825536d 100644
--- a/turbo/stages/mock_sentry.go
+++ b/turbo/stages/mock_sentry.go
@@ -13,6 +13,7 @@ import (
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/direct"
"github.com/ledgerwatch/erigon-lib/gointerfaces"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
proto_sentry "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry"
ptypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types"
"github.com/ledgerwatch/erigon-lib/kv"
@@ -20,7 +21,7 @@ import (
"github.com/ledgerwatch/erigon-lib/kv/memdb"
"github.com/ledgerwatch/erigon-lib/kv/remotedbserver"
"github.com/ledgerwatch/erigon-lib/txpool"
- "github.com/ledgerwatch/erigon/cmd/sentry/download"
+ "github.com/ledgerwatch/erigon/cmd/sentry/sentry"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/consensus"
"github.com/ledgerwatch/erigon/consensus/ethash"
@@ -60,7 +61,7 @@ type MockSentry struct {
MiningSync *stagedsync.Sync
PendingBlocks chan *types.Block
MinedBlocks chan *types.Block
- downloader *download.ControlServerImpl
+ downloader *sentry.ControlServerImpl
Key *ecdsa.PrivateKey
Genesis *types.Block
SentryClient direct.SentryClient
@@ -260,7 +261,7 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
blockDownloaderWindow := 65536
networkID := uint64(1)
- mock.downloader, err = download.NewControlServer(mock.DB, "mock", mock.ChainConfig, mock.Genesis.Hash(), mock.Engine, networkID, sentries, blockDownloaderWindow)
+ mock.downloader, err = sentry.NewControlServer(mock.DB, "mock", mock.ChainConfig, mock.Genesis.Hash(), mock.Engine, networkID, sentries, blockDownloaderWindow)
if err != nil {
if t != nil {
t.Fatal(err)
@@ -271,6 +272,7 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
blockReader := snapshotsync.NewBlockReader()
var allSnapshots *snapshotsync.AllSnapshots
+ var snapshotsDownloader proto_downloader.DownloaderClient
mock.Sync = stagedsync.New(
stagedsync.DefaultStages(mock.Ctx, prune, stagedsync.StageHeadersCfg(
mock.DB,
@@ -285,6 +287,7 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
nil,
nil,
allSnapshots,
+ snapshotsDownloader,
blockReader,
mock.tmpdir,
), stagedsync.StageBlockHashesCfg(mock.DB, mock.tmpdir, mock.ChainConfig), stagedsync.StageBodiesCfg(
@@ -344,13 +347,13 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
)
mock.StreamWg.Add(1)
- go download.RecvMessageLoop(mock.Ctx, mock.SentryClient, mock.downloader, &mock.ReceiveWg)
+ go sentry.RecvMessageLoop(mock.Ctx, mock.SentryClient, mock.downloader, &mock.ReceiveWg)
mock.StreamWg.Wait()
mock.StreamWg.Add(1)
- go download.RecvUploadMessageLoop(mock.Ctx, mock.SentryClient, mock.downloader, &mock.ReceiveWg)
+ go sentry.RecvUploadMessageLoop(mock.Ctx, mock.SentryClient, mock.downloader, &mock.ReceiveWg)
mock.StreamWg.Wait()
mock.StreamWg.Add(1)
- go download.RecvUploadHeadersMessageLoop(mock.Ctx, mock.SentryClient, mock.downloader, &mock.ReceiveWg)
+ go sentry.RecvUploadHeadersMessageLoop(mock.Ctx, mock.SentryClient, mock.downloader, &mock.ReceiveWg)
mock.StreamWg.Wait()
return mock
diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go
index 47122e46b9..45e78f2ea4 100644
--- a/turbo/stages/stageloop.go
+++ b/turbo/stages/stageloop.go
@@ -10,9 +10,10 @@ import (
"github.com/holiman/uint256"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/dbg"
+ proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/cmd/rpcdaemon/interfaces"
- "github.com/ledgerwatch/erigon/cmd/sentry/download"
+ "github.com/ledgerwatch/erigon/cmd/sentry/sentry"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/consensus/misc"
"github.com/ledgerwatch/erigon/core/rawdb"
@@ -23,9 +24,9 @@ import (
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb/privateapi"
"github.com/ledgerwatch/erigon/p2p"
- "github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/turbo/shards"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
+ "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/erigon/turbo/stages/headerdownload"
"github.com/ledgerwatch/log/v3"
)
@@ -224,17 +225,18 @@ func NewStagedSync(
p2pCfg p2p.Config,
cfg ethconfig.Config,
terminalTotalDifficulty *big.Int,
- controlServer *download.ControlServerImpl,
+ controlServer *sentry.ControlServerImpl,
tmpdir string,
accumulator *shards.Accumulator,
reverseDownloadCh chan types.Header,
statusCh chan privateapi.ExecutionStatus,
waitingForPOSHeaders *bool,
+ snapshotDownloader proto_downloader.DownloaderClient,
) (*stagedsync.Sync, error) {
var blockReader interfaces.FullBlockReader
var allSnapshots *snapshotsync.AllSnapshots
if cfg.Snapshot.Enabled {
- allSnapshots = snapshotsync.NewAllSnapshots(cfg.Snapshot.Dir, params.KnownSnapshots(controlServer.ChainConfig.ChainName))
+ allSnapshots = snapshotsync.NewAllSnapshots(cfg.Snapshot.Dir, snapshothashes.KnownConfig(controlServer.ChainConfig.ChainName))
blockReader = snapshotsync.NewBlockReaderWithSnapshots(allSnapshots)
} else {
blockReader = snapshotsync.NewBlockReader()
@@ -254,6 +256,7 @@ func NewStagedSync(
reverseDownloadCh,
waitingForPOSHeaders,
allSnapshots,
+ snapshotDownloader,
blockReader,
tmpdir,
), stagedsync.StageBlockHashesCfg(db, tmpdir, controlServer.ChainConfig), stagedsync.StageBodiesCfg(
--
GitLab