From 586ad297e6bd2c0b0cda569954ae1dbcd91ae499 Mon Sep 17 00:00:00 2001
From: Alex Sharov <AskAlexSharov@gmail.com>
Date: Mon, 21 Mar 2022 11:22:37 +0700
Subject: [PATCH] Snapshots: script to auto-commit torrent hashes (#3745)
---
Makefile | 3 +++
cmd/downloader/torrent_hashes_update.sh | 23 +++++++++++++++++++++++
go.mod | 2 +-
go.sum | 4 ++--
turbo/snapshotsync/block_snapshots.go | 25 +++++--------------------
5 files changed, 34 insertions(+), 23 deletions(-)
create mode 100644 cmd/downloader/torrent_hashes_update.sh
diff --git a/Makefile b/Makefile
index a2cf9828cc..36fbbae810 100644
--- a/Makefile
+++ b/Makefile
@@ -85,6 +85,9 @@ db-tools:
cp libmdbx/mdbx_stat $(GOBIN)
@echo "Run \"$(GOBIN)/mdbx_stat -h\" to get info about mdbx db file."
+torrent_hashes_update:
+ ./cmd/downloader/auto_update.sh mainnet
+
test:
$(GOTEST) --timeout 30m
diff --git a/cmd/downloader/torrent_hashes_update.sh b/cmd/downloader/torrent_hashes_update.sh
new file mode 100644
index 0000000000..1b859cc1d6
--- /dev/null
+++ b/cmd/downloader/torrent_hashes_update.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+network=$1
+datadir=$2
+
+git reset --hard
+git checkout devel
+git pull
+# it will return only .seg of 500K (because Erigon send to Downloader only such files)
+go run -trimpath ./cmd/downloader torrent_hashes --datadir="$datadir" >./../erigon-snpshots/"$network".toml
+cd ./../erigon-snapshots
+git add "$network".toml
+git commit -m "cd: $network"
+git push
+# update Erigon submodule
+cd ./../erigon
+cd turbo/snapshotsync/snapshothashes/erigon-snapshots
+git checkout main
+git pull
+cd ../../../../
+git add turbo/snapshotsync/snapshothashes/erigon-snapshots
+git commit -m "cd: $network snapshots"
+git push
diff --git a/go.mod b/go.mod
index ed4548f0a6..afb146c54b 100644
--- a/go.mod
+++ b/go.mod
@@ -40,7 +40,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
- github.com/ledgerwatch/erigon-lib v0.0.0-20220320161426-7939c56571cc
+ github.com/ledgerwatch/erigon-lib v0.0.0-20220321040948-b3cbcf709805
github.com/ledgerwatch/log/v3 v3.4.1
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/magiconair/properties v1.8.6 // indirect
diff --git a/go.sum b/go.sum
index 54d41b0afe..7b1a1f2b96 100644
--- a/go.sum
+++ b/go.sum
@@ -615,8 +615,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220320161426-7939c56571cc h1:kXIBqOMownH+1eUNqrwVLtcPmfREwPBCYwwsMCNF/s0=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220320161426-7939c56571cc/go.mod h1:1Ao0yXNx7956h1fm/3A8AOB2pzckvlpBvxdYvJNfgl8=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220321040948-b3cbcf709805 h1:U0M5S3dFFdSm3BHfXdrvxmIFRXmAdbx9oRc6UZN05NQ=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220321040948-b3cbcf709805/go.mod h1:1Ao0yXNx7956h1fm/3A8AOB2pzckvlpBvxdYvJNfgl8=
github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc=
github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
diff --git a/turbo/snapshotsync/block_snapshots.go b/turbo/snapshotsync/block_snapshots.go
index f7560d7bea..b07f5a7410 100644
--- a/turbo/snapshotsync/block_snapshots.go
+++ b/turbo/snapshotsync/block_snapshots.go
@@ -675,22 +675,6 @@ func latestSegment(dir string, ofType Type) (uint64, error) {
}
return maxBlock - 1, nil
}
-func latestIdx(dir string, ofType Type) (uint64, error) {
- files, err := idxFilesOfType(dir, ofType)
- if err != nil {
- return 0, err
- }
- var maxBlock uint64
- for _, f := range files {
- if maxBlock < f.To {
- maxBlock = f.To
- }
- }
- if maxBlock == 0 {
- return 0, nil
- }
- return maxBlock - 1, nil
-}
// FileInfo - parsed file metadata
type FileInfo struct {
@@ -813,6 +797,7 @@ func segmentsOfType(dir string, ofType Type) (res []FileInfo, err error) {
return noGaps(noOverlaps(res))
}
+// nolint
func idxFilesOfType(dir string, ofType Type) (res []FileInfo, err error) {
files, err := IdxFiles(dir)
if err != nil {
@@ -1385,12 +1370,11 @@ func TransactionsHashIdx(ctx context.Context, chainID uint256.Int, txsSegment *T
if err != nil {
return err
}
+ txnHashIdx.LogLvl(log.LvlDebug)
+ txnIdIdx.LogLvl(log.LvlDebug)
+ txnHash2BlockNumIdx.LogLvl(log.LvlDebug)
RETRY:
- txnHashIdx.NoLogs(true)
- txnIdIdx.NoLogs(true)
- txnHash2BlockNumIdx.NoLogs(true)
-
ch := forEachAsync(ctx, d)
type txHashWithOffet struct {
txnHash [32]byte
@@ -1675,6 +1659,7 @@ func Idx(ctx context.Context, d *compress.Decompressor, firstDataID uint64, tmpD
if err != nil {
return err
}
+ rs.LogLvl(log.LvlDebug)
RETRY:
ch := forEachAsync(ctx, d)
--
GitLab