From f5a8230016a1adad67f1ed6e9ee277dcba8d6451 Mon Sep 17 00:00:00 2001
From: Alex Sharov <AskAlexSharov@gmail.com>
Date: Mon, 28 Mar 2022 14:44:11 +0700
Subject: [PATCH] Snapshots: write hashes to file only if amount of them growth
(#3784)
---
cmd/downloader/main.go | 45 ++++++++++++++++---------
cmd/downloader/torrent_hashes_update.sh | 2 +-
2 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go
index f459d6b9ea..91c6a01c20 100644
--- a/cmd/downloader/main.go
+++ b/cmd/downloader/main.go
@@ -2,8 +2,8 @@ package main
import (
"context"
- "encoding/json"
"fmt"
+ "io/ioutil"
"net"
"os"
"path/filepath"
@@ -35,13 +35,13 @@ import (
var (
datadir string
- asJson bool
forceRebuild bool
forceVerify bool
downloaderApiAddr string
torrentVerbosity string
downloadRateStr, uploadRateStr string
torrentPort int
+ targetFile string
)
func init() {
@@ -57,9 +57,12 @@ func init() {
rootCmd.Flags().IntVar(&torrentPort, "torrent.port", 42069, "port to listen and serve BitTorrent protocol")
withDataDir(printTorrentHashes)
- printTorrentHashes.PersistentFlags().BoolVar(&asJson, "json", false, "Print in json format (default: toml)")
printTorrentHashes.PersistentFlags().BoolVar(&forceRebuild, "rebuild", false, "Force re-create .torrent files")
printTorrentHashes.PersistentFlags().BoolVar(&forceVerify, "verify", false, "Force verify data files if have .torrent files")
+ printTorrentHashes.Flags().StringVar(&targetFile, "targetfile", "", "write output to file")
+ if err := printTorrentHashes.MarkFlagFilename("targetfile"); err != nil {
+ panic(err)
+ }
rootCmd.AddCommand(printTorrentHashes)
}
@@ -205,19 +208,31 @@ var printTorrentHashes = &cobra.Command{
}
res[info.Name] = mi.HashInfoBytes().String()
}
- var serialized []byte
- if asJson {
- serialized, err = json.Marshal(res)
- if err != nil {
- return err
- }
- } else {
- serialized, err = toml.Marshal(res)
- if err != nil {
- return err
- }
+ serialized, err := toml.Marshal(res)
+ if err != nil {
+ return err
+ }
+
+ if targetFile == "" {
+ fmt.Printf("%s\n", serialized)
+ return nil
+ }
+
+ oldContent, err := ioutil.ReadFile(targetFile)
+ if err != nil {
+ return err
+ }
+ oldLines := map[string]string{}
+ if err := toml.Unmarshal(oldContent, &oldLines); err != nil {
+ return fmt.Errorf("unmarshal: %w", err)
+ }
+ if len(oldLines) >= len(res) {
+ log.Info("amount of lines in target file is equal or greater than amount of lines in snapshot dir", "old", len(oldLines), "new", len(res))
+ return nil
+ }
+ if err := ioutil.WriteFile(targetFile, serialized, 0644); err != nil {
+ return err
}
- fmt.Printf("%s\n", serialized)
return nil
},
}
diff --git a/cmd/downloader/torrent_hashes_update.sh b/cmd/downloader/torrent_hashes_update.sh
index 738a9b30e0..1e309542f2 100755
--- a/cmd/downloader/torrent_hashes_update.sh
+++ b/cmd/downloader/torrent_hashes_update.sh
@@ -14,7 +14,7 @@ git pull
cd ../erigon
# it will return only .seg of 500K (because Erigon send to Downloader only such files)
-go run -trimpath ./cmd/downloader torrent_hashes --datadir="$datadir" >./../erigon-snapshot/"$network".toml
+go run -trimpath ./cmd/downloader torrent_hashes --datadir="$datadir" --targetfile=./../erigon-snapshot/"$network".toml
cd ./../erigon-snapshot
git add "$network".toml
git commit -m "ci: $network"
--
GitLab