From 92db317e0658824df0db015f29728ce08a5e7880 Mon Sep 17 00:00:00 2001
From: battlmonstr <battlmonstr@users.noreply.github.com>
Date: Fri, 22 Apr 2022 13:02:12 +0200
Subject: [PATCH] Observer - P2P network crawler (#3928)
Observer crawls the Ethereum network and collects information about the nodes.
---
Makefile | 1 +
cmd/observer/README.md | 35 +
cmd/observer/database/db.go | 73 ++
cmd/observer/database/db_retrier.go | 240 +++++
cmd/observer/database/db_sqlite.go | 866 ++++++++++++++++++
cmd/observer/database/db_sqlite_test.go | 40 +
cmd/observer/main.go | 108 +++
cmd/observer/observer/client_id.go | 133 +++
cmd/observer/observer/command.go | 233 +++++
cmd/observer/observer/crawler.go | 495 ++++++++++
cmd/observer/observer/diplomacy.go | 252 +++++
cmd/observer/observer/diplomat.go | 150 +++
cmd/observer/observer/handshake.go | 252 +++++
cmd/observer/observer/handshake_test.go | 35 +
cmd/observer/observer/interrogation_error.go | 53 ++
cmd/observer/observer/interrogator.go | 230 +++++
cmd/observer/observer/keygen.go | 76 ++
cmd/observer/observer/keygen_test.go | 23 +
cmd/observer/observer/node.go | 145 +++
cmd/observer/observer/server.go | 183 ++++
cmd/observer/observer/status_logger.go | 44 +
.../reports/clients_estimate_report.go | 95 ++
cmd/observer/reports/clients_report.go | 97 ++
cmd/observer/reports/command.go | 92 ++
cmd/observer/reports/status_report.go | 40 +
cmd/observer/utils/retry.go | 32 +
cmd/observer/utils/sleep.go | 15 +
cmd/observer/utils/task_queue.go | 51 ++
eth/protocols/eth/discovery.go | 13 +
go.mod | 12 +
go.sum | 120 +++
p2p/discover/v4_udp.go | 10 +-
32 files changed, 4242 insertions(+), 2 deletions(-)
create mode 100644 cmd/observer/README.md
create mode 100644 cmd/observer/database/db.go
create mode 100644 cmd/observer/database/db_retrier.go
create mode 100644 cmd/observer/database/db_sqlite.go
create mode 100644 cmd/observer/database/db_sqlite_test.go
create mode 100644 cmd/observer/main.go
create mode 100644 cmd/observer/observer/client_id.go
create mode 100644 cmd/observer/observer/command.go
create mode 100644 cmd/observer/observer/crawler.go
create mode 100644 cmd/observer/observer/diplomacy.go
create mode 100644 cmd/observer/observer/diplomat.go
create mode 100644 cmd/observer/observer/handshake.go
create mode 100644 cmd/observer/observer/handshake_test.go
create mode 100644 cmd/observer/observer/interrogation_error.go
create mode 100644 cmd/observer/observer/interrogator.go
create mode 100644 cmd/observer/observer/keygen.go
create mode 100644 cmd/observer/observer/keygen_test.go
create mode 100644 cmd/observer/observer/node.go
create mode 100644 cmd/observer/observer/server.go
create mode 100644 cmd/observer/observer/status_logger.go
create mode 100644 cmd/observer/reports/clients_estimate_report.go
create mode 100644 cmd/observer/reports/clients_report.go
create mode 100644 cmd/observer/reports/command.go
create mode 100644 cmd/observer/reports/status_report.go
create mode 100644 cmd/observer/utils/retry.go
create mode 100644 cmd/observer/utils/sleep.go
create mode 100644 cmd/observer/utils/task_queue.go
diff --git a/Makefile b/Makefile
index b0fab7bdf5..4845587a9c 100644
--- a/Makefile
+++ b/Makefile
@@ -61,6 +61,7 @@ COMMANDS += downloader
COMMANDS += evm
COMMANDS += hack
COMMANDS += integration
+COMMANDS += observer
COMMANDS += pics
COMMANDS += rpcdaemon
COMMANDS += rpctest
diff --git a/cmd/observer/README.md b/cmd/observer/README.md
new file mode 100644
index 0000000000..d094f06b3b
--- /dev/null
+++ b/cmd/observer/README.md
@@ -0,0 +1,35 @@
+# Observer - P2P network crawler
+
+Observer crawls the Ethereum network and collects information about the nodes.
+
+### Build
+
+ make observer
+
+### Run
+
+ observer --datadir ... --nat extip:<IP> --port <PORT>
+
+Where `IP` is your public IP, and `PORT` has to be open for incoming UDP traffic.
+
+See `observer --help` for available options.
+
+### Report
+
+To get the report about the currently known network state run:
+
+ observer report --datadir ...
+
+## Description
+
+Observer uses [discv4](https://github.com/ethereum/devp2p/blob/master/discv4.md) protocol to discover new nodes.
+Starting from a list of preconfigured "bootnodes" it uses FindNode
+to obtain their "neighbor" nodes, and then recursively crawls neighbors of neighbors and so on.
+Each found node is re-crawled again a few times.
+If the node fails to be pinged after maximum attempts, it is considered "dead", but still re-crawled less often.
+
+A separate "diplomacy" process is doing "handshakes" to obtain information about the discovered nodes.
+It tries to get [RLPx Hello](https://github.com/ethereum/devp2p/blob/master/rlpx.md#hello-0x00)
+and [Eth Status](https://github.com/ethereum/devp2p/blob/master/caps/eth.md#status-0x00)
+from each node.
+The handshake repeats a few times according to the configured delays.
diff --git a/cmd/observer/database/db.go b/cmd/observer/database/db.go
new file mode 100644
index 0000000000..a647606f77
--- /dev/null
+++ b/cmd/observer/database/db.go
@@ -0,0 +1,73 @@
+package database
+
+import (
+ "context"
+ "io"
+ "net"
+ "time"
+)
+
+type NodeID string
+
+type NodeAddr1 struct {
+ IP net.IP
+ PortDisc uint16
+ PortRLPx uint16
+}
+
+type NodeAddr struct {
+ NodeAddr1
+ IPv6 NodeAddr1
+}
+
+type HandshakeError struct {
+ StringCode string
+ Time time.Time
+}
+
+type DB interface {
+ io.Closer
+
+ UpsertNodeAddr(ctx context.Context, id NodeID, addr NodeAddr) error
+ FindNodeAddr(ctx context.Context, id NodeID) (*NodeAddr, error)
+
+ ResetPingError(ctx context.Context, id NodeID) error
+ UpdatePingError(ctx context.Context, id NodeID) error
+ CountPingErrors(ctx context.Context, id NodeID) (*uint, error)
+
+ UpdateClientID(ctx context.Context, id NodeID, clientID string) error
+ UpdateNetworkID(ctx context.Context, id NodeID, networkID uint) error
+ UpdateEthVersion(ctx context.Context, id NodeID, ethVersion uint) error
+ UpdateHandshakeTransientError(ctx context.Context, id NodeID, hasTransientErr bool) error
+ InsertHandshakeError(ctx context.Context, id NodeID, handshakeErr string) error
+ DeleteHandshakeErrors(ctx context.Context, id NodeID) error
+ FindHandshakeLastErrors(ctx context.Context, id NodeID, limit uint) ([]HandshakeError, error)
+ UpdateHandshakeRetryTime(ctx context.Context, id NodeID, retryTime time.Time) error
+ FindHandshakeRetryTime(ctx context.Context, id NodeID) (*time.Time, error)
+ CountHandshakeCandidates(ctx context.Context) (uint, error)
+ FindHandshakeCandidates(ctx context.Context, limit uint) ([]NodeID, error)
+ MarkTakenHandshakeCandidates(ctx context.Context, nodes []NodeID) error
+ // TakeHandshakeCandidates runs FindHandshakeCandidates + MarkTakenHandshakeCandidates in a transaction.
+ TakeHandshakeCandidates(ctx context.Context, limit uint) ([]NodeID, error)
+
+ UpdateForkCompatibility(ctx context.Context, id NodeID, isCompatFork bool) error
+
+ UpdateNeighborBucketKeys(ctx context.Context, id NodeID, keys []string) error
+ FindNeighborBucketKeys(ctx context.Context, id NodeID) ([]string, error)
+
+ UpdateCrawlRetryTime(ctx context.Context, id NodeID, retryTime time.Time) error
+ CountCandidates(ctx context.Context) (uint, error)
+ FindCandidates(ctx context.Context, limit uint) ([]NodeID, error)
+ MarkTakenNodes(ctx context.Context, nodes []NodeID) error
+ // TakeCandidates runs FindCandidates + MarkTakenNodes in a transaction.
+ TakeCandidates(ctx context.Context, limit uint) ([]NodeID, error)
+
+ IsConflictError(err error) bool
+
+ CountNodes(ctx context.Context, maxPingTries uint, networkID uint) (uint, error)
+ CountIPs(ctx context.Context, maxPingTries uint, networkID uint) (uint, error)
+ CountClients(ctx context.Context, clientIDPrefix string, maxPingTries uint, networkID uint) (uint, error)
+ CountClientsWithNetworkID(ctx context.Context, clientIDPrefix string, maxPingTries uint) (uint, error)
+ CountClientsWithHandshakeTransientError(ctx context.Context, clientIDPrefix string, maxPingTries uint) (uint, error)
+ EnumerateClientIDs(ctx context.Context, maxPingTries uint, networkID uint, enumFunc func(clientID *string)) error
+}
diff --git a/cmd/observer/database/db_retrier.go b/cmd/observer/database/db_retrier.go
new file mode 100644
index 0000000000..87db46d68f
--- /dev/null
+++ b/cmd/observer/database/db_retrier.go
@@ -0,0 +1,240 @@
+package database
+
+import (
+ "context"
+ "github.com/ledgerwatch/erigon/cmd/observer/utils"
+ "github.com/ledgerwatch/log/v3"
+ "math/rand"
+ "time"
+)
+
+type DBRetrier struct {
+ db DB
+ log log.Logger
+}
+
+func NewDBRetrier(db DB, logger log.Logger) DBRetrier {
+ return DBRetrier{db, logger}
+}
+
+func retryBackoffTime(attempt int) time.Duration {
+ if attempt <= 0 {
+ return 0
+ }
+ jitter := rand.Int63n(30 * time.Millisecond.Nanoseconds() * int64(attempt))
+ var ns int64
+ if attempt <= 6 {
+ ns = ((50 * time.Millisecond.Nanoseconds()) << (attempt - 1)) + jitter
+ } else {
+ ns = 1600*time.Millisecond.Nanoseconds() + jitter
+ }
+ return time.Duration(ns)
+}
+
+func (db DBRetrier) retry(ctx context.Context, opName string, op func(context.Context) (interface{}, error)) (interface{}, error) {
+ const retryCount = 40
+ return utils.Retry(ctx, retryCount, retryBackoffTime, db.db.IsConflictError, db.log, opName, op)
+}
+
+func (db DBRetrier) UpsertNodeAddr(ctx context.Context, id NodeID, addr NodeAddr) error {
+ _, err := db.retry(ctx, "UpsertNodeAddr", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpsertNodeAddr(ctx, id, addr)
+ })
+ return err
+}
+
+func (db DBRetrier) FindNodeAddr(ctx context.Context, id NodeID) (*NodeAddr, error) {
+ resultAny, err := db.retry(ctx, "FindNodeAddr", func(ctx context.Context) (interface{}, error) {
+ return db.db.FindNodeAddr(ctx, id)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.(*NodeAddr)
+ return result, err
+}
+
+func (db DBRetrier) ResetPingError(ctx context.Context, id NodeID) error {
+ _, err := db.retry(ctx, "ResetPingError", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.ResetPingError(ctx, id)
+ })
+ return err
+}
+
+func (db DBRetrier) UpdatePingError(ctx context.Context, id NodeID) error {
+ _, err := db.retry(ctx, "UpdatePingError", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdatePingError(ctx, id)
+ })
+ return err
+}
+
+func (db DBRetrier) CountPingErrors(ctx context.Context, id NodeID) (*uint, error) {
+ resultAny, err := db.retry(ctx, "CountPingErrors", func(ctx context.Context) (interface{}, error) {
+ return db.db.CountPingErrors(ctx, id)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.(*uint)
+ return result, err
+}
+
+func (db DBRetrier) UpdateClientID(ctx context.Context, id NodeID, clientID string) error {
+ _, err := db.retry(ctx, "UpdateClientID", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateClientID(ctx, id, clientID)
+ })
+ return err
+}
+
+func (db DBRetrier) UpdateNetworkID(ctx context.Context, id NodeID, networkID uint) error {
+ _, err := db.retry(ctx, "UpdateNetworkID", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateNetworkID(ctx, id, networkID)
+ })
+ return err
+}
+
+func (db DBRetrier) UpdateEthVersion(ctx context.Context, id NodeID, ethVersion uint) error {
+ _, err := db.retry(ctx, "UpdateEthVersion", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateEthVersion(ctx, id, ethVersion)
+ })
+ return err
+}
+
+func (db DBRetrier) UpdateHandshakeTransientError(ctx context.Context, id NodeID, hasTransientErr bool) error {
+ _, err := db.retry(ctx, "UpdateHandshakeTransientError", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateHandshakeTransientError(ctx, id, hasTransientErr)
+ })
+ return err
+}
+
+func (db DBRetrier) InsertHandshakeError(ctx context.Context, id NodeID, handshakeErr string) error {
+ _, err := db.retry(ctx, "InsertHandshakeError", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.InsertHandshakeError(ctx, id, handshakeErr)
+ })
+ return err
+}
+
+func (db DBRetrier) DeleteHandshakeErrors(ctx context.Context, id NodeID) error {
+ _, err := db.retry(ctx, "DeleteHandshakeErrors", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.DeleteHandshakeErrors(ctx, id)
+ })
+ return err
+}
+
+func (db DBRetrier) FindHandshakeLastErrors(ctx context.Context, id NodeID, limit uint) ([]HandshakeError, error) {
+ resultAny, err := db.retry(ctx, "FindHandshakeLastErrors", func(ctx context.Context) (interface{}, error) {
+ return db.db.FindHandshakeLastErrors(ctx, id, limit)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.([]HandshakeError)
+ return result, err
+}
+
+func (db DBRetrier) UpdateHandshakeRetryTime(ctx context.Context, id NodeID, retryTime time.Time) error {
+ _, err := db.retry(ctx, "UpdateHandshakeRetryTime", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateHandshakeRetryTime(ctx, id, retryTime)
+ })
+ return err
+}
+
+func (db DBRetrier) FindHandshakeRetryTime(ctx context.Context, id NodeID) (*time.Time, error) {
+ resultAny, err := db.retry(ctx, "FindHandshakeRetryTime", func(ctx context.Context) (interface{}, error) {
+ return db.db.FindHandshakeRetryTime(ctx, id)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.(*time.Time)
+ return result, err
+}
+
+func (db DBRetrier) CountHandshakeCandidates(ctx context.Context) (uint, error) {
+ resultAny, err := db.retry(ctx, "CountHandshakeCandidates", func(ctx context.Context) (interface{}, error) {
+ return db.db.CountHandshakeCandidates(ctx)
+ })
+
+ if resultAny == nil {
+ return 0, err
+ }
+ result := resultAny.(uint)
+ return result, err
+}
+
+func (db DBRetrier) TakeHandshakeCandidates(ctx context.Context, limit uint) ([]NodeID, error) {
+ resultAny, err := db.retry(ctx, "TakeHandshakeCandidates", func(ctx context.Context) (interface{}, error) {
+ return db.db.TakeHandshakeCandidates(ctx, limit)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.([]NodeID)
+ return result, err
+}
+
+func (db DBRetrier) UpdateForkCompatibility(ctx context.Context, id NodeID, isCompatFork bool) error {
+ _, err := db.retry(ctx, "UpdateForkCompatibility", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateForkCompatibility(ctx, id, isCompatFork)
+ })
+ return err
+}
+
+func (db DBRetrier) UpdateNeighborBucketKeys(ctx context.Context, id NodeID, keys []string) error {
+ _, err := db.retry(ctx, "UpdateNeighborBucketKeys", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateNeighborBucketKeys(ctx, id, keys)
+ })
+ return err
+}
+
+func (db DBRetrier) FindNeighborBucketKeys(ctx context.Context, id NodeID) ([]string, error) {
+ resultAny, err := db.retry(ctx, "FindNeighborBucketKeys", func(ctx context.Context) (interface{}, error) {
+ return db.db.FindNeighborBucketKeys(ctx, id)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.([]string)
+ return result, err
+}
+
+func (db DBRetrier) UpdateCrawlRetryTime(ctx context.Context, id NodeID, retryTime time.Time) error {
+ _, err := db.retry(ctx, "UpdateCrawlRetryTime", func(ctx context.Context) (interface{}, error) {
+ return nil, db.db.UpdateCrawlRetryTime(ctx, id, retryTime)
+ })
+ return err
+}
+
+func (db DBRetrier) CountCandidates(ctx context.Context) (uint, error) {
+ resultAny, err := db.retry(ctx, "CountCandidates", func(ctx context.Context) (interface{}, error) {
+ return db.db.CountCandidates(ctx)
+ })
+
+ if resultAny == nil {
+ return 0, err
+ }
+ result := resultAny.(uint)
+ return result, err
+}
+
+func (db DBRetrier) TakeCandidates(ctx context.Context, limit uint) ([]NodeID, error) {
+ resultAny, err := db.retry(ctx, "TakeCandidates", func(ctx context.Context) (interface{}, error) {
+ return db.db.TakeCandidates(ctx, limit)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.([]NodeID)
+ return result, err
+}
+
+func (db DBRetrier) IsConflictError(err error) bool {
+ return db.db.IsConflictError(err)
+}
diff --git a/cmd/observer/database/db_sqlite.go b/cmd/observer/database/db_sqlite.go
new file mode 100644
index 0000000000..9d27727eab
--- /dev/null
+++ b/cmd/observer/database/db_sqlite.go
@@ -0,0 +1,866 @@
+package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ _ "modernc.org/sqlite"
+ "net"
+ "strings"
+ "time"
+)
+
+type DBSQLite struct {
+ db *sql.DB
+}
+
+// language=SQL
+const (
+ sqlCreateSchema = `
+PRAGMA journal_mode = WAL;
+
+CREATE TABLE IF NOT EXISTS nodes (
+ id TEXT PRIMARY KEY,
+
+ ip TEXT,
+ port_disc INTEGER,
+ port_rlpx INTEGER,
+ ip_v6 TEXT,
+ ip_v6_port_disc INTEGER,
+ ip_v6_port_rlpx INTEGER,
+ addr_updated INTEGER NOT NULL,
+
+ ping_try INTEGER NOT NULL DEFAULT 0,
+
+ compat_fork INTEGER,
+ compat_fork_updated INTEGER,
+
+ client_id TEXT,
+ network_id INTEGER,
+ eth_version INTEGER,
+ handshake_transient_err INTEGER NOT NULL DEFAULT 0,
+ handshake_updated INTEGER,
+ handshake_retry_time INTEGER,
+
+ neighbor_keys TEXT,
+
+ crawl_retry_time INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS handshake_errors (
+ id TEXT NOT NULL,
+ err TEXT NOT NULL,
+ updated INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_nodes_crawl_retry_time ON nodes (crawl_retry_time);
+CREATE INDEX IF NOT EXISTS idx_nodes_ip ON nodes (ip);
+CREATE INDEX IF NOT EXISTS idx_nodes_ip_v6 ON nodes (ip_v6);
+CREATE INDEX IF NOT EXISTS idx_nodes_ping_try ON nodes (ping_try);
+CREATE INDEX IF NOT EXISTS idx_nodes_compat_fork ON nodes (compat_fork);
+CREATE INDEX IF NOT EXISTS idx_nodes_network_id ON nodes (network_id);
+CREATE INDEX IF NOT EXISTS idx_nodes_handshake_retry_time ON nodes (handshake_retry_time);
+CREATE INDEX IF NOT EXISTS idx_handshake_errors_id ON handshake_errors (id);
+`
+
+ sqlUpsertNodeAddr = `
+INSERT INTO nodes(
+ id,
+ ip,
+ port_disc,
+ port_rlpx,
+ ip_v6,
+ ip_v6_port_disc,
+ ip_v6_port_rlpx,
+ addr_updated
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(id) DO UPDATE SET
+ ip = excluded.ip,
+ port_disc = excluded.port_disc,
+ port_rlpx = excluded.port_rlpx,
+ ip_v6 = excluded.ip_v6,
+ ip_v6_port_disc = excluded.ip_v6_port_disc,
+ ip_v6_port_rlpx = excluded.ip_v6_port_rlpx,
+ addr_updated = excluded.addr_updated
+`
+
+ sqlFindNodeAddr = `
+SELECT
+ ip,
+ port_disc,
+ port_rlpx,
+ ip_v6,
+ ip_v6_port_disc,
+ ip_v6_port_rlpx
+FROM nodes
+WHERE id = ?
+`
+
+ sqlResetPingError = `
+UPDATE nodes SET ping_try = 0 WHERE id = ?
+`
+
+ sqlUpdatePingError = `
+UPDATE nodes SET ping_try = nodes.ping_try + 1 WHERE id = ?
+`
+
+ sqlCountPingErrors = `
+SELECT ping_try FROM nodes WHERE id = ?
+`
+
+ sqlUpdateClientID = `
+UPDATE nodes SET
+ client_id = ?,
+ handshake_updated = ?
+WHERE id = ?
+`
+
+ sqlUpdateNetworkID = `
+UPDATE nodes SET
+ network_id = ?,
+ handshake_updated = ?
+WHERE id = ?
+`
+
+ sqlUpdateEthVersion = `
+UPDATE nodes SET
+ eth_version = ?,
+ handshake_updated = ?
+WHERE id = ?
+`
+
+ sqlUpdateHandshakeTransientError = `
+UPDATE nodes SET
+ handshake_transient_err = ?,
+ handshake_updated = ?
+WHERE id = ?
+`
+
+ sqlInsertHandshakeError = `
+INSERT INTO handshake_errors(
+ id,
+ err,
+ updated
+) VALUES (?, ?, ?)
+`
+
+ sqlDeleteHandshakeErrors = `
+DELETE FROM handshake_errors WHERE id = ?
+`
+
+ sqlFindHandshakeLastErrors = `
+SELECT err, updated FROM handshake_errors
+WHERE id = ?
+ORDER BY updated DESC
+LIMIT ?
+`
+
+ sqlUpdateHandshakeRetryTime = `
+UPDATE nodes SET handshake_retry_time = ? WHERE id = ?
+`
+
+ sqlFindHandshakeRetryTime = `
+SELECT handshake_retry_time FROM nodes WHERE id = ?
+`
+
+ sqlCountHandshakeCandidates = `
+SELECT COUNT(*) FROM nodes
+WHERE ((handshake_retry_time IS NULL) OR (handshake_retry_time < ?))
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+`
+
+ sqlFindHandshakeCandidates = `
+SELECT id FROM nodes
+WHERE ((handshake_retry_time IS NULL) OR (handshake_retry_time < ?))
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+ORDER BY handshake_retry_time
+LIMIT ?
+`
+
+ sqlMarkTakenHandshakeCandidates = `
+UPDATE nodes SET handshake_retry_time = ? WHERE id IN (123)
+`
+
+ sqlUpdateForkCompatibility = `
+UPDATE nodes SET compat_fork = ?, compat_fork_updated = ? WHERE id = ?
+`
+
+ sqlUpdateNeighborBucketKeys = `
+UPDATE nodes SET neighbor_keys = ? WHERE id = ?
+`
+
+ sqlFindNeighborBucketKeys = `
+SELECT neighbor_keys FROM nodes WHERE id = ?
+`
+
+ sqlUpdateCrawlRetryTime = `
+UPDATE nodes SET crawl_retry_time = ? WHERE id = ?
+`
+
+ sqlCountCandidates = `
+SELECT COUNT(*) FROM nodes
+WHERE ((crawl_retry_time IS NULL) OR (crawl_retry_time < ?))
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+`
+
+ sqlFindCandidates = `
+SELECT id FROM nodes
+WHERE ((crawl_retry_time IS NULL) OR (crawl_retry_time < ?))
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+ORDER BY crawl_retry_time
+LIMIT ?
+`
+
+ sqlMarkTakenNodes = `
+UPDATE nodes SET crawl_retry_time = ? WHERE id IN (123)
+`
+
+ sqlCountNodes = `
+SELECT COUNT(*) FROM nodes
+WHERE (ping_try < ?)
+ AND ((network_id = ?) OR (network_id IS NULL))
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+`
+
+ sqlCountIPs = `
+SELECT COUNT(DISTINCT ip) FROM nodes
+WHERE (ping_try < ?)
+ AND ((network_id = ?) OR (network_id IS NULL))
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+`
+
+ sqlCountClients = `
+SELECT COUNT(*) FROM nodes
+WHERE (ping_try < ?)
+ AND (network_id = ?)
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+ AND (client_id LIKE ?)
+`
+
+ sqlCountClientsWithNetworkID = `
+SELECT COUNT(*) FROM nodes
+WHERE (ping_try < ?)
+ AND (network_id IS NOT NULL)
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+ AND (client_id LIKE ?)
+`
+
+ sqlCountClientsWithHandshakeTransientError = `
+SELECT COUNT(*) FROM nodes
+WHERE (ping_try < ?)
+ AND (handshake_transient_err = 1)
+ AND (network_id IS NULL)
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+ AND (client_id LIKE ?)
+`
+
+ sqlEnumerateClientIDs = `
+SELECT client_id FROM nodes
+WHERE (ping_try < ?)
+ AND ((network_id = ?) OR (network_id IS NULL))
+ AND ((compat_fork == TRUE) OR (compat_fork IS NULL))
+`
+)
+
+func NewDBSQLite(filePath string) (*DBSQLite, error) {
+ db, err := sql.Open("sqlite", filePath)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open DB: %w", err)
+ }
+
+ _, err = db.Exec(sqlCreateSchema)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create the DB schema: %w", err)
+ }
+
+ instance := DBSQLite{db}
+ return &instance, nil
+}
+
+func (db *DBSQLite) Close() error {
+ return db.db.Close()
+}
+
+func (db *DBSQLite) UpsertNodeAddr(ctx context.Context, id NodeID, addr NodeAddr) error {
+ var ip *string
+ if addr.IP != nil {
+ value := addr.IP.String()
+ ip = &value
+ }
+
+ var ipV6 *string
+ if addr.IPv6.IP != nil {
+ value := addr.IPv6.IP.String()
+ ipV6 = &value
+ }
+
+ var portDisc *int
+ if (ip != nil) && (addr.PortDisc != 0) {
+ value := int(addr.PortDisc)
+ portDisc = &value
+ }
+
+ var ipV6PortDisc *int
+ if (ipV6 != nil) && (addr.IPv6.PortDisc != 0) {
+ value := int(addr.IPv6.PortDisc)
+ ipV6PortDisc = &value
+ }
+
+ var portRLPx *int
+ if (ip != nil) && (addr.PortRLPx != 0) {
+ value := int(addr.PortRLPx)
+ portRLPx = &value
+ }
+
+ var ipV6PortRLPx *int
+ if (ipV6 != nil) && (addr.IPv6.PortRLPx != 0) {
+ value := int(addr.IPv6.PortRLPx)
+ ipV6PortRLPx = &value
+ }
+
+ updated := time.Now().Unix()
+
+ _, err := db.db.ExecContext(ctx, sqlUpsertNodeAddr,
+ id,
+ ip, portDisc, portRLPx,
+ ipV6, ipV6PortDisc, ipV6PortRLPx,
+ updated)
+ if err != nil {
+ return fmt.Errorf("failed to upsert a node address: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) FindNodeAddr(ctx context.Context, id NodeID) (*NodeAddr, error) {
+ row := db.db.QueryRowContext(ctx, sqlFindNodeAddr, id)
+
+ var ip sql.NullString
+ var portDisc sql.NullInt32
+ var portRLPx sql.NullInt32
+ var ipV6 sql.NullString
+ var ipV6PortDisc sql.NullInt32
+ var ipV6PortRLPx sql.NullInt32
+
+ err := row.Scan(
+ &ip,
+ &portDisc,
+ &portRLPx,
+ &ipV6,
+ &ipV6PortDisc,
+ &ipV6PortRLPx)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("FindNodeAddr failed: %w", err)
+ }
+
+ var addr NodeAddr
+
+ if ip.Valid {
+ value := net.ParseIP(ip.String)
+ if value == nil {
+ return nil, errors.New("FindNodeAddr failed to parse IP")
+ }
+ addr.IP = value
+ }
+ if ipV6.Valid {
+ value := net.ParseIP(ipV6.String)
+ if value == nil {
+ return nil, errors.New("FindNodeAddr failed to parse IPv6")
+ }
+ addr.IPv6.IP = value
+ }
+ if portDisc.Valid {
+ value := uint16(portDisc.Int32)
+ addr.PortDisc = value
+ }
+ if portRLPx.Valid {
+ value := uint16(portRLPx.Int32)
+ addr.PortRLPx = value
+ }
+ if ipV6PortDisc.Valid {
+ value := uint16(ipV6PortDisc.Int32)
+ addr.IPv6.PortDisc = value
+ }
+ if ipV6PortRLPx.Valid {
+ value := uint16(ipV6PortRLPx.Int32)
+ addr.IPv6.PortRLPx = value
+ }
+
+ return &addr, nil
+}
+
+func (db *DBSQLite) ResetPingError(ctx context.Context, id NodeID) error {
+ _, err := db.db.ExecContext(ctx, sqlResetPingError, id)
+ if err != nil {
+ return fmt.Errorf("ResetPingError failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) UpdatePingError(ctx context.Context, id NodeID) error {
+ _, err := db.db.ExecContext(ctx, sqlUpdatePingError, id)
+ if err != nil {
+ return fmt.Errorf("UpdatePingError failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) CountPingErrors(ctx context.Context, id NodeID) (*uint, error) {
+ row := db.db.QueryRowContext(ctx, sqlCountPingErrors, id)
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("CountPingErrors failed: %w", err)
+ }
+ return &count, nil
+}
+
+func (db *DBSQLite) UpdateClientID(ctx context.Context, id NodeID, clientID string) error {
+ updated := time.Now().Unix()
+
+ _, err := db.db.ExecContext(ctx, sqlUpdateClientID, clientID, updated, id)
+ if err != nil {
+ return fmt.Errorf("UpdateClientID failed to update a node: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) UpdateNetworkID(ctx context.Context, id NodeID, networkID uint) error {
+ updated := time.Now().Unix()
+
+ _, err := db.db.ExecContext(ctx, sqlUpdateNetworkID, networkID, updated, id)
+ if err != nil {
+ return fmt.Errorf("UpdateNetworkID failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) UpdateEthVersion(ctx context.Context, id NodeID, ethVersion uint) error {
+ updated := time.Now().Unix()
+
+ _, err := db.db.ExecContext(ctx, sqlUpdateEthVersion, ethVersion, updated, id)
+ if err != nil {
+ return fmt.Errorf("UpdateEthVersion failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) UpdateHandshakeTransientError(ctx context.Context, id NodeID, hasTransientErr bool) error {
+ updated := time.Now().Unix()
+
+ _, err := db.db.ExecContext(ctx, sqlUpdateHandshakeTransientError, hasTransientErr, updated, id)
+ if err != nil {
+ return fmt.Errorf("UpdateHandshakeTransientError failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) InsertHandshakeError(ctx context.Context, id NodeID, handshakeErr string) error {
+ updated := time.Now().Unix()
+
+ _, err := db.db.ExecContext(ctx, sqlInsertHandshakeError, id, handshakeErr, updated)
+ if err != nil {
+ return fmt.Errorf("InsertHandshakeError failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) DeleteHandshakeErrors(ctx context.Context, id NodeID) error {
+ _, err := db.db.ExecContext(ctx, sqlDeleteHandshakeErrors, id)
+ if err != nil {
+ return fmt.Errorf("DeleteHandshakeErrors failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) FindHandshakeLastErrors(ctx context.Context, id NodeID, limit uint) ([]HandshakeError, error) {
+ cursor, err := db.db.QueryContext(
+ ctx,
+ sqlFindHandshakeLastErrors,
+ id,
+ limit)
+ if err != nil {
+ return nil, fmt.Errorf("FindHandshakeLastErrors failed to query: %w", err)
+ }
+ defer func() {
+ _ = cursor.Close()
+ }()
+
+ var handshakeErrors []HandshakeError
+ for cursor.Next() {
+ var stringCode string
+ var updatedTimestamp int64
+ err := cursor.Scan(&stringCode, &updatedTimestamp)
+ if err != nil {
+ return nil, fmt.Errorf("FindHandshakeLastErrors failed to read data: %w", err)
+ }
+
+ handshakeError := HandshakeError{
+ stringCode,
+ time.Unix(updatedTimestamp, 0),
+ }
+
+ handshakeErrors = append(handshakeErrors, handshakeError)
+ }
+
+ if err := cursor.Err(); err != nil {
+ return nil, fmt.Errorf("FindHandshakeLastErrors failed to iterate over rows: %w", err)
+ }
+ return handshakeErrors, nil
+}
+
+func (db *DBSQLite) UpdateHandshakeRetryTime(ctx context.Context, id NodeID, retryTime time.Time) error {
+ _, err := db.db.ExecContext(ctx, sqlUpdateHandshakeRetryTime, retryTime.Unix(), id)
+ if err != nil {
+ return fmt.Errorf("UpdateHandshakeRetryTime failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) FindHandshakeRetryTime(ctx context.Context, id NodeID) (*time.Time, error) {
+ row := db.db.QueryRowContext(ctx, sqlFindHandshakeRetryTime, id)
+
+ var timestamp sql.NullInt64
+
+ if err := row.Scan(×tamp); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("FindHandshakeRetryTime failed: %w", err)
+ }
+
+ // if never we tried to handshake then the time is NULL
+ if !timestamp.Valid {
+ return nil, nil
+ }
+
+ retryTime := time.Unix(timestamp.Int64, 0)
+ return &retryTime, nil
+}
+
+func (db *DBSQLite) CountHandshakeCandidates(ctx context.Context) (uint, error) {
+ retryTimeBefore := time.Now().Unix()
+ row := db.db.QueryRowContext(ctx, sqlCountHandshakeCandidates, retryTimeBefore)
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("CountHandshakeCandidates failed: %w", err)
+ }
+ return count, nil
+}
+
+func (db *DBSQLite) FindHandshakeCandidates(
+ ctx context.Context,
+ limit uint,
+) ([]NodeID, error) {
+ retryTimeBefore := time.Now().Unix()
+ cursor, err := db.db.QueryContext(
+ ctx,
+ sqlFindHandshakeCandidates,
+ retryTimeBefore,
+ limit)
+ if err != nil {
+ return nil, fmt.Errorf("FindHandshakeCandidates failed to query candidates: %w", err)
+ }
+ defer func() {
+ _ = cursor.Close()
+ }()
+
+ var nodes []NodeID
+ for cursor.Next() {
+ var id string
+ err := cursor.Scan(&id)
+ if err != nil {
+ return nil, fmt.Errorf("FindHandshakeCandidates failed to read candidate data: %w", err)
+ }
+
+ nodes = append(nodes, NodeID(id))
+ }
+
+ if err := cursor.Err(); err != nil {
+ return nil, fmt.Errorf("FindHandshakeCandidates failed to iterate over candidates: %w", err)
+ }
+ return nodes, nil
+}
+
+func (db *DBSQLite) MarkTakenHandshakeCandidates(ctx context.Context, ids []NodeID) error {
+ if len(ids) == 0 {
+ return nil
+ }
+
+ delayedRetryTime := time.Now().Add(time.Hour).Unix()
+
+ idsPlaceholders := strings.TrimRight(strings.Repeat("?,", len(ids)), ",")
+ query := strings.Replace(sqlMarkTakenHandshakeCandidates, "123", idsPlaceholders, 1)
+ args := append([]interface{}{delayedRetryTime}, stringsToAny(ids)...)
+
+ _, err := db.db.ExecContext(ctx, query, args...)
+ if err != nil {
+ return fmt.Errorf("failed to mark taken handshake candidates: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) TakeHandshakeCandidates(
+ ctx context.Context,
+ limit uint,
+) ([]NodeID, error) {
+ tx, err := db.db.BeginTx(ctx, nil)
+ if err != nil {
+ return nil, fmt.Errorf("TakeHandshakeCandidates failed to start transaction: %w", err)
+ }
+
+ ids, err := db.FindHandshakeCandidates(
+ ctx,
+ limit)
+ if err != nil {
+ _ = tx.Rollback()
+ return nil, err
+ }
+
+ err = db.MarkTakenHandshakeCandidates(ctx, ids)
+ if err != nil {
+ _ = tx.Rollback()
+ return nil, err
+ }
+
+ err = tx.Commit()
+ if err != nil {
+ return nil, fmt.Errorf("TakeHandshakeCandidates failed to commit transaction: %w", err)
+ }
+ return ids, nil
+}
+
+func (db *DBSQLite) UpdateForkCompatibility(ctx context.Context, id NodeID, isCompatFork bool) error {
+ updated := time.Now().Unix()
+
+ _, err := db.db.ExecContext(ctx, sqlUpdateForkCompatibility, isCompatFork, updated, id)
+ if err != nil {
+ return fmt.Errorf("UpdateForkCompatibility failed to update a node: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) UpdateNeighborBucketKeys(ctx context.Context, id NodeID, keys []string) error {
+ keysStr := strings.Join(keys, ",")
+
+ _, err := db.db.ExecContext(ctx, sqlUpdateNeighborBucketKeys, keysStr, id)
+ if err != nil {
+ return fmt.Errorf("UpdateNeighborBucketKeys failed to update a node: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) FindNeighborBucketKeys(ctx context.Context, id NodeID) ([]string, error) {
+ row := db.db.QueryRowContext(ctx, sqlFindNeighborBucketKeys, id)
+
+ var keysStr sql.NullString
+ if err := row.Scan(&keysStr); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("FindNeighborBucketKeys failed: %w", err)
+ }
+
+ if !keysStr.Valid {
+ return nil, nil
+ }
+ return strings.Split(keysStr.String, ","), nil
+}
+
+func (db *DBSQLite) UpdateCrawlRetryTime(ctx context.Context, id NodeID, retryTime time.Time) error {
+ _, err := db.db.ExecContext(ctx, sqlUpdateCrawlRetryTime, retryTime.Unix(), id)
+ if err != nil {
+ return fmt.Errorf("UpdateCrawlRetryTime failed: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) CountCandidates(ctx context.Context) (uint, error) {
+ retryTimeBefore := time.Now().Unix()
+ row := db.db.QueryRowContext(ctx, sqlCountCandidates, retryTimeBefore)
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("CountCandidates failed: %w", err)
+ }
+ return count, nil
+}
+
+func (db *DBSQLite) FindCandidates(
+ ctx context.Context,
+ limit uint,
+) ([]NodeID, error) {
+ retryTimeBefore := time.Now().Unix()
+ cursor, err := db.db.QueryContext(
+ ctx,
+ sqlFindCandidates,
+ retryTimeBefore,
+ limit)
+ if err != nil {
+ return nil, fmt.Errorf("FindCandidates failed to query candidates: %w", err)
+ }
+ defer func() {
+ _ = cursor.Close()
+ }()
+
+ var nodes []NodeID
+ for cursor.Next() {
+ var id string
+ err := cursor.Scan(&id)
+ if err != nil {
+ return nil, fmt.Errorf("FindCandidates failed to read candidate data: %w", err)
+ }
+
+ nodes = append(nodes, NodeID(id))
+ }
+
+ if err := cursor.Err(); err != nil {
+ return nil, fmt.Errorf("FindCandidates failed to iterate over candidates: %w", err)
+ }
+ return nodes, nil
+}
+
+func (db *DBSQLite) MarkTakenNodes(ctx context.Context, ids []NodeID) error {
+ if len(ids) == 0 {
+ return nil
+ }
+
+ delayedRetryTime := time.Now().Add(time.Hour).Unix()
+
+ idsPlaceholders := strings.TrimRight(strings.Repeat("?,", len(ids)), ",")
+ query := strings.Replace(sqlMarkTakenNodes, "123", idsPlaceholders, 1)
+ args := append([]interface{}{delayedRetryTime}, stringsToAny(ids)...)
+
+ _, err := db.db.ExecContext(ctx, query, args...)
+ if err != nil {
+ return fmt.Errorf("failed to mark taken nodes: %w", err)
+ }
+ return nil
+}
+
+func (db *DBSQLite) TakeCandidates(
+ ctx context.Context,
+ limit uint,
+) ([]NodeID, error) {
+ tx, err := db.db.BeginTx(ctx, nil)
+ if err != nil {
+ return nil, fmt.Errorf("TakeCandidates failed to start transaction: %w", err)
+ }
+
+ ids, err := db.FindCandidates(
+ ctx,
+ limit)
+ if err != nil {
+ _ = tx.Rollback()
+ return nil, err
+ }
+
+ err = db.MarkTakenNodes(ctx, ids)
+ if err != nil {
+ _ = tx.Rollback()
+ return nil, err
+ }
+
+ err = tx.Commit()
+ if err != nil {
+ return nil, fmt.Errorf("TakeCandidates failed to commit transaction: %w", err)
+ }
+ return ids, nil
+}
+
+func (db *DBSQLite) IsConflictError(err error) bool {
+ if err == nil {
+ return false
+ }
+ return strings.Contains(err.Error(), "SQLITE_BUSY")
+}
+
+func (db *DBSQLite) CountNodes(ctx context.Context, maxPingTries uint, networkID uint) (uint, error) {
+ row := db.db.QueryRowContext(ctx, sqlCountNodes, maxPingTries, networkID)
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("CountNodes failed: %w", err)
+ }
+ return count, nil
+}
+
+func (db *DBSQLite) CountIPs(ctx context.Context, maxPingTries uint, networkID uint) (uint, error) {
+ row := db.db.QueryRowContext(ctx, sqlCountIPs, maxPingTries, networkID)
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("CountIPs failed: %w", err)
+ }
+ return count, nil
+}
+
+func (db *DBSQLite) CountClients(ctx context.Context, clientIDPrefix string, maxPingTries uint, networkID uint) (uint, error) {
+ row := db.db.QueryRowContext(ctx, sqlCountClients, maxPingTries, networkID, clientIDPrefix+"%")
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("CountClients failed: %w", err)
+ }
+ return count, nil
+}
+
+func (db *DBSQLite) CountClientsWithNetworkID(ctx context.Context, clientIDPrefix string, maxPingTries uint) (uint, error) {
+ row := db.db.QueryRowContext(ctx, sqlCountClientsWithNetworkID, maxPingTries, clientIDPrefix+"%")
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("CountClientsWithNetworkID failed: %w", err)
+ }
+ return count, nil
+}
+
+func (db *DBSQLite) CountClientsWithHandshakeTransientError(ctx context.Context, clientIDPrefix string, maxPingTries uint) (uint, error) {
+ row := db.db.QueryRowContext(ctx, sqlCountClientsWithHandshakeTransientError, maxPingTries, clientIDPrefix+"%")
+ var count uint
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("CountClientsWithHandshakeTransientError failed: %w", err)
+ }
+ return count, nil
+}
+
+func (db *DBSQLite) EnumerateClientIDs(
+ ctx context.Context,
+ maxPingTries uint,
+ networkID uint,
+ enumFunc func(clientID *string),
+) error {
+ cursor, err := db.db.QueryContext(ctx, sqlEnumerateClientIDs, maxPingTries, networkID)
+ if err != nil {
+ return fmt.Errorf("EnumerateClientIDs failed to query: %w", err)
+ }
+ defer func() {
+ _ = cursor.Close()
+ }()
+
+ for cursor.Next() {
+ var clientID sql.NullString
+ err := cursor.Scan(&clientID)
+ if err != nil {
+ return fmt.Errorf("EnumerateClientIDs failed to read data: %w", err)
+ }
+ if clientID.Valid {
+ enumFunc(&clientID.String)
+ } else {
+ enumFunc(nil)
+ }
+ }
+
+ if err := cursor.Err(); err != nil {
+ return fmt.Errorf("EnumerateClientIDs failed to iterate: %w", err)
+ }
+ return nil
+}
+
+func stringsToAny(strValues []NodeID) []interface{} {
+ values := make([]interface{}, 0, len(strValues))
+ for _, value := range strValues {
+ values = append(values, value)
+ }
+ return values
+}
diff --git a/cmd/observer/database/db_sqlite_test.go b/cmd/observer/database/db_sqlite_test.go
new file mode 100644
index 0000000000..c43e39f15b
--- /dev/null
+++ b/cmd/observer/database/db_sqlite_test.go
@@ -0,0 +1,40 @@
+package database
+
+import (
+ "context"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "net"
+ "path/filepath"
+ "testing"
+)
+
+func TestDBSQLiteInsertAndFind(t *testing.T) {
+ ctx := context.Background()
+ db, err := NewDBSQLite(filepath.Join(t.TempDir(), "observer.sqlite"))
+ require.Nil(t, err)
+ defer func() { _ = db.Close() }()
+
+ var id NodeID = "ba85011c70bcc5c04d8607d3a0ed29aa6179c092cbdda10d5d32684fb33ed01bd94f588ca8f91ac48318087dcb02eaf36773a7a453f0eedd6742af668097b29c"
+ var addr NodeAddr
+ addr.IP = net.ParseIP("10.0.1.16")
+ addr.PortRLPx = 30303
+ addr.PortDisc = 30304
+
+ err = db.UpsertNodeAddr(ctx, id, addr)
+ require.Nil(t, err)
+
+ candidates, err := db.FindCandidates(ctx, 1)
+ require.Nil(t, err)
+ require.Equal(t, 1, len(candidates))
+
+ candidateID := candidates[0]
+ assert.Equal(t, id, candidateID)
+
+ candidate, err := db.FindNodeAddr(ctx, candidateID)
+ require.Nil(t, err)
+
+ assert.Equal(t, addr.IP, candidate.IP)
+ assert.Equal(t, addr.PortDisc, candidate.PortDisc)
+ assert.Equal(t, addr.PortRLPx, candidate.PortRLPx)
+}
diff --git a/cmd/observer/main.go b/cmd/observer/main.go
new file mode 100644
index 0000000000..c7fdad7d08
--- /dev/null
+++ b/cmd/observer/main.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "github.com/ledgerwatch/erigon-lib/common"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "github.com/ledgerwatch/erigon/cmd/observer/observer"
+ "github.com/ledgerwatch/erigon/cmd/observer/reports"
+ "github.com/ledgerwatch/erigon/cmd/utils"
+ "github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/log/v3"
+ "path/filepath"
+)
+
+func mainWithFlags(ctx context.Context, flags observer.CommandFlags) error {
+ server, err := observer.NewServer(flags)
+ if err != nil {
+ return err
+ }
+
+ db, err := database.NewDBSQLite(filepath.Join(flags.DataDir, "observer.sqlite"))
+ if err != nil {
+ return err
+ }
+ defer func() { _ = db.Close() }()
+
+ discV4, err := server.Listen(ctx)
+ if err != nil {
+ return err
+ }
+
+ networkID := uint(params.NetworkIDByChainName(flags.Chain))
+ go observer.StatusLoggerLoop(ctx, db, networkID, flags.StatusLogPeriod, log.Root())
+
+ crawlerConfig := observer.CrawlerConfig{
+ Chain: flags.Chain,
+ Bootnodes: server.Bootnodes(),
+ PrivateKey: server.PrivateKey(),
+ ConcurrencyLimit: flags.CrawlerConcurrency,
+ RefreshTimeout: flags.RefreshTimeout,
+ MaxPingTries: flags.MaxPingTries,
+ StatusLogPeriod: flags.StatusLogPeriod,
+
+ HandshakeRefreshTimeout: flags.HandshakeRefreshTimeout,
+ HandshakeRetryDelay: flags.HandshakeRetryDelay,
+ HandshakeMaxTries: flags.HandshakeMaxTries,
+
+ KeygenTimeout: flags.KeygenTimeout,
+ KeygenConcurrency: flags.KeygenConcurrency,
+ }
+
+ crawler, err := observer.NewCrawler(discV4, db, crawlerConfig, log.Root())
+ if err != nil {
+ return err
+ }
+
+ return crawler.Run(ctx)
+}
+
+func reportWithFlags(ctx context.Context, flags reports.CommandFlags) error {
+ db, err := database.NewDBSQLite(filepath.Join(flags.DataDir, "observer.sqlite"))
+ if err != nil {
+ return err
+ }
+ defer func() { _ = db.Close() }()
+
+ networkID := uint(params.NetworkIDByChainName(flags.Chain))
+
+ if flags.Estimate {
+ report, err := reports.CreateClientsEstimateReport(ctx, db, flags.ClientsLimit, flags.MaxPingTries, networkID)
+ if err != nil {
+ return err
+ }
+ fmt.Println(report)
+ return nil
+ }
+
+ statusReport, err := reports.CreateStatusReport(ctx, db, flags.MaxPingTries, networkID)
+ if err != nil {
+ return err
+ }
+ clientsReport, err := reports.CreateClientsReport(ctx, db, flags.ClientsLimit, flags.MaxPingTries, networkID)
+ if err != nil {
+ return err
+ }
+
+ fmt.Println(statusReport)
+ fmt.Println(clientsReport)
+ return nil
+}
+
+func main() {
+ ctx, cancel := common.RootContext()
+ defer cancel()
+
+ command := observer.NewCommand()
+
+ reportCommand := reports.NewCommand()
+ reportCommand.OnRun(reportWithFlags)
+ command.AddSubCommand(reportCommand.RawCommand())
+
+ err := command.ExecuteContext(ctx, mainWithFlags)
+ if (err != nil) && !errors.Is(err, context.Canceled) {
+ utils.Fatalf("%v", err)
+ }
+}
diff --git a/cmd/observer/observer/client_id.go b/cmd/observer/observer/client_id.go
new file mode 100644
index 0000000000..2dec521ac2
--- /dev/null
+++ b/cmd/observer/observer/client_id.go
@@ -0,0 +1,133 @@
+package observer
+
+import "strings"
+
+func clientNameBlacklist() []string {
+ return []string{
+ // bor/v0.2.14-stable-9edb2836/linux-amd64/go1.17.7
+ // https://polygon.technology
+ "bor",
+
+ // Cypher/v1.9.24-unstable-a7d8c0f9/linux-amd64/go1.11
+ // unknown, but it's likely outdated since almost all nodes are running on go 1.11 (2018)
+ "Cypher",
+
+ // Ecoball/v1.0.2-stable-ac03aee-20211125/x86_64-linux-gnu/rustc1.52.1
+ // https://ecoball.org
+ "Ecoball",
+
+ // egem/v1.1.4-titanus-9b056f56-20210808/linux-amd64/go1.15.13
+ // https://egem.io
+ "egem",
+
+ // energi3/v3.1.1-stable/linux-amd64/go1.15.8
+ // https://energi.world
+ "energi3",
+
+ // Gdbix/v1.5.3-fluxdbix-f6911ea5/linux/go1.8.3
+ // https://www.arabianchain.org
+ "Gdbix",
+
+ // Gddm/v0.8.1-master-7155d9dd/linux-amd64/go1.9.2
+ // unknown, but it's likely outdated since all nodes are running on go 1.9 (2017)
+ "Gddm",
+
+ // Gero/v1.1.3-dev-f8efb930/linux-amd64/go1.13.4
+ // https://sero.cash
+ "Gero",
+
+ // Gesn/v0.3.13-stable-b6c12eb2/linux-amd64/go1.12.4
+ // https://ethersocial.org
+ "Gesn",
+
+ // Gexp/v1.10.8-stable-1eb55798/linux-amd64/go1.17
+ // https://expanse.tech
+ "Gexp",
+
+ // Gnekonium/v1.6.6-stable-820982d6/linux-amd64/go1.9.2
+ // https://nekonium.github.io
+ "Gnekonium",
+
+ // go-corex/v1.0.0-rc.1-6197c8bf-1638348709/linux-amd64/go1.17.3
+ // https://www.corexchain.io
+ "go-corex",
+
+ // go-opera/v1.1.0-rc.4-91951f74-1647353617/linux-amd64/go1.17.8
+ // https://www.fantom.foundation
+ "go-opera",
+
+ // go-photon/v1.0.2-rc.5-32a52936-1646808549/linux-amd64/go1.17.8
+ // https://github.com/TechPay-io/go-photon
+ "go-photon",
+
+ // GoChain/v4.0.2/linux-amd64/go1.17.3
+ // https://gochain.io
+ "GoChain",
+
+ // gqdc/v1.5.2-stable-53b6a36d/linux-amd64/go1.13.4
+ // https://quadrans.io
+ "gqdc",
+
+ // Gtsf/v1.2.1-stable-df201e7e/linux-amd64/go1.13.4
+ // https://tsf-network.com
+ "Gtsf",
+
+ // Gubiq/v7.0.0-monoceros-c9009e89/linux-amd64/go1.17.6
+ // https://ubiqsmart.com
+ "Gubiq",
+
+ // Gvns/v3.2.0-unstable/linux-amd64/go1.12.4
+ // https://github.com/AMTcommunity/go-vnscoin
+ "Gvns",
+
+ // Moac/v2.1.5-stable-af7bea47/linux-amd64/go1.13.4
+ // https://www.moac.io
+ "Moac",
+
+ // pchain/linux-amd64/go1.13.3
+ // http://pchain.org
+ "pchain",
+
+ // Pirl/v1.9.12-v7-masternode-premium-lion-ea07aebf-20200407/linux-amd64/go1.13.6
+ // https://pirl.io
+ "Pirl",
+
+ // Q-Client/v1.0.8-stable/Geth/v1.10.8-stable-850a0145/linux-amd64/go1.16.15
+ // https://q.org
+ "Q-Client",
+
+ // qk_node/v1.10.16-stable-75ceb6c6-20220308/linux-amd64/go1.17.8
+ // https://quarkblockchain.medium.com
+ "qk_node",
+
+ // Quai/v1.10.10-unstable-b1b52e79-20220226/linux-amd64/go1.17.7
+ // https://quai.network
+ "Quai",
+
+ // REOSC/v2.2.4-unstable-6bcba06-20190321/x86_64-linux-gnu/rustc1.37.0
+ // https://www.reosc.io
+ "REOSC",
+
+ // ronin/v2.3.0-stable-f07cd8d1/linux-amd64/go1.15.5
+ // https://wallet.roninchain.com
+ "ronin",
+ }
+}
+
+func IsClientIDBlacklisted(clientID string) bool {
+ // some unknown clients return an empty string
+ if clientID == "" {
+ return true
+ }
+ for _, clientName := range clientNameBlacklist() {
+ if strings.HasPrefix(clientID, clientName) {
+ return true
+ }
+ }
+ return false
+}
+
+func NameFromClientID(clientID string) string {
+ parts := strings.SplitN(clientID, "/", 2)
+ return parts[0]
+}
diff --git a/cmd/observer/observer/command.go b/cmd/observer/observer/command.go
new file mode 100644
index 0000000000..ac72355ccf
--- /dev/null
+++ b/cmd/observer/observer/command.go
@@ -0,0 +1,233 @@
+package observer
+
+import (
+ "context"
+ "errors"
+ "github.com/ledgerwatch/erigon/cmd/utils"
+ "github.com/ledgerwatch/erigon/internal/debug"
+ "github.com/spf13/cobra"
+ "github.com/urfave/cli"
+ "runtime"
+ "time"
+)
+
+type CommandFlags struct {
+ DataDir string
+ StatusLogPeriod time.Duration
+
+ Chain string
+ Bootnodes string
+
+ ListenPort int
+ NATDesc string
+ NetRestrict string
+
+ NodeKeyFile string
+ NodeKeyHex string
+
+ CrawlerConcurrency uint
+ RefreshTimeout time.Duration
+ MaxPingTries uint
+
+ KeygenTimeout time.Duration
+ KeygenConcurrency uint
+
+ HandshakeRefreshTimeout time.Duration
+ HandshakeRetryDelay time.Duration
+ HandshakeMaxTries uint
+}
+
+type Command struct {
+ command cobra.Command
+ flags CommandFlags
+}
+
+func NewCommand() *Command {
+ command := cobra.Command{
+ Short: "P2P network crawler",
+ }
+
+ // debug flags
+ utils.CobraFlags(&command, append(debug.Flags, utils.MetricFlags...))
+
+ instance := Command{
+ command: command,
+ }
+
+ instance.withDatadir()
+ instance.withStatusLogPeriod()
+
+ instance.withChain()
+ instance.withBootnodes()
+
+ instance.withListenPort()
+ instance.withNAT()
+ instance.withNetRestrict()
+
+ instance.withNodeKeyFile()
+ instance.withNodeKeyHex()
+
+ instance.withCrawlerConcurrency()
+ instance.withRefreshTimeout()
+ instance.withMaxPingTries()
+
+ instance.withKeygenTimeout()
+ instance.withKeygenConcurrency()
+
+ instance.withHandshakeRefreshTimeout()
+ instance.withHandshakeRetryDelay()
+ instance.withHandshakeMaxTries()
+
+ return &instance
+}
+
+func (command *Command) withDatadir() {
+ flag := utils.DataDirFlag
+ command.command.Flags().StringVar(&command.flags.DataDir, flag.Name, flag.Value.String(), flag.Usage)
+ must(command.command.MarkFlagDirname(utils.DataDirFlag.Name))
+}
+
+func (command *Command) withStatusLogPeriod() {
+ flag := cli.DurationFlag{
+ Name: "status-log-period",
+ Usage: "How often to log status summaries",
+ Value: 10 * time.Second,
+ }
+ command.command.Flags().DurationVar(&command.flags.StatusLogPeriod, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withChain() {
+ flag := utils.ChainFlag
+ command.command.Flags().StringVar(&command.flags.Chain, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withBootnodes() {
+ flag := utils.BootnodesFlag
+ command.command.Flags().StringVar(&command.flags.Bootnodes, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withListenPort() {
+ flag := utils.ListenPortFlag
+ command.command.Flags().IntVar(&command.flags.ListenPort, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withNAT() {
+ flag := utils.NATFlag
+ command.command.Flags().StringVar(&command.flags.NATDesc, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withNetRestrict() {
+ flag := utils.NetrestrictFlag
+ command.command.Flags().StringVar(&command.flags.NetRestrict, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withNodeKeyFile() {
+ flag := utils.NodeKeyFileFlag
+ command.command.Flags().StringVar(&command.flags.NodeKeyFile, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withNodeKeyHex() {
+ flag := utils.NodeKeyHexFlag
+ command.command.Flags().StringVar(&command.flags.NodeKeyHex, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withCrawlerConcurrency() {
+ flag := cli.UintFlag{
+ Name: "crawler-concurrency",
+ Usage: "A number of maximum parallel node interrogations",
+ Value: uint(runtime.GOMAXPROCS(-1)) * 10,
+ }
+ command.command.Flags().UintVar(&command.flags.CrawlerConcurrency, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withRefreshTimeout() {
+ flag := cli.DurationFlag{
+ Name: "refresh-timeout",
+ Usage: "A timeout to wait before considering to re-crawl a node",
+ Value: 2 * 24 * time.Hour,
+ }
+ command.command.Flags().DurationVar(&command.flags.RefreshTimeout, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withMaxPingTries() {
+ flag := cli.UintFlag{
+ Name: "max-ping-tries",
+ Usage: "How many times to try PING before applying exponential back-off logic",
+ Value: 3,
+ }
+ command.command.Flags().UintVar(&command.flags.MaxPingTries, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withKeygenTimeout() {
+ flag := cli.DurationFlag{
+ Name: "keygen-timeout",
+ Usage: "How much time can be used to generate node bucket keys",
+ Value: 2 * time.Second,
+ }
+ command.command.Flags().DurationVar(&command.flags.KeygenTimeout, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withKeygenConcurrency() {
+ flag := cli.UintFlag{
+ Name: "keygen-concurrency",
+ Usage: "How many parallel goroutines can be used by the node bucket keys generator",
+ Value: 2,
+ }
+ command.command.Flags().UintVar(&command.flags.KeygenConcurrency, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withHandshakeRefreshTimeout() {
+ flag := cli.DurationFlag{
+ Name: "handshake-refresh-timeout",
+ Usage: "When a node's handshake data is considered expired and needs to be re-crawled",
+ Value: 20 * 24 * time.Hour,
+ }
+ command.command.Flags().DurationVar(&command.flags.HandshakeRefreshTimeout, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withHandshakeRetryDelay() {
+ flag := cli.DurationFlag{
+ Name: "handshake-retry-delay",
+ Usage: "How long to wait before retrying a failed handshake",
+ Value: 4 * time.Hour,
+ }
+ command.command.Flags().DurationVar(&command.flags.HandshakeRetryDelay, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withHandshakeMaxTries() {
+ flag := cli.UintFlag{
+ Name: "handshake-max-tries",
+ Usage: "How many times to retry handshake before abandoning a candidate",
+ Value: 3,
+ }
+ command.command.Flags().UintVar(&command.flags.HandshakeMaxTries, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) ExecuteContext(ctx context.Context, runFunc func(ctx context.Context, flags CommandFlags) error) error {
+ command.command.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
+ // apply debug flags
+ return utils.SetupCobra(cmd)
+ }
+ command.command.PersistentPostRun = func(cmd *cobra.Command, args []string) {
+ debug.Exit()
+ }
+ command.command.RunE = func(cmd *cobra.Command, args []string) error {
+ defer debug.Exit()
+ err := runFunc(cmd.Context(), command.flags)
+ if errors.Is(err, context.Canceled) {
+ return nil
+ }
+ return err
+ }
+ return command.command.ExecuteContext(ctx)
+}
+
+func (command *Command) AddSubCommand(subCommand *cobra.Command) {
+ command.command.AddCommand(subCommand)
+}
+
+func must(err error) {
+ if err != nil {
+ panic(err)
+ }
+}
diff --git a/cmd/observer/observer/crawler.go b/cmd/observer/observer/crawler.go
new file mode 100644
index 0000000000..07005dc368
--- /dev/null
+++ b/cmd/observer/observer/crawler.go
@@ -0,0 +1,495 @@
+package observer
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "errors"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "github.com/ledgerwatch/erigon/cmd/observer/utils"
+ "github.com/ledgerwatch/erigon/core/forkid"
+ "github.com/ledgerwatch/erigon/p2p/enode"
+ "github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/log/v3"
+ "golang.org/x/sync/semaphore"
+ "sync/atomic"
+ "time"
+)
+
+type Crawler struct {
+ transport DiscV4Transport
+
+ db database.DBRetrier
+ saveQueue *utils.TaskQueue
+
+ config CrawlerConfig
+ forkFilter forkid.Filter
+
+ diplomacy *Diplomacy
+
+ log log.Logger
+}
+
+type CrawlerConfig struct {
+ Chain string
+ Bootnodes []*enode.Node
+ PrivateKey *ecdsa.PrivateKey
+ ConcurrencyLimit uint
+ RefreshTimeout time.Duration
+ MaxPingTries uint
+ StatusLogPeriod time.Duration
+
+ HandshakeRefreshTimeout time.Duration
+ HandshakeRetryDelay time.Duration
+ HandshakeMaxTries uint
+
+ KeygenTimeout time.Duration
+ KeygenConcurrency uint
+}
+
+func NewCrawler(
+ transport DiscV4Transport,
+ db database.DB,
+ config CrawlerConfig,
+ logger log.Logger,
+) (*Crawler, error) {
+ saveQueueLogFuncProvider := func(err error) func(msg string, ctx ...interface{}) {
+ if db.IsConflictError(err) {
+ return logger.Warn
+ }
+ return logger.Error
+ }
+ saveQueue := utils.NewTaskQueue("Crawler.saveQueue", config.ConcurrencyLimit*2, saveQueueLogFuncProvider)
+
+ chain := config.Chain
+ chainConfig := params.ChainConfigByChainName(chain)
+ genesisHash := params.GenesisHashByChainName(chain)
+ if (chainConfig == nil) || (genesisHash == nil) {
+ return nil, fmt.Errorf("unknown chain %s", chain)
+ }
+
+ forkFilter := forkid.NewStaticFilter(chainConfig, *genesisHash)
+
+ diplomacy := NewDiplomacy(
+ database.NewDBRetrier(db, logger),
+ saveQueue,
+ config.PrivateKey,
+ config.ConcurrencyLimit,
+ config.HandshakeRefreshTimeout,
+ config.HandshakeRetryDelay,
+ config.HandshakeMaxTries,
+ config.StatusLogPeriod,
+ logger)
+
+ instance := Crawler{
+ transport,
+ database.NewDBRetrier(db, logger),
+ saveQueue,
+ config,
+ forkFilter,
+ diplomacy,
+ logger,
+ }
+ return &instance, nil
+}
+
+func (crawler *Crawler) startSaveQueue(ctx context.Context) {
+ go crawler.saveQueue.Run(ctx)
+}
+
+func (crawler *Crawler) startDiplomacy(ctx context.Context) {
+ go func() {
+ err := crawler.diplomacy.Run(ctx)
+ if (err != nil) && !errors.Is(err, context.Canceled) {
+ crawler.log.Error("Diplomacy has failed", "err", err)
+ }
+ }()
+}
+
+type candidateNode struct {
+ id database.NodeID
+ node *enode.Node
+}
+
+func (crawler *Crawler) startSelectCandidates(ctx context.Context) <-chan candidateNode {
+ nodes := make(chan candidateNode)
+ go func() {
+ err := crawler.selectCandidates(ctx, nodes)
+ if (err != nil) && !errors.Is(err, context.Canceled) {
+ crawler.log.Error("Failed to select candidates", "err", err)
+ }
+ close(nodes)
+ }()
+ return nodes
+}
+
+func (crawler *Crawler) selectCandidates(ctx context.Context, nodes chan<- candidateNode) error {
+ for _, node := range crawler.config.Bootnodes {
+ id, err := nodeID(node)
+ if err != nil {
+ return fmt.Errorf("failed to get a bootnode ID: %w", err)
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case nodes <- candidateNode{id, node}:
+ }
+ }
+
+ for ctx.Err() == nil {
+ candidates, err := crawler.db.TakeCandidates(
+ ctx,
+ crawler.config.ConcurrencyLimit)
+ if err != nil {
+ if crawler.db.IsConflictError(err) {
+ crawler.log.Warn("Failed to take candidates", "err", err)
+ } else {
+ return err
+ }
+ }
+
+ if len(candidates) == 0 {
+ utils.Sleep(ctx, 1*time.Second)
+ }
+
+ for _, id := range candidates {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case nodes <- candidateNode{id, nil}:
+ }
+ }
+ }
+
+ return ctx.Err()
+}
+
+func (crawler *Crawler) Run(ctx context.Context) error {
+ crawler.startSaveQueue(ctx)
+ crawler.startDiplomacy(ctx)
+
+ nodes := crawler.startSelectCandidates(ctx)
+ sem := semaphore.NewWeighted(int64(crawler.config.ConcurrencyLimit))
+ // allow only 1 keygen at a time
+ keygenSem := semaphore.NewWeighted(int64(1))
+
+ crawledCount := 0
+ crawledCountLogDate := time.Now()
+ foundPeersCountPtr := new(uint64)
+
+ for candidate := range nodes {
+ if err := sem.Acquire(ctx, 1); err != nil {
+ if !errors.Is(err, context.Canceled) {
+ return fmt.Errorf("failed to acquire semaphore: %w", err)
+ } else {
+ break
+ }
+ }
+
+ crawledCount++
+ if time.Since(crawledCountLogDate) > crawler.config.StatusLogPeriod {
+ foundPeersCount := atomic.LoadUint64(foundPeersCountPtr)
+
+ remainingCount, err := crawler.db.CountCandidates(ctx)
+ if err != nil {
+ if crawler.db.IsConflictError(err) {
+ crawler.log.Warn("Failed to count candidates", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to count candidates: %w", err)
+ }
+
+ crawler.log.Info(
+ "Crawling",
+ "crawled", crawledCount,
+ "remaining", remainingCount,
+ "foundPeers", foundPeersCount,
+ )
+ crawledCountLogDate = time.Now()
+ }
+
+ id := candidate.id
+ node := candidate.node
+
+ if node == nil {
+ nodeAddr, err := crawler.db.FindNodeAddr(ctx, id)
+ if err != nil {
+ if crawler.db.IsConflictError(err) {
+ crawler.log.Warn("Failed to get the node address", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to get the node address: %w", err)
+ }
+
+ node, err = makeNodeFromAddr(id, *nodeAddr)
+ if err != nil {
+ return fmt.Errorf("failed to make node from node address: %w", err)
+ }
+ }
+
+ nodeDesc := node.URLv4()
+ logger := crawler.log.New("node", nodeDesc)
+
+ prevPingTries, err := crawler.db.CountPingErrors(ctx, id)
+ if err != nil {
+ if crawler.db.IsConflictError(err) {
+ crawler.log.Warn("Failed to count ping errors", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to count ping errors: %w", err)
+ }
+
+ handshakeNextRetryTime, err := crawler.db.FindHandshakeRetryTime(ctx, id)
+ if err != nil {
+ if crawler.db.IsConflictError(err) {
+ crawler.log.Warn("Failed to get handshake next retry time", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to get handshake next retry time: %w", err)
+ }
+
+ handshakeLastErrors, err := crawler.db.FindHandshakeLastErrors(ctx, id, crawler.config.HandshakeMaxTries)
+ if err != nil {
+ if crawler.db.IsConflictError(err) {
+ crawler.log.Warn("Failed to get handshake last errors", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to get handshake last errors: %w", err)
+ }
+
+ diplomat := NewDiplomat(
+ node,
+ crawler.config.PrivateKey,
+ handshakeLastErrors,
+ crawler.config.HandshakeRefreshTimeout,
+ crawler.config.HandshakeRetryDelay,
+ crawler.config.HandshakeMaxTries,
+ logger)
+
+ keygenCachedHexKeys, err := crawler.db.FindNeighborBucketKeys(ctx, id)
+ if err != nil {
+ if crawler.db.IsConflictError(err) {
+ crawler.log.Warn("Failed to get neighbor bucket keys", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to get neighbor bucket keys: %w", err)
+ }
+ keygenCachedKeys, err := parseHexPublicKeys(keygenCachedHexKeys)
+ if err != nil {
+ return fmt.Errorf("failed to parse cached neighbor bucket keys: %w", err)
+ }
+
+ interrogator, err := NewInterrogator(
+ node,
+ crawler.transport,
+ crawler.forkFilter,
+ diplomat,
+ handshakeNextRetryTime,
+ crawler.config.KeygenTimeout,
+ crawler.config.KeygenConcurrency,
+ keygenSem,
+ keygenCachedKeys,
+ logger)
+ if err != nil {
+ return fmt.Errorf("failed to create Interrogator for node %s: %w", nodeDesc, err)
+ }
+
+ go func() {
+ defer sem.Release(1)
+
+ result, err := interrogator.Run(ctx)
+
+ isPingError := (err != nil) && (err.id == InterrogationErrorPing)
+ nextRetryTime := crawler.nextRetryTime(isPingError, *prevPingTries)
+
+ var isCompatFork *bool
+ if result != nil {
+ isCompatFork = result.IsCompatFork
+ } else if (err != nil) &&
+ ((err.id == InterrogationErrorIncompatibleForkID) ||
+ (err.id == InterrogationErrorBlacklistedClientID)) {
+ isCompatFork = new(bool)
+ *isCompatFork = false
+ }
+
+ var clientID *string
+ var handshakeRetryTime *time.Time
+ if (result != nil) && (result.HandshakeResult != nil) {
+ clientID = result.HandshakeResult.ClientID
+ handshakeRetryTime = result.HandshakeRetryTime
+ } else if (err != nil) && (err.id == InterrogationErrorBlacklistedClientID) {
+ clientID = new(string)
+ *clientID = err.wrappedErr.Error()
+ handshakeRetryTime = new(time.Time)
+ *handshakeRetryTime = time.Now().Add(crawler.config.HandshakeRefreshTimeout)
+ }
+
+ if err != nil {
+ if !errors.Is(err, context.Canceled) {
+ var logFunc func(msg string, ctx ...interface{})
+ switch err.id {
+ case InterrogationErrorPing:
+ fallthrough
+ case InterrogationErrorIncompatibleForkID:
+ fallthrough
+ case InterrogationErrorBlacklistedClientID:
+ fallthrough
+ case InterrogationErrorFindNodeTimeout:
+ logFunc = logger.Debug
+ default:
+ logFunc = logger.Warn
+ }
+ logFunc("Failed to interrogate node", "err", err)
+ }
+ }
+
+ if result != nil {
+ peers := result.Peers
+ logger.Debug(fmt.Sprintf("Got %d peers", len(peers)))
+ atomic.AddUint64(foundPeersCountPtr, uint64(len(peers)))
+ }
+
+ crawler.saveQueue.EnqueueTask(ctx, func(ctx context.Context) error {
+ return crawler.saveInterrogationResult(
+ ctx,
+ id,
+ result,
+ isPingError,
+ isCompatFork,
+ clientID,
+ handshakeRetryTime,
+ nextRetryTime)
+ })
+ }()
+ }
+ return nil
+}
+
+func (crawler *Crawler) saveInterrogationResult(
+ ctx context.Context,
+ id database.NodeID,
+ result *InterrogationResult,
+ isPingError bool,
+ isCompatFork *bool,
+ clientID *string,
+ handshakeRetryTime *time.Time,
+ nextRetryTime time.Time,
+) error {
+ var peers []*enode.Node
+ if result != nil {
+ peers = result.Peers
+ }
+
+ for _, peer := range peers {
+ peerID, err := nodeID(peer)
+ if err != nil {
+ return fmt.Errorf("failed to get peer node ID: %w", err)
+ }
+
+ dbErr := crawler.db.UpsertNodeAddr(ctx, peerID, makeNodeAddr(peer))
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if (result != nil) && (len(result.KeygenKeys) >= 15) {
+ keygenHexKeys := hexEncodePublicKeys(result.KeygenKeys)
+ dbErr := crawler.db.UpdateNeighborBucketKeys(ctx, id, keygenHexKeys)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if isPingError {
+ dbErr := crawler.db.UpdatePingError(ctx, id)
+ if dbErr != nil {
+ return dbErr
+ }
+ } else {
+ dbErr := crawler.db.ResetPingError(ctx, id)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if isCompatFork != nil {
+ dbErr := crawler.db.UpdateForkCompatibility(ctx, id, *isCompatFork)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if clientID != nil {
+ dbErr := crawler.db.UpdateClientID(ctx, id, *clientID)
+ if dbErr != nil {
+ return dbErr
+ }
+
+ dbErr = crawler.db.DeleteHandshakeErrors(ctx, id)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if (result != nil) && (result.HandshakeResult != nil) && (result.HandshakeResult.NetworkID != nil) {
+ dbErr := crawler.db.UpdateNetworkID(ctx, id, uint(*result.HandshakeResult.NetworkID))
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if (result != nil) && (result.HandshakeResult != nil) && (result.HandshakeResult.EthVersion != nil) {
+ dbErr := crawler.db.UpdateEthVersion(ctx, id, uint(*result.HandshakeResult.EthVersion))
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if (result != nil) && (result.HandshakeResult != nil) && (result.HandshakeResult.HandshakeErr != nil) {
+ dbErr := crawler.db.InsertHandshakeError(ctx, id, result.HandshakeResult.HandshakeErr.StringCode())
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if (result != nil) && (result.HandshakeResult != nil) {
+ dbErr := crawler.db.UpdateHandshakeTransientError(ctx, id, result.HandshakeResult.HasTransientErr)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if handshakeRetryTime != nil {
+ dbErr := crawler.db.UpdateHandshakeRetryTime(ctx, id, *handshakeRetryTime)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ return crawler.db.UpdateCrawlRetryTime(ctx, id, nextRetryTime)
+}
+
+func (crawler *Crawler) nextRetryTime(isPingError bool, prevPingTries uint) time.Time {
+ return time.Now().Add(crawler.nextRetryDelay(isPingError, prevPingTries))
+}
+
+func (crawler *Crawler) nextRetryDelay(isPingError bool, prevPingTries uint) time.Duration {
+ if !isPingError {
+ return crawler.config.RefreshTimeout
+ }
+
+ pingTries := prevPingTries + 1
+ if pingTries < crawler.config.MaxPingTries {
+ return crawler.config.RefreshTimeout
+ }
+
+ // back off: double for each next retry
+ return crawler.config.RefreshTimeout << (pingTries - crawler.config.MaxPingTries + 1)
+}
diff --git a/cmd/observer/observer/diplomacy.go b/cmd/observer/observer/diplomacy.go
new file mode 100644
index 0000000000..35765732f5
--- /dev/null
+++ b/cmd/observer/observer/diplomacy.go
@@ -0,0 +1,252 @@
+package observer
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "errors"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "github.com/ledgerwatch/erigon/cmd/observer/utils"
+ "github.com/ledgerwatch/log/v3"
+ "golang.org/x/sync/semaphore"
+ "sync/atomic"
+ "time"
+)
+
+type Diplomacy struct {
+ db database.DBRetrier
+ saveQueue *utils.TaskQueue
+
+ privateKey *ecdsa.PrivateKey
+ concurrencyLimit uint
+ refreshTimeout time.Duration
+ retryDelay time.Duration
+ maxHandshakeTries uint
+
+ statusLogPeriod time.Duration
+ log log.Logger
+}
+
+func NewDiplomacy(
+ db database.DBRetrier,
+ saveQueue *utils.TaskQueue,
+ privateKey *ecdsa.PrivateKey,
+ concurrencyLimit uint,
+ refreshTimeout time.Duration,
+ retryDelay time.Duration,
+ maxHandshakeTries uint,
+ statusLogPeriod time.Duration,
+ logger log.Logger,
+) *Diplomacy {
+ instance := Diplomacy{
+ db,
+ saveQueue,
+ privateKey,
+ concurrencyLimit,
+ refreshTimeout,
+ retryDelay,
+ maxHandshakeTries,
+ statusLogPeriod,
+ logger,
+ }
+ return &instance
+}
+
+func (diplomacy *Diplomacy) startSelectCandidates(ctx context.Context) <-chan database.NodeID {
+ candidatesChan := make(chan database.NodeID)
+ go func() {
+ err := diplomacy.selectCandidates(ctx, candidatesChan)
+ if (err != nil) && !errors.Is(err, context.Canceled) {
+ diplomacy.log.Error("Failed to select handshake candidates", "err", err)
+ }
+ close(candidatesChan)
+ }()
+ return candidatesChan
+}
+
+func (diplomacy *Diplomacy) selectCandidates(ctx context.Context, candidatesChan chan<- database.NodeID) error {
+ for ctx.Err() == nil {
+ candidates, err := diplomacy.db.TakeHandshakeCandidates(
+ ctx,
+ diplomacy.concurrencyLimit)
+ if err != nil {
+ if diplomacy.db.IsConflictError(err) {
+ diplomacy.log.Warn("Failed to take handshake candidates", "err", err)
+ } else {
+ return err
+ }
+ }
+
+ if len(candidates) == 0 {
+ utils.Sleep(ctx, 1*time.Second)
+ }
+
+ for _, id := range candidates {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case candidatesChan <- id:
+ }
+ }
+ }
+
+ return ctx.Err()
+}
+
+func (diplomacy *Diplomacy) Run(ctx context.Context) error {
+ candidatesChan := diplomacy.startSelectCandidates(ctx)
+ sem := semaphore.NewWeighted(int64(diplomacy.concurrencyLimit))
+
+ doneCount := 0
+ statusLogDate := time.Now()
+ clientIDCountPtr := new(uint64)
+
+ for id := range candidatesChan {
+ if err := sem.Acquire(ctx, 1); err != nil {
+ if !errors.Is(err, context.Canceled) {
+ return fmt.Errorf("failed to acquire semaphore: %w", err)
+ } else {
+ break
+ }
+ }
+
+ doneCount++
+ if time.Since(statusLogDate) > diplomacy.statusLogPeriod {
+ clientIDCount := atomic.LoadUint64(clientIDCountPtr)
+
+ remainingCount, err := diplomacy.db.CountHandshakeCandidates(ctx)
+ if err != nil {
+ if diplomacy.db.IsConflictError(err) {
+ diplomacy.log.Warn("Failed to count handshake candidates", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to count handshake candidates: %w", err)
+ }
+
+ diplomacy.log.Info(
+ "Handshaking",
+ "done", doneCount,
+ "remaining", remainingCount,
+ "clientIDs", clientIDCount,
+ )
+ statusLogDate = time.Now()
+ }
+
+ nodeAddr, err := diplomacy.db.FindNodeAddr(ctx, id)
+ if err != nil {
+ if diplomacy.db.IsConflictError(err) {
+ diplomacy.log.Warn("Failed to get the node address", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to get the node address: %w", err)
+ }
+
+ node, err := makeNodeFromAddr(id, *nodeAddr)
+ if err != nil {
+ return fmt.Errorf("failed to make node from node address: %w", err)
+ }
+
+ nodeDesc := node.URLv4()
+ logger := diplomacy.log.New("node", nodeDesc)
+
+ handshakeLastErrors, err := diplomacy.db.FindHandshakeLastErrors(ctx, id, diplomacy.maxHandshakeTries)
+ if err != nil {
+ if diplomacy.db.IsConflictError(err) {
+ diplomacy.log.Warn("Failed to get handshake last errors", "err", err)
+ sem.Release(1)
+ continue
+ }
+ return fmt.Errorf("failed to get handshake last errors: %w", err)
+ }
+
+ diplomat := NewDiplomat(
+ node,
+ diplomacy.privateKey,
+ handshakeLastErrors,
+ diplomacy.refreshTimeout,
+ diplomacy.retryDelay,
+ diplomacy.maxHandshakeTries,
+ logger)
+
+ go func(id database.NodeID) {
+ defer sem.Release(1)
+
+ result := diplomat.Run(ctx)
+ clientID := result.ClientID
+
+ if clientID != nil {
+ atomic.AddUint64(clientIDCountPtr, 1)
+ }
+
+ var isCompatFork *bool
+ if (clientID != nil) && IsClientIDBlacklisted(*clientID) {
+ isCompatFork = new(bool)
+ *isCompatFork = false
+ }
+
+ nextRetryTime := diplomat.NextRetryTime(result.HandshakeErr)
+
+ diplomacy.saveQueue.EnqueueTask(ctx, func(ctx context.Context) error {
+ return diplomacy.saveDiplomatResult(ctx, id, result, isCompatFork, nextRetryTime)
+ })
+ }(id)
+ }
+ return nil
+}
+
+func (diplomacy *Diplomacy) saveDiplomatResult(
+ ctx context.Context,
+ id database.NodeID,
+ result DiplomatResult,
+ isCompatFork *bool,
+ nextRetryTime time.Time,
+) error {
+ if result.ClientID != nil {
+ dbErr := diplomacy.db.UpdateClientID(ctx, id, *result.ClientID)
+ if dbErr != nil {
+ return dbErr
+ }
+
+ dbErr = diplomacy.db.DeleteHandshakeErrors(ctx, id)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if result.NetworkID != nil {
+ dbErr := diplomacy.db.UpdateNetworkID(ctx, id, uint(*result.NetworkID))
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if result.EthVersion != nil {
+ dbErr := diplomacy.db.UpdateEthVersion(ctx, id, uint(*result.EthVersion))
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ if result.HandshakeErr != nil {
+ dbErr := diplomacy.db.InsertHandshakeError(ctx, id, result.HandshakeErr.StringCode())
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ dbErr := diplomacy.db.UpdateHandshakeTransientError(ctx, id, result.HasTransientErr)
+ if dbErr != nil {
+ return dbErr
+ }
+
+ if isCompatFork != nil {
+ dbErr := diplomacy.db.UpdateForkCompatibility(ctx, id, *isCompatFork)
+ if dbErr != nil {
+ return dbErr
+ }
+ }
+
+ return diplomacy.db.UpdateHandshakeRetryTime(ctx, id, nextRetryTime)
+}
diff --git a/cmd/observer/observer/diplomat.go b/cmd/observer/observer/diplomat.go
new file mode 100644
index 0000000000..9812b3bc03
--- /dev/null
+++ b/cmd/observer/observer/diplomat.go
@@ -0,0 +1,150 @@
+package observer
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "errors"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "github.com/ledgerwatch/erigon/p2p"
+ "github.com/ledgerwatch/erigon/p2p/enode"
+ "github.com/ledgerwatch/log/v3"
+ "time"
+)
+
+type Diplomat struct {
+ node *enode.Node
+ privateKey *ecdsa.PrivateKey
+
+ handshakeLastErrors []database.HandshakeError
+ handshakeRefreshTimeout time.Duration
+ handshakeRetryDelay time.Duration
+ handshakeMaxTries uint
+
+ log log.Logger
+}
+
+type DiplomatResult struct {
+ ClientID *string
+ NetworkID *uint64
+ EthVersion *uint32
+ HandshakeErr *HandshakeError
+ HasTransientErr bool
+}
+
+func NewDiplomat(
+ node *enode.Node,
+ privateKey *ecdsa.PrivateKey,
+ handshakeLastErrors []database.HandshakeError,
+ handshakeRefreshTimeout time.Duration,
+ handshakeRetryDelay time.Duration,
+ handshakeMaxTries uint,
+ logger log.Logger,
+) *Diplomat {
+ instance := Diplomat{
+ node,
+ privateKey,
+ handshakeLastErrors,
+ handshakeRefreshTimeout,
+ handshakeRetryDelay,
+ handshakeMaxTries,
+ logger,
+ }
+ return &instance
+}
+
+func (diplomat *Diplomat) handshake(ctx context.Context) (*HelloMessage, *StatusMessage, *HandshakeError) {
+ node := diplomat.node
+ return Handshake(ctx, node.IP(), node.TCP(), node.Pubkey(), diplomat.privateKey)
+}
+
+func (diplomat *Diplomat) Run(ctx context.Context) DiplomatResult {
+ diplomat.log.Debug("Handshaking with a node")
+ hello, status, handshakeErr := diplomat.handshake(ctx)
+
+ var result DiplomatResult
+
+ if (handshakeErr != nil) && !errors.Is(handshakeErr, context.Canceled) {
+ result.HandshakeErr = handshakeErr
+ diplomat.log.Debug("Failed to handshake", "err", handshakeErr)
+ }
+ result.HasTransientErr = diplomat.hasRecentTransientError(handshakeErr)
+
+ if hello != nil {
+ result.ClientID = &hello.ClientID
+ diplomat.log.Debug("Got client ID", "clientID", *result.ClientID)
+ }
+
+ if status != nil {
+ result.NetworkID = &status.NetworkID
+ diplomat.log.Debug("Got network ID", "networkID", *result.NetworkID)
+ }
+ if status != nil {
+ result.EthVersion = &status.ProtocolVersion
+ diplomat.log.Debug("Got eth version", "ethVersion", *result.EthVersion)
+ }
+
+ return result
+}
+
+func (diplomat *Diplomat) NextRetryTime(handshakeErr *HandshakeError) time.Time {
+ return time.Now().Add(diplomat.NextRetryDelay(handshakeErr))
+}
+
+func (diplomat *Diplomat) NextRetryDelay(handshakeErr *HandshakeError) time.Duration {
+ if handshakeErr == nil {
+ return diplomat.handshakeRefreshTimeout
+ }
+
+ dbHandshakeErr := database.HandshakeError{
+ StringCode: handshakeErr.StringCode(),
+ Time: time.Now(),
+ }
+
+ lastErrors := append([]database.HandshakeError{dbHandshakeErr}, diplomat.handshakeLastErrors...)
+
+ if uint(len(lastErrors)) < diplomat.handshakeMaxTries {
+ return diplomat.handshakeRetryDelay
+ }
+
+ if containsHandshakeError(diplomat.transientError(), lastErrors) {
+ return diplomat.handshakeRetryDelay
+ }
+
+ if len(lastErrors) < 2 {
+ return 1000000 * time.Hour // never
+ }
+
+ backOffDelay := 2 * lastErrors[0].Time.Sub(lastErrors[1].Time)
+ if backOffDelay < diplomat.handshakeRetryDelay {
+ return diplomat.handshakeRetryDelay
+ }
+
+ return backOffDelay
+}
+
+func (diplomat *Diplomat) transientError() *HandshakeError {
+ return NewHandshakeError(HandshakeErrorIDDisconnect, p2p.DiscTooManyPeers, uint64(p2p.DiscTooManyPeers))
+}
+
+func (diplomat *Diplomat) hasRecentTransientError(handshakeErr *HandshakeError) bool {
+ if handshakeErr == nil {
+ return false
+ }
+
+ dbHandshakeErr := database.HandshakeError{
+ StringCode: handshakeErr.StringCode(),
+ Time: time.Now(),
+ }
+
+ lastErrors := append([]database.HandshakeError{dbHandshakeErr}, diplomat.handshakeLastErrors...)
+ return containsHandshakeError(diplomat.transientError(), lastErrors)
+}
+
+func containsHandshakeError(target *HandshakeError, list []database.HandshakeError) bool {
+ for _, err := range list {
+ if err.StringCode == target.StringCode() {
+ return true
+ }
+ }
+ return false
+}
diff --git a/cmd/observer/observer/handshake.go b/cmd/observer/observer/handshake.go
new file mode 100644
index 0000000000..6fdcd3414a
--- /dev/null
+++ b/cmd/observer/observer/handshake.go
@@ -0,0 +1,252 @@
+package observer
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "fmt"
+ "github.com/ledgerwatch/erigon/common"
+ "github.com/ledgerwatch/erigon/core/forkid"
+ "github.com/ledgerwatch/erigon/crypto"
+ "github.com/ledgerwatch/erigon/eth/protocols/eth"
+ "github.com/ledgerwatch/erigon/p2p"
+ "github.com/ledgerwatch/erigon/p2p/rlpx"
+ "github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/erigon/rlp"
+ "math/big"
+ "net"
+ "strings"
+ "time"
+)
+
+// https://github.com/ethereum/devp2p/blob/master/rlpx.md#p2p-capability
+const (
+ RLPxMessageIDHello = 0
+ RLPxMessageIDDisconnect = 1
+ RLPxMessageIDPing = 2
+ RLPxMessageIDPong = 3
+)
+
+// HelloMessage is the RLPx Hello message.
+// (same as protoHandshake in p2p/peer.go)
+// https://github.com/ethereum/devp2p/blob/master/rlpx.md#hello-0x00
+type HelloMessage struct {
+ Version uint64
+ ClientID string
+ Caps []p2p.Cap
+ ListenPort uint64
+ Pubkey []byte // secp256k1 public key
+
+ // Ignore additional fields (for forward compatibility).
+ Rest []rlp.RawValue `rlp:"tail"`
+}
+
+// StatusMessage is the Ethereum Status message v63+.
+// (same as StatusPacket in eth/protocols/eth/protocol.go)
+// https://github.com/ethereum/devp2p/blob/master/caps/eth.md#status-0x00
+type StatusMessage struct {
+ ProtocolVersion uint32
+ NetworkID uint64
+ TD *big.Int
+ Head common.Hash
+ Genesis common.Hash
+ ForkID *forkid.ID `rlp:"-"` // parsed from Rest if exists in v64+
+ Rest []rlp.RawValue `rlp:"tail"`
+}
+
+type HandshakeErrorID string
+
+const (
+ HandshakeErrorIDConnect HandshakeErrorID = "connect"
+ HandshakeErrorIDSetTimeout HandshakeErrorID = "set-timeout"
+ HandshakeErrorIDAuth HandshakeErrorID = "auth"
+ HandshakeErrorIDRead HandshakeErrorID = "read"
+ HandshakeErrorIDUnexpectedMessage HandshakeErrorID = "unexpected-message"
+ HandshakeErrorIDDisconnectDecode HandshakeErrorID = "disconnect-decode"
+ HandshakeErrorIDDisconnect HandshakeErrorID = "disconnect"
+ HandshakeErrorIDHelloEncode HandshakeErrorID = "hello-encode"
+ HandshakeErrorIDHelloDecode HandshakeErrorID = "hello-decode"
+ HandshakeErrorIDStatusDecode HandshakeErrorID = "status-decode"
+)
+
+type HandshakeError struct {
+ id HandshakeErrorID
+ wrappedErr error
+ param uint64
+}
+
+func NewHandshakeError(id HandshakeErrorID, wrappedErr error, param uint64) *HandshakeError {
+ instance := HandshakeError{
+ id,
+ wrappedErr,
+ param,
+ }
+ return &instance
+}
+
+func (e *HandshakeError) Unwrap() error {
+ return e.wrappedErr
+}
+
+func (e *HandshakeError) Error() string {
+ switch e.id {
+ case HandshakeErrorIDConnect:
+ return fmt.Sprintf("handshake failed to connect: %v", e.wrappedErr)
+ case HandshakeErrorIDSetTimeout:
+ return fmt.Sprintf("handshake failed to set timeout: %v", e.wrappedErr)
+ case HandshakeErrorIDAuth:
+ return fmt.Sprintf("handshake RLPx auth failed: %v", e.wrappedErr)
+ case HandshakeErrorIDRead:
+ return fmt.Sprintf("handshake RLPx read failed: %v", e.wrappedErr)
+ case HandshakeErrorIDUnexpectedMessage:
+ return fmt.Sprintf("handshake got unexpected message ID: %d", e.param)
+ case HandshakeErrorIDDisconnectDecode:
+ return fmt.Sprintf("handshake failed to parse disconnect reason: %v", e.wrappedErr)
+ case HandshakeErrorIDDisconnect:
+ return fmt.Sprintf("handshake got disconnected: %v", e.wrappedErr)
+ case HandshakeErrorIDHelloEncode:
+ return fmt.Sprintf("handshake failed to encode outgoing Hello message: %v", e.wrappedErr)
+ case HandshakeErrorIDHelloDecode:
+ return fmt.Sprintf("handshake failed to parse Hello message: %v", e.wrappedErr)
+ case HandshakeErrorIDStatusDecode:
+ return fmt.Sprintf("handshake failed to parse Status message: %v", e.wrappedErr)
+ default:
+ return "<unhandled HandshakeErrorID>"
+ }
+}
+
+func (e *HandshakeError) StringCode() string {
+ switch e.id {
+ case HandshakeErrorIDUnexpectedMessage:
+ fallthrough
+ case HandshakeErrorIDDisconnect:
+ return fmt.Sprintf("%s-%d", e.id, e.param)
+ default:
+ return string(e.id)
+ }
+}
+
+func Handshake(
+ ctx context.Context,
+ ip net.IP,
+ rlpxPort int,
+ pubkey *ecdsa.PublicKey,
+ myPrivateKey *ecdsa.PrivateKey,
+) (*HelloMessage, *StatusMessage, *HandshakeError) {
+ connectTimeout := 10 * time.Second
+ dialer := net.Dialer{
+ Timeout: connectTimeout,
+ }
+ addr := net.TCPAddr{IP: ip, Port: rlpxPort}
+
+ tcpConn, err := dialer.DialContext(ctx, "tcp", addr.String())
+ if err != nil {
+ return nil, nil, NewHandshakeError(HandshakeErrorIDConnect, err, 0)
+ }
+
+ conn := rlpx.NewConn(tcpConn, pubkey)
+ defer func() { _ = conn.Close() }()
+
+ handshakeTimeout := 5 * time.Second
+ handshakeDeadline := time.Now().Add(handshakeTimeout)
+ err = conn.SetDeadline(handshakeDeadline)
+ if err != nil {
+ return nil, nil, NewHandshakeError(HandshakeErrorIDSetTimeout, err, 0)
+ }
+ err = conn.SetWriteDeadline(handshakeDeadline)
+ if err != nil {
+ return nil, nil, NewHandshakeError(HandshakeErrorIDSetTimeout, err, 0)
+ }
+
+ _, err = conn.Handshake(myPrivateKey)
+ if err != nil {
+ return nil, nil, NewHandshakeError(HandshakeErrorIDAuth, err, 0)
+ }
+
+ ourHelloMessage := makeOurHelloMessage(myPrivateKey)
+ ourHelloData, err := rlp.EncodeToBytes(&ourHelloMessage)
+ if err != nil {
+ return nil, nil, NewHandshakeError(HandshakeErrorIDHelloEncode, err, 0)
+ }
+ go func() { _, _ = conn.Write(RLPxMessageIDHello, ourHelloData) }()
+
+ var helloMessage HelloMessage
+ if err := readMessage(conn, RLPxMessageIDHello, HandshakeErrorIDHelloDecode, &helloMessage); err != nil {
+ return nil, nil, err
+ }
+
+ // All messages following Hello are compressed using the Snappy algorithm.
+ if helloMessage.Version >= 5 {
+ conn.SetSnappy(true)
+ }
+
+ var statusMessage StatusMessage
+ if err := readMessage(conn, 16+eth.StatusMsg, HandshakeErrorIDStatusDecode, &statusMessage); err != nil {
+ return &helloMessage, nil, err
+ }
+
+ // parse fork ID
+ if (statusMessage.ProtocolVersion >= 64) && (len(statusMessage.Rest) > 0) {
+ var forkID forkid.ID
+ if err := rlp.DecodeBytes(statusMessage.Rest[0], &forkID); err != nil {
+ return &helloMessage, nil, NewHandshakeError(HandshakeErrorIDStatusDecode, err, 0)
+ }
+ statusMessage.ForkID = &forkID
+ }
+
+ return &helloMessage, &statusMessage, nil
+}
+
+func readMessage(conn *rlpx.Conn, expectedMessageID uint64, decodeError HandshakeErrorID, message interface{}) *HandshakeError {
+ messageID, data, _, err := conn.Read()
+ if err != nil {
+ return NewHandshakeError(HandshakeErrorIDRead, err, 0)
+ }
+
+ if messageID == RLPxMessageIDPing {
+ pongData, _ := rlp.EncodeToBytes(make([]string, 0, 1))
+ go func() { _, _ = conn.Write(RLPxMessageIDPong, pongData) }()
+ return readMessage(conn, expectedMessageID, decodeError, message)
+ }
+ if messageID == 16+eth.GetPooledTransactionsMsg {
+ return readMessage(conn, expectedMessageID, decodeError, message)
+ }
+ if messageID == RLPxMessageIDDisconnect {
+ var reason [1]p2p.DiscReason
+ err = rlp.DecodeBytes(data, &reason)
+ if (err != nil) && strings.Contains(err.Error(), "rlp: expected input list") {
+ err = rlp.DecodeBytes(data, &reason[0])
+ }
+ if err != nil {
+ return NewHandshakeError(HandshakeErrorIDDisconnectDecode, err, 0)
+ }
+ return NewHandshakeError(HandshakeErrorIDDisconnect, reason[0], uint64(reason[0]))
+ }
+ if messageID != expectedMessageID {
+ return NewHandshakeError(HandshakeErrorIDUnexpectedMessage, nil, messageID)
+ }
+
+ if err = rlp.DecodeBytes(data, message); err != nil {
+ return NewHandshakeError(decodeError, err, 0)
+ }
+ return nil
+}
+
+func makeOurHelloMessage(myPrivateKey *ecdsa.PrivateKey) HelloMessage {
+ version := params.VersionWithCommit(params.GitCommit, "")
+ clientID := common.MakeName("observer", version)
+
+ caps := []p2p.Cap{
+ {Name: eth.ProtocolName, Version: 63},
+ {Name: eth.ProtocolName, Version: 64},
+ {Name: eth.ProtocolName, Version: 65},
+ {Name: eth.ProtocolName, Version: eth.ETH66},
+ }
+
+ return HelloMessage{
+ Version: 5,
+ ClientID: clientID,
+ Caps: caps,
+ ListenPort: 0, // not listening
+ Pubkey: crypto.MarshalPubkey(&myPrivateKey.PublicKey),
+ }
+}
diff --git a/cmd/observer/observer/handshake_test.go b/cmd/observer/observer/handshake_test.go
new file mode 100644
index 0000000000..2691cd2400
--- /dev/null
+++ b/cmd/observer/observer/handshake_test.go
@@ -0,0 +1,35 @@
+package observer
+
+import (
+ "context"
+ "github.com/ledgerwatch/erigon/crypto"
+ "github.com/ledgerwatch/erigon/eth/protocols/eth"
+ "github.com/ledgerwatch/erigon/p2p/enode"
+ "github.com/ledgerwatch/erigon/params"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "testing"
+)
+
+func TestHandshake(t *testing.T) {
+ t.Skip("only for dev")
+
+ // grep 'self=enode' the log, and paste it here
+ // url := "enode://..."
+ url := params.MainnetBootnodes[0]
+ node := enode.MustParseV4(url)
+ myPrivateKey, _ := crypto.GenerateKey()
+
+ ctx := context.Background()
+ hello, status, err := Handshake(ctx, node.IP(), node.TCP(), node.Pubkey(), myPrivateKey)
+
+ require.Nil(t, err)
+ require.NotNil(t, hello)
+ assert.Equal(t, uint64(5), hello.Version)
+ assert.NotEmpty(t, hello.ClientID)
+ assert.Contains(t, hello.ClientID, "erigon")
+
+ require.NotNil(t, status)
+ assert.Equal(t, uint32(eth.ETH66), status.ProtocolVersion)
+ assert.Equal(t, uint64(1), status.NetworkID)
+}
diff --git a/cmd/observer/observer/interrogation_error.go b/cmd/observer/observer/interrogation_error.go
new file mode 100644
index 0000000000..bc99dd93a9
--- /dev/null
+++ b/cmd/observer/observer/interrogation_error.go
@@ -0,0 +1,53 @@
+package observer
+
+import "fmt"
+
+type InterrogationErrorID int
+
+const (
+ InterrogationErrorPing InterrogationErrorID = iota + 1
+ InterrogationErrorENRDecode
+ InterrogationErrorIncompatibleForkID
+ InterrogationErrorBlacklistedClientID
+ InterrogationErrorKeygen
+ InterrogationErrorFindNode
+ InterrogationErrorFindNodeTimeout
+)
+
+type InterrogationError struct {
+ id InterrogationErrorID
+ wrappedErr error
+}
+
+func NewInterrogationError(id InterrogationErrorID, wrappedErr error) *InterrogationError {
+ instance := InterrogationError{
+ id,
+ wrappedErr,
+ }
+ return &instance
+}
+
+func (e *InterrogationError) Unwrap() error {
+ return e.wrappedErr
+}
+
+func (e *InterrogationError) Error() string {
+ switch e.id {
+ case InterrogationErrorPing:
+ return fmt.Sprintf("ping-pong failed: %v", e.wrappedErr)
+ case InterrogationErrorENRDecode:
+ return e.wrappedErr.Error()
+ case InterrogationErrorIncompatibleForkID:
+ return fmt.Sprintf("incompatible ENR fork ID %v", e.wrappedErr)
+ case InterrogationErrorBlacklistedClientID:
+ return fmt.Sprintf("incompatible client ID %v", e.wrappedErr)
+ case InterrogationErrorKeygen:
+ return fmt.Sprintf("keygen failed: %v", e.wrappedErr)
+ case InterrogationErrorFindNode:
+ return fmt.Sprintf("FindNode request failed: %v", e.wrappedErr)
+ case InterrogationErrorFindNodeTimeout:
+ return fmt.Sprintf("FindNode request timeout: %v", e.wrappedErr)
+ default:
+ return "<unhandled InterrogationErrorID>"
+ }
+}
diff --git a/cmd/observer/observer/interrogator.go b/cmd/observer/observer/interrogator.go
new file mode 100644
index 0000000000..a66bd81160
--- /dev/null
+++ b/cmd/observer/observer/interrogator.go
@@ -0,0 +1,230 @@
+package observer
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "errors"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/observer/utils"
+ "github.com/ledgerwatch/erigon/core/forkid"
+ "github.com/ledgerwatch/erigon/eth/protocols/eth"
+ "github.com/ledgerwatch/erigon/p2p/enode"
+ "github.com/ledgerwatch/log/v3"
+ "golang.org/x/sync/semaphore"
+ "strings"
+ "time"
+)
+
+type DiscV4Transport interface {
+ RequestENR(*enode.Node) (*enode.Node, error)
+ Ping(*enode.Node) error
+ FindNode(toNode *enode.Node, targetKey *ecdsa.PublicKey) ([]*enode.Node, error)
+}
+
+type Interrogator struct {
+ node *enode.Node
+ transport DiscV4Transport
+ forkFilter forkid.Filter
+
+ diplomat *Diplomat
+ handshakeRetryTime *time.Time
+
+ keygenTimeout time.Duration
+ keygenConcurrency uint
+ keygenSemaphore *semaphore.Weighted
+ keygenCachedKeys []*ecdsa.PublicKey
+
+ log log.Logger
+}
+
+type InterrogationResult struct {
+ Node *enode.Node
+ IsCompatFork *bool
+ HandshakeResult *DiplomatResult
+ HandshakeRetryTime *time.Time
+ KeygenKeys []*ecdsa.PublicKey
+ Peers []*enode.Node
+}
+
+func NewInterrogator(
+ node *enode.Node,
+ transport DiscV4Transport,
+ forkFilter forkid.Filter,
+ diplomat *Diplomat,
+ handshakeRetryTime *time.Time,
+ keygenTimeout time.Duration,
+ keygenConcurrency uint,
+ keygenSemaphore *semaphore.Weighted,
+ keygenCachedKeys []*ecdsa.PublicKey,
+ logger log.Logger,
+) (*Interrogator, error) {
+ instance := Interrogator{
+ node,
+ transport,
+ forkFilter,
+ diplomat,
+ handshakeRetryTime,
+ keygenTimeout,
+ keygenConcurrency,
+ keygenSemaphore,
+ keygenCachedKeys,
+ logger,
+ }
+ return &instance, nil
+}
+
+func (interrogator *Interrogator) Run(ctx context.Context) (*InterrogationResult, *InterrogationError) {
+ interrogator.log.Debug("Interrogating a node")
+
+ err := interrogator.transport.Ping(interrogator.node)
+ if err != nil {
+ return nil, NewInterrogationError(InterrogationErrorPing, err)
+ }
+
+ // The outgoing Ping above triggers an incoming Ping.
+ // We need to wait until Server sends a Pong reply to that.
+ // The remote side is waiting for this Pong no longer than v4_udp.respTimeout.
+ // If we don't wait, the ENRRequest/FindNode might fail due to errUnknownNode.
+ utils.Sleep(ctx, 500*time.Millisecond)
+
+ // request client ID
+ var handshakeResult *DiplomatResult
+ var handshakeRetryTime *time.Time
+ if (interrogator.handshakeRetryTime == nil) || interrogator.handshakeRetryTime.Before(time.Now()) {
+ result := interrogator.diplomat.Run(ctx)
+ clientID := result.ClientID
+ if (clientID != nil) && IsClientIDBlacklisted(*clientID) {
+ return nil, NewInterrogationError(InterrogationErrorBlacklistedClientID, errors.New(*clientID))
+ }
+ handshakeResult = &result
+ handshakeRetryTime = new(time.Time)
+ *handshakeRetryTime = interrogator.diplomat.NextRetryTime(result.HandshakeErr)
+ }
+
+ // request ENR
+ var forkID *forkid.ID
+ var enr *enode.Node
+ if (handshakeResult == nil) || (handshakeResult.ClientID == nil) || isENRRequestSupportedByClientID(*handshakeResult.ClientID) {
+ enr, err = interrogator.transport.RequestENR(interrogator.node)
+ }
+ if err != nil {
+ interrogator.log.Debug("ENR request failed", "err", err)
+ } else if enr != nil {
+ interrogator.log.Debug("Got ENR", "enr", enr)
+ forkID, err = eth.LoadENRForkID(enr.Record())
+ if err != nil {
+ return nil, NewInterrogationError(InterrogationErrorENRDecode, err)
+ }
+ if forkID == nil {
+ interrogator.log.Debug("Got ENR, but it doesn't contain a ForkID")
+ }
+ }
+
+ // filter by fork ID
+ var isCompatFork *bool
+ if forkID != nil {
+ err := interrogator.forkFilter(*forkID)
+ isCompatFork = new(bool)
+ *isCompatFork = (err == nil) || !errors.Is(err, forkid.ErrLocalIncompatibleOrStale)
+ if !*isCompatFork {
+ return nil, NewInterrogationError(InterrogationErrorIncompatibleForkID, err)
+ }
+ }
+
+ // keygen
+ keys, err := interrogator.keygen(ctx)
+ if err != nil {
+ return nil, NewInterrogationError(InterrogationErrorKeygen, err)
+ }
+
+ // FindNode
+ peersByID := make(map[enode.ID]*enode.Node)
+ for _, key := range keys {
+ neighbors, err := interrogator.findNode(ctx, key)
+ if err != nil {
+ if isFindNodeTimeoutError(err) {
+ return nil, NewInterrogationError(InterrogationErrorFindNodeTimeout, err)
+ }
+ return nil, NewInterrogationError(InterrogationErrorFindNode, err)
+ }
+
+ for _, node := range neighbors {
+ if node.Incomplete() {
+ continue
+ }
+ peersByID[node.ID()] = node
+ }
+
+ utils.Sleep(ctx, 1*time.Second)
+ }
+
+ peers := valuesOfIDToNodeMap(peersByID)
+
+ result := InterrogationResult{
+ interrogator.node,
+ isCompatFork,
+ handshakeResult,
+ handshakeRetryTime,
+ keys,
+ peers,
+ }
+ return &result, nil
+}
+
+func (interrogator *Interrogator) keygen(ctx context.Context) ([]*ecdsa.PublicKey, error) {
+ if interrogator.keygenCachedKeys != nil {
+ return interrogator.keygenCachedKeys, nil
+ }
+
+ if err := interrogator.keygenSemaphore.Acquire(ctx, 1); err != nil {
+ return nil, err
+ }
+ defer interrogator.keygenSemaphore.Release(1)
+
+ keys := keygen(
+ ctx,
+ interrogator.node.Pubkey(),
+ interrogator.keygenTimeout,
+ interrogator.keygenConcurrency,
+ interrogator.log)
+
+ interrogator.log.Trace(fmt.Sprintf("Generated %d keys", len(keys)))
+ if (len(keys) < 13) && (ctx.Err() == nil) {
+ msg := "Generated just %d keys within a given timeout and concurrency (expected 16-17). " +
+ "If this happens too often, try to increase keygen-timeout/keygen-concurrency parameters."
+ interrogator.log.Warn(fmt.Sprintf(msg, len(keys)))
+ }
+ return keys, ctx.Err()
+}
+
+func (interrogator *Interrogator) findNode(ctx context.Context, targetKey *ecdsa.PublicKey) ([]*enode.Node, error) {
+ delayForAttempt := func(attempt int) time.Duration { return 2 * time.Second }
+ resultAny, err := utils.Retry(ctx, 2, delayForAttempt, isFindNodeTimeoutError, interrogator.log, "FindNode", func(ctx context.Context) (interface{}, error) {
+ return interrogator.transport.FindNode(interrogator.node, targetKey)
+ })
+
+ if resultAny == nil {
+ return nil, err
+ }
+ result := resultAny.([]*enode.Node)
+ return result, err
+}
+
+func isFindNodeTimeoutError(err error) bool {
+ return (err != nil) && (err.Error() == "RPC timeout")
+}
+
+func isENRRequestSupportedByClientID(clientID string) bool {
+ isUnsupported := strings.HasPrefix(clientID, "Parity-Ethereum") ||
+ strings.HasPrefix(clientID, "OpenEthereum") ||
+ strings.HasPrefix(clientID, "Nethermind")
+ return !isUnsupported
+}
+
+func valuesOfIDToNodeMap(m map[enode.ID]*enode.Node) []*enode.Node {
+ values := make([]*enode.Node, 0, len(m))
+ for _, value := range m {
+ values = append(values, value)
+ }
+ return values
+}
diff --git a/cmd/observer/observer/keygen.go b/cmd/observer/observer/keygen.go
new file mode 100644
index 0000000000..5de9bc3f9c
--- /dev/null
+++ b/cmd/observer/observer/keygen.go
@@ -0,0 +1,76 @@
+package observer
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "github.com/ledgerwatch/erigon/crypto"
+ "github.com/ledgerwatch/erigon/p2p/enode"
+ "github.com/ledgerwatch/log/v3"
+ "time"
+)
+
+func keygen(
+ parentContext context.Context,
+ targetKey *ecdsa.PublicKey,
+ timeout time.Duration,
+ concurrencyLimit uint,
+ logger log.Logger,
+) []*ecdsa.PublicKey {
+ ctx, cancel := context.WithTimeout(parentContext, timeout)
+ defer cancel()
+
+ targetID := enode.PubkeyToIDV4(targetKey)
+ cpus := concurrencyLimit
+
+ type result struct {
+ key *ecdsa.PublicKey
+ distance int
+ }
+
+ generatedKeys := make(chan result, cpus)
+
+ for i := uint(0); i < cpus; i++ {
+ go func() {
+ for ctx.Err() == nil {
+ keyPair, err := crypto.GenerateKey()
+ if err != nil {
+ logger.Error("keygen has failed to generate a key", "err", err)
+ break
+ }
+
+ key := &keyPair.PublicKey
+ id := enode.PubkeyToIDV4(key)
+ distance := enode.LogDist(targetID, id)
+
+ select {
+ case generatedKeys <- result{key, distance}:
+ case <-ctx.Done():
+ break
+ }
+ }
+ }()
+ }
+
+ keysAtDist := make(map[int]*ecdsa.PublicKey)
+
+ for ctx.Err() == nil {
+ select {
+ case res := <-generatedKeys:
+ keysAtDist[res.distance] = res.key
+ case <-ctx.Done():
+ break
+ }
+ }
+
+ keys := valuesOfIntToPubkeyMap(keysAtDist)
+
+ return keys
+}
+
+func valuesOfIntToPubkeyMap(m map[int]*ecdsa.PublicKey) []*ecdsa.PublicKey {
+ values := make([]*ecdsa.PublicKey, 0, len(m))
+ for _, value := range m {
+ values = append(values, value)
+ }
+ return values
+}
diff --git a/cmd/observer/observer/keygen_test.go b/cmd/observer/observer/keygen_test.go
new file mode 100644
index 0000000000..b57e56b0d2
--- /dev/null
+++ b/cmd/observer/observer/keygen_test.go
@@ -0,0 +1,23 @@
+package observer
+
+import (
+ "context"
+ "github.com/ledgerwatch/erigon/crypto"
+ "github.com/ledgerwatch/log/v3"
+ "github.com/stretchr/testify/assert"
+ "runtime"
+ "testing"
+ "time"
+)
+
+func TestKeygen(t *testing.T) {
+ targetKeyPair, err := crypto.GenerateKey()
+ assert.NotNil(t, targetKeyPair)
+ assert.Nil(t, err)
+
+ targetKey := &targetKeyPair.PublicKey
+ keys := keygen(context.Background(), targetKey, 50*time.Millisecond, uint(runtime.GOMAXPROCS(-1)), log.Root())
+
+ assert.NotNil(t, keys)
+ assert.GreaterOrEqual(t, len(keys), 4)
+}
diff --git a/cmd/observer/observer/node.go b/cmd/observer/observer/node.go
new file mode 100644
index 0000000000..184d9842dd
--- /dev/null
+++ b/cmd/observer/observer/node.go
@@ -0,0 +1,145 @@
+package observer
+
+import (
+ "crypto/ecdsa"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "github.com/ledgerwatch/erigon/crypto"
+ "github.com/ledgerwatch/erigon/p2p/enode"
+ "github.com/ledgerwatch/erigon/p2p/enr"
+ "net"
+ "net/url"
+)
+
+func nodeID(node *enode.Node) (database.NodeID, error) {
+ if node.Incomplete() {
+ return "", errors.New("nodeID not implemented for incomplete nodes")
+ }
+ nodeURL, err := url.Parse(node.URLv4())
+ if err != nil {
+ return "", fmt.Errorf("failed to parse node URL: %w", err)
+ }
+ id := nodeURL.User.Username()
+ return database.NodeID(id), nil
+}
+
+func makeNodeAddr(node *enode.Node) database.NodeAddr {
+ var addr database.NodeAddr
+
+ var ipEntry enr.IPv4
+ if node.Load(&ipEntry) == nil {
+ addr.IP = net.IP(ipEntry)
+ }
+
+ var ipV6Entry enr.IPv6
+ if node.Load(&ipV6Entry) == nil {
+ addr.IPv6.IP = net.IP(ipEntry)
+ }
+
+ var portDiscEntry enr.UDP
+ if (addr.IP != nil) && (node.Load(&portDiscEntry) == nil) {
+ addr.PortDisc = uint16(portDiscEntry)
+ }
+
+ var ipV6PortDiscEntry enr.UDP6
+ if (addr.IPv6.IP != nil) && (node.Load(&ipV6PortDiscEntry) == nil) {
+ addr.IPv6.PortDisc = uint16(ipV6PortDiscEntry)
+ }
+
+ var portRLPxEntry enr.TCP
+ if (addr.IP != nil) && (node.Load(&portRLPxEntry) == nil) {
+ addr.PortRLPx = uint16(portRLPxEntry)
+ }
+
+ var ipV6PortRLPxEntry enr.TCP
+ if (addr.IPv6.IP != nil) && (node.Load(&ipV6PortRLPxEntry) == nil) {
+ addr.IPv6.PortRLPx = uint16(ipV6PortRLPxEntry)
+ }
+
+ return addr
+}
+
+func makeNodeFromAddr(id database.NodeID, addr database.NodeAddr) (*enode.Node, error) {
+ rec := new(enr.Record)
+
+ pubkey, err := parseHexPublicKey(string(id))
+ if err != nil {
+ return nil, err
+ }
+ rec.Set((*enode.Secp256k1)(pubkey))
+
+ if addr.IP != nil {
+ rec.Set(enr.IP(addr.IP))
+ }
+ if addr.IPv6.IP != nil {
+ rec.Set(enr.IPv6(addr.IPv6.IP))
+ }
+ if addr.PortDisc != 0 {
+ rec.Set(enr.UDP(addr.PortDisc))
+ }
+ if addr.PortRLPx != 0 {
+ rec.Set(enr.TCP(addr.PortRLPx))
+ }
+ if addr.IPv6.PortDisc != 0 {
+ rec.Set(enr.UDP6(addr.IPv6.PortDisc))
+ }
+ if addr.IPv6.PortRLPx != 0 {
+ rec.Set(enr.TCP6(addr.IPv6.PortRLPx))
+ }
+
+ rec.Set(enr.ID("unsigned"))
+ node, err := enode.New(enr.SchemeMap{"unsigned": noSignatureIDScheme{}}, rec)
+ if err != nil {
+ return nil, fmt.Errorf("failed to make a node: %w", err)
+ }
+ return node, nil
+}
+
+type noSignatureIDScheme struct {
+ enode.V4ID
+}
+
+func (noSignatureIDScheme) Verify(_ *enr.Record, _ []byte) error {
+ return nil
+}
+
+func parseHexPublicKey(keyStr string) (*ecdsa.PublicKey, error) {
+ nodeWithPubkey, err := enode.ParseV4("enode://" + keyStr)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode a public key: %w", err)
+ }
+ return nodeWithPubkey.Pubkey(), nil
+}
+
+func parseHexPublicKeys(hexKeys []string) ([]*ecdsa.PublicKey, error) {
+ if hexKeys == nil {
+ return nil, nil
+ }
+ keys := make([]*ecdsa.PublicKey, 0, len(hexKeys))
+ for _, keyStr := range hexKeys {
+ key, err := parseHexPublicKey(keyStr)
+ if err != nil {
+ return nil, err
+ }
+ keys = append(keys, key)
+ }
+ return keys, nil
+}
+
+func hexEncodePublicKey(key *ecdsa.PublicKey) string {
+ return hex.EncodeToString(crypto.MarshalPubkey(key))
+}
+
+func hexEncodePublicKeys(keys []*ecdsa.PublicKey) []string {
+ if keys == nil {
+ return nil
+ }
+ hexKeys := make([]string, 0, len(keys))
+ for _, key := range keys {
+ keyStr := hexEncodePublicKey(key)
+ hexKeys = append(hexKeys, keyStr)
+ }
+ return hexKeys
+}
diff --git a/cmd/observer/observer/server.go b/cmd/observer/observer/server.go
new file mode 100644
index 0000000000..5aca0fa1a4
--- /dev/null
+++ b/cmd/observer/observer/server.go
@@ -0,0 +1,183 @@
+package observer
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "errors"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/utils"
+ "github.com/ledgerwatch/erigon/common/debug"
+ "github.com/ledgerwatch/erigon/core/forkid"
+ "github.com/ledgerwatch/erigon/eth/protocols/eth"
+ "github.com/ledgerwatch/erigon/p2p"
+ "github.com/ledgerwatch/erigon/p2p/discover"
+ "github.com/ledgerwatch/erigon/p2p/enode"
+ "github.com/ledgerwatch/erigon/p2p/enr"
+ "github.com/ledgerwatch/erigon/p2p/nat"
+ "github.com/ledgerwatch/erigon/p2p/netutil"
+ "github.com/ledgerwatch/erigon/params"
+ "github.com/ledgerwatch/log/v3"
+ "net"
+ "path/filepath"
+)
+
+type Server struct {
+ localNode *enode.LocalNode
+
+ listenAddr string
+ natInterface nat.Interface
+ discConfig discover.Config
+
+ log log.Logger
+}
+
+func NewServer(flags CommandFlags) (*Server, error) {
+ nodeDBPath := filepath.Join(flags.DataDir, "nodes", "eth66")
+
+ nodeKeyConfig := p2p.NodeKeyConfig{}
+ privateKey, err := nodeKeyConfig.LoadOrParseOrGenerateAndSave(flags.NodeKeyFile, flags.NodeKeyHex, flags.DataDir)
+ if err != nil {
+ return nil, err
+ }
+
+ localNode, err := makeLocalNode(nodeDBPath, privateKey, flags.Chain)
+ if err != nil {
+ return nil, err
+ }
+
+ listenAddr := fmt.Sprintf(":%d", flags.ListenPort)
+
+ natInterface, err := nat.Parse(flags.NATDesc)
+ if err != nil {
+ return nil, fmt.Errorf("NAT parse error: %w", err)
+ }
+
+ var netRestrictList *netutil.Netlist
+ if flags.NetRestrict != "" {
+ netRestrictList, err = netutil.ParseNetlist(flags.NetRestrict)
+ if err != nil {
+ return nil, fmt.Errorf("net restrict parse error: %w", err)
+ }
+ }
+
+ bootnodes, err := utils.GetBootnodesFromFlags(flags.Bootnodes, flags.Chain)
+ if err != nil {
+ return nil, fmt.Errorf("bootnodes parse error: %w", err)
+ }
+
+ logger := log.New()
+
+ discConfig := discover.Config{
+ PrivateKey: privateKey,
+ NetRestrict: netRestrictList,
+ Bootnodes: bootnodes,
+ Log: logger.New(),
+ }
+
+ instance := Server{
+ localNode,
+ listenAddr,
+ natInterface,
+ discConfig,
+ logger,
+ }
+ return &instance, nil
+}
+
+func makeLocalNode(nodeDBPath string, privateKey *ecdsa.PrivateKey, chain string) (*enode.LocalNode, error) {
+ db, err := enode.OpenDB(nodeDBPath)
+ if err != nil {
+ return nil, err
+ }
+ localNode := enode.NewLocalNode(db, privateKey)
+ localNode.SetFallbackIP(net.IP{127, 0, 0, 1})
+
+ forksEntry, err := makeForksENREntry(chain)
+ if err != nil {
+ return nil, err
+ }
+ localNode.Set(forksEntry)
+
+ return localNode, nil
+}
+
+func makeForksENREntry(chain string) (enr.Entry, error) {
+ chainConfig := params.ChainConfigByChainName(chain)
+ genesisHash := params.GenesisHashByChainName(chain)
+ if (chainConfig == nil) || (genesisHash == nil) {
+ return nil, fmt.Errorf("unknown chain %s", chain)
+ }
+
+ forks := forkid.GatherForks(chainConfig)
+ return eth.CurrentENREntryFromForks(forks, *genesisHash, 0), nil
+}
+
+func (server *Server) Bootnodes() []*enode.Node {
+ return server.discConfig.Bootnodes
+}
+
+func (server *Server) PrivateKey() *ecdsa.PrivateKey {
+ return server.discConfig.PrivateKey
+}
+
+func (server *Server) mapNATPort(ctx context.Context, realAddr *net.UDPAddr) {
+ if server.natInterface == nil {
+ return
+ }
+ if realAddr.IP.IsLoopback() {
+ return
+ }
+ if !server.natInterface.SupportsMapping() {
+ return
+ }
+
+ go func() {
+ defer debug.LogPanic()
+ nat.Map(server.natInterface, ctx.Done(), "udp", realAddr.Port, realAddr.Port, "ethereum discovery")
+ }()
+}
+
+func (server *Server) detectNATExternalIP() (net.IP, error) {
+ if server.natInterface == nil {
+ return nil, errors.New("no NAT flag configured")
+ }
+ if _, hasExtIP := server.natInterface.(nat.ExtIP); !hasExtIP {
+ server.log.Info("Detecting external IP...")
+ }
+ ip, err := server.natInterface.ExternalIP()
+ if err != nil {
+ return nil, fmt.Errorf("NAT ExternalIP error: %w", err)
+ }
+ server.log.Debug("External IP detected", "ip", ip)
+ return ip, nil
+}
+
+func (server *Server) Listen(ctx context.Context) (*discover.UDPv4, error) {
+ if server.natInterface != nil {
+ ip, err := server.detectNATExternalIP()
+ if err != nil {
+ return nil, err
+ }
+ server.localNode.SetStaticIP(ip)
+ }
+
+ addr, err := net.ResolveUDPAddr("udp", server.listenAddr)
+ if err != nil {
+ return nil, fmt.Errorf("ResolveUDPAddr error: %w", err)
+ }
+ conn, err := net.ListenUDP("udp", addr)
+ if err != nil {
+ return nil, fmt.Errorf("ListenUDP error: %w", err)
+ }
+
+ realAddr := conn.LocalAddr().(*net.UDPAddr)
+ server.localNode.SetFallbackUDP(realAddr.Port)
+
+ if server.natInterface != nil {
+ server.mapNATPort(ctx, realAddr)
+ }
+
+ server.log.Debug("Discovery UDP listener is up", "addr", realAddr)
+
+ return discover.ListenV4(ctx, conn, server.localNode, server.discConfig)
+}
diff --git a/cmd/observer/observer/status_logger.go b/cmd/observer/observer/status_logger.go
new file mode 100644
index 0000000000..2757ecd7ca
--- /dev/null
+++ b/cmd/observer/observer/status_logger.go
@@ -0,0 +1,44 @@
+package observer
+
+import (
+ "context"
+ "errors"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "github.com/ledgerwatch/erigon/cmd/observer/utils"
+ "github.com/ledgerwatch/log/v3"
+ "time"
+)
+
+func StatusLoggerLoop(ctx context.Context, db database.DB, networkID uint, period time.Duration, logger log.Logger) {
+ var maxPingTries uint = 1000000 // unlimited (include dead nodes)
+ var prevTotalCount uint
+ var prevDistinctIPCount uint
+
+ for ctx.Err() == nil {
+ utils.Sleep(ctx, period)
+
+ totalCount, err := db.CountNodes(ctx, maxPingTries, networkID)
+ if err != nil {
+ if !errors.Is(err, context.Canceled) {
+ logger.Error("Failed to count nodes", "err", err)
+ }
+ continue
+ }
+
+ distinctIPCount, err := db.CountIPs(ctx, maxPingTries, networkID)
+ if err != nil {
+ if !errors.Is(err, context.Canceled) {
+ logger.Error("Failed to count IPs", "err", err)
+ }
+ continue
+ }
+
+ if (totalCount == prevTotalCount) && (distinctIPCount == prevDistinctIPCount) {
+ continue
+ }
+
+ logger.Info("Status", "totalCount", totalCount, "distinctIPCount", distinctIPCount)
+ prevTotalCount = totalCount
+ prevDistinctIPCount = distinctIPCount
+ }
+}
diff --git a/cmd/observer/reports/clients_estimate_report.go b/cmd/observer/reports/clients_estimate_report.go
new file mode 100644
index 0000000000..4ec7d149bf
--- /dev/null
+++ b/cmd/observer/reports/clients_estimate_report.go
@@ -0,0 +1,95 @@
+package reports
+
+import (
+ "context"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "math"
+ "strings"
+)
+
+type ClientsEstimateReportEntry struct {
+ Name string
+ CountLow uint
+ CountHigh uint
+}
+
+type ClientsEstimateReport struct {
+ Clients []ClientsEstimateReportEntry
+}
+
+func CreateClientsEstimateReport(
+ ctx context.Context,
+ db database.DB,
+ limit uint,
+ maxPingTries uint,
+ networkID uint,
+) (*ClientsEstimateReport, error) {
+ clientsReport, err := CreateClientsReport(ctx, db, limit, maxPingTries, networkID)
+ if err != nil {
+ return nil, err
+ }
+
+ report := ClientsEstimateReport{}
+
+ for i, topClient := range clientsReport.Clients {
+ if uint(i) >= limit {
+ break
+ }
+ clientName := topClient.Name
+
+ sameNetworkCount, err := db.CountClients(ctx, clientName+"/", maxPingTries, networkID)
+ if err != nil {
+ return nil, err
+ }
+ if sameNetworkCount == 0 {
+ continue
+ }
+
+ knownNetworkCount, err := db.CountClientsWithNetworkID(ctx, clientName+"/", maxPingTries)
+ if err != nil {
+ return nil, err
+ }
+ if knownNetworkCount == 0 {
+ continue
+ }
+
+ // 1 - (1 - p)/2 percentile for 95% confidence
+ const z = 1.96
+ intervalLow, intervalHigh := waldInterval(knownNetworkCount, sameNetworkCount, z)
+
+ transientErrCount, err := db.CountClientsWithHandshakeTransientError(ctx, clientName+"/", maxPingTries)
+ if err != nil {
+ return nil, err
+ }
+
+ countLow := sameNetworkCount + uint(math.Round(float64(transientErrCount)*intervalLow))
+ countHigh := sameNetworkCount + uint(math.Round(float64(transientErrCount)*intervalHigh))
+
+ client := ClientsEstimateReportEntry{
+ clientName,
+ countLow,
+ countHigh,
+ }
+ report.Clients = append(report.Clients, client)
+ }
+
+ return &report, nil
+}
+
+// https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Normal_approximation_interval_or_Wald_interval
+func waldInterval(n uint, ns uint, z float64) (float64, float64) {
+ nf := n - ns
+ p := float64(ns) / float64(n)
+ interval := z * math.Sqrt(float64(ns*nf)) / (float64(n) * math.Sqrt(float64(n)))
+ return p - interval, p + interval
+}
+
+func (report *ClientsEstimateReport) String() string {
+ var builder strings.Builder
+ for _, client := range report.Clients {
+ builder.WriteString(fmt.Sprintf("%6d - %-6d %s", client.CountLow, client.CountHigh, client.Name))
+ builder.WriteRune('\n')
+ }
+ return builder.String()
+}
diff --git a/cmd/observer/reports/clients_report.go b/cmd/observer/reports/clients_report.go
new file mode 100644
index 0000000000..e41b82e49d
--- /dev/null
+++ b/cmd/observer/reports/clients_report.go
@@ -0,0 +1,97 @@
+package reports
+
+import (
+ "context"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "github.com/ledgerwatch/erigon/cmd/observer/observer"
+ "strings"
+)
+
+type ClientsReportEntry struct {
+ Name string
+ Count uint
+}
+
+type ClientsReport struct {
+ Clients []ClientsReportEntry
+}
+
+func CreateClientsReport(ctx context.Context, db database.DB, limit uint, maxPingTries uint, networkID uint) (*ClientsReport, error) {
+ groups := make(map[string]uint)
+ unknownCount := uint(0)
+ enumFunc := func(clientID *string) {
+ if clientID != nil {
+ if observer.IsClientIDBlacklisted(*clientID) {
+ return
+ }
+ clientName := observer.NameFromClientID(*clientID)
+ groups[clientName]++
+ } else {
+ unknownCount++
+ }
+ }
+ if err := db.EnumerateClientIDs(ctx, maxPingTries, networkID, enumFunc); err != nil {
+ return nil, err
+ }
+
+ totalCount := sumMapValues(groups)
+
+ report := ClientsReport{}
+
+ for i := uint(0); i < limit; i++ {
+ clientName, count := takeMapMaxValue(groups)
+ if count == 0 {
+ break
+ }
+
+ client := ClientsReportEntry{
+ clientName,
+ count,
+ }
+ report.Clients = append(report.Clients, client)
+ }
+
+ othersCount := sumMapValues(groups)
+
+ report.Clients = append(report.Clients,
+ ClientsReportEntry{"...", othersCount},
+ ClientsReportEntry{"total", totalCount},
+ ClientsReportEntry{"unknown", unknownCount})
+
+ return &report, nil
+}
+
+func (report *ClientsReport) String() string {
+ var builder strings.Builder
+ builder.WriteString("clients:")
+ builder.WriteRune('\n')
+ for _, client := range report.Clients {
+ builder.WriteString(fmt.Sprintf("%6d %s", client.Count, client.Name))
+ builder.WriteRune('\n')
+ }
+ return builder.String()
+}
+
+func takeMapMaxValue(m map[string]uint) (string, uint) {
+ maxKey := ""
+ maxValue := uint(0)
+
+ for k, v := range m {
+ if v > maxValue {
+ maxKey = k
+ maxValue = v
+ }
+ }
+
+ delete(m, maxKey)
+ return maxKey, maxValue
+}
+
+func sumMapValues(m map[string]uint) uint {
+ sum := uint(0)
+ for _, v := range m {
+ sum += v
+ }
+ return sum
+}
diff --git a/cmd/observer/reports/command.go b/cmd/observer/reports/command.go
new file mode 100644
index 0000000000..b3c4f13a71
--- /dev/null
+++ b/cmd/observer/reports/command.go
@@ -0,0 +1,92 @@
+package reports
+
+import (
+ "context"
+ "github.com/ledgerwatch/erigon/cmd/utils"
+ "github.com/spf13/cobra"
+ "github.com/urfave/cli"
+)
+
+type CommandFlags struct {
+ DataDir string
+ Chain string
+ ClientsLimit uint
+ MaxPingTries uint
+ Estimate bool
+}
+
+type Command struct {
+ command cobra.Command
+ flags CommandFlags
+}
+
+func NewCommand() *Command {
+ command := cobra.Command{
+ Use: "report",
+ Short: "P2P network crawler database report",
+ }
+
+ instance := Command{
+ command: command,
+ }
+ instance.withDatadir()
+ instance.withChain()
+ instance.withClientsLimit()
+ instance.withMaxPingTries()
+ instance.withEstimate()
+
+ return &instance
+}
+
+func (command *Command) withDatadir() {
+ flag := utils.DataDirFlag
+ command.command.Flags().StringVar(&command.flags.DataDir, flag.Name, flag.Value.String(), flag.Usage)
+ must(command.command.MarkFlagDirname(utils.DataDirFlag.Name))
+}
+
+func (command *Command) withChain() {
+ flag := utils.ChainFlag
+ command.command.Flags().StringVar(&command.flags.Chain, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withClientsLimit() {
+ flag := cli.UintFlag{
+ Name: "clients-limit",
+ Usage: "A number of top clients to show",
+ Value: uint(10),
+ }
+ command.command.Flags().UintVar(&command.flags.ClientsLimit, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withMaxPingTries() {
+ flag := cli.UintFlag{
+ Name: "max-ping-tries",
+ Usage: "A number of PING failures for a node to be considered dead",
+ Value: 3,
+ }
+ command.command.Flags().UintVar(&command.flags.MaxPingTries, flag.Name, flag.Value, flag.Usage)
+}
+
+func (command *Command) withEstimate() {
+ flag := cli.BoolFlag{
+ Name: "estimate",
+ Usage: "Estimate totals including nodes that replied with 'too many peers'",
+ }
+ command.command.Flags().BoolVar(&command.flags.Estimate, flag.Name, false, flag.Usage)
+}
+
+func (command *Command) RawCommand() *cobra.Command {
+ return &command.command
+}
+
+func (command *Command) OnRun(runFunc func(ctx context.Context, flags CommandFlags) error) {
+ command.command.RunE = func(cmd *cobra.Command, args []string) error {
+ return runFunc(cmd.Context(), command.flags)
+ }
+}
+
+func must(err error) {
+ if err != nil {
+ panic(err)
+ }
+}
diff --git a/cmd/observer/reports/status_report.go b/cmd/observer/reports/status_report.go
new file mode 100644
index 0000000000..c094e9b203
--- /dev/null
+++ b/cmd/observer/reports/status_report.go
@@ -0,0 +1,40 @@
+package reports
+
+import (
+ "context"
+ "fmt"
+ "github.com/ledgerwatch/erigon/cmd/observer/database"
+ "strings"
+)
+
+type StatusReport struct {
+ TotalCount uint
+ DistinctIPCount uint
+}
+
+func CreateStatusReport(ctx context.Context, db database.DB, maxPingTries uint, networkID uint) (*StatusReport, error) {
+ totalCount, err := db.CountNodes(ctx, maxPingTries, networkID)
+ if err != nil {
+ return nil, err
+ }
+
+ distinctIPCount, err := db.CountIPs(ctx, maxPingTries, networkID)
+ if err != nil {
+ return nil, err
+ }
+
+ report := StatusReport{
+ totalCount,
+ distinctIPCount,
+ }
+ return &report, nil
+}
+
+func (report *StatusReport) String() string {
+ var builder strings.Builder
+ builder.WriteString(fmt.Sprintf("total: %d", report.TotalCount))
+ builder.WriteRune('\n')
+ builder.WriteString(fmt.Sprintf("distinct IPs: %d", report.DistinctIPCount))
+ builder.WriteRune('\n')
+ return builder.String()
+}
diff --git a/cmd/observer/utils/retry.go b/cmd/observer/utils/retry.go
new file mode 100644
index 0000000000..ab745415a3
--- /dev/null
+++ b/cmd/observer/utils/retry.go
@@ -0,0 +1,32 @@
+package utils
+
+import (
+ "context"
+ "github.com/ledgerwatch/log/v3"
+ "time"
+)
+
+func Retry(
+ ctx context.Context,
+ retryCount int,
+ delayForAttempt func(attempt int) time.Duration,
+ isRecoverableError func(error) bool,
+ logger log.Logger,
+ opName string,
+ op func(context.Context) (interface{}, error),
+) (interface{}, error) {
+ var result interface{}
+ var err error
+
+ for i := 0; i <= retryCount; i++ {
+ if i > 0 {
+ logger.Trace("retrying", "op", opName, "attempt", i, "err", err)
+ Sleep(ctx, delayForAttempt(i))
+ }
+ result, err = op(ctx)
+ if (err == nil) || !isRecoverableError(err) {
+ break
+ }
+ }
+ return result, err
+}
diff --git a/cmd/observer/utils/sleep.go b/cmd/observer/utils/sleep.go
new file mode 100644
index 0000000000..336f531402
--- /dev/null
+++ b/cmd/observer/utils/sleep.go
@@ -0,0 +1,15 @@
+package utils
+
+import (
+ "context"
+ "time"
+)
+
+func Sleep(parentContext context.Context, timeout time.Duration) {
+ if timeout <= 0 {
+ return
+ }
+ ctx, cancel := context.WithTimeout(parentContext, timeout)
+ defer cancel()
+ <-ctx.Done()
+}
diff --git a/cmd/observer/utils/task_queue.go b/cmd/observer/utils/task_queue.go
new file mode 100644
index 0000000000..b01af3c60b
--- /dev/null
+++ b/cmd/observer/utils/task_queue.go
@@ -0,0 +1,51 @@
+package utils
+
+import (
+ "context"
+ "errors"
+)
+
+type TaskQueue struct {
+ name string
+ queue chan func(context.Context) error
+
+ logFuncProvider func(err error) func(msg string, ctx ...interface{})
+}
+
+func NewTaskQueue(
+ name string,
+ capacity uint,
+ logFuncProvider func(err error) func(msg string, ctx ...interface{}),
+) *TaskQueue {
+ queue := make(chan func(context.Context) error, capacity)
+
+ instance := TaskQueue{
+ name,
+ queue,
+ logFuncProvider,
+ }
+ return &instance
+}
+
+func (queue *TaskQueue) Run(ctx context.Context) {
+ for ctx.Err() == nil {
+ select {
+ case <-ctx.Done():
+ break
+ case op := <-queue.queue:
+ err := op(ctx)
+ if (err != nil) && !errors.Is(err, context.Canceled) {
+ logFunc := queue.logFuncProvider(err)
+ logFunc("Task failed", "queue", queue.name, "err", err)
+ }
+ }
+ }
+}
+
+func (queue *TaskQueue) EnqueueTask(ctx context.Context, op func(context.Context) error) {
+ select {
+ case <-ctx.Done():
+ break
+ case queue.queue <- op:
+ }
+}
diff --git a/eth/protocols/eth/discovery.go b/eth/protocols/eth/discovery.go
index a6cf92b88d..4bd009da7e 100644
--- a/eth/protocols/eth/discovery.go
+++ b/eth/protocols/eth/discovery.go
@@ -17,8 +17,10 @@
package eth
import (
+ "fmt"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/core/forkid"
+ "github.com/ledgerwatch/erigon/p2p/enr"
"github.com/ledgerwatch/erigon/rlp"
)
@@ -41,3 +43,14 @@ func CurrentENREntryFromForks(forks []uint64, genesisHash common.Hash, headHeigh
ForkID: forkid.NewIDFromForks(forks, genesisHash, headHeight),
}
}
+
+func LoadENRForkID(r *enr.Record) (*forkid.ID, error) {
+ var entry enrEntry
+ if err := r.Load(&entry); err != nil {
+ if enr.IsNotFound(err) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to load fork ID from ENR: %w", err)
+ }
+ return &entry.ForkID, nil
+}
diff --git a/go.mod b/go.mod
index 974371eb99..4fcddaea31 100644
--- a/go.mod
+++ b/go.mod
@@ -66,6 +66,7 @@ require (
google.golang.org/protobuf v1.28.0
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6
+ modernc.org/sqlite v1.14.2-0.20211125151325-d4ed92c0a70f
pgregory.net/rapid v0.4.7
)
@@ -107,6 +108,7 @@ require (
github.com/huandu/xstrings v1.3.2 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmhodges/levigo v1.0.0 // indirect
+ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
@@ -142,6 +144,7 @@ require (
github.com/prometheus/common v0.30.0 // indirect
github.com/prometheus/procfs v0.7.2 // indirect
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect
+ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -161,4 +164,13 @@ require (
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+ lukechampine.com/uint128 v1.1.1 // indirect
+ modernc.org/cc/v3 v3.35.18 // indirect
+ modernc.org/ccgo/v3 v3.12.73 // indirect
+ modernc.org/libc v1.11.82 // indirect
+ modernc.org/mathutil v1.4.1 // indirect
+ modernc.org/memory v1.0.5 // indirect
+ modernc.org/opt v0.1.1 // indirect
+ modernc.org/strutil v1.1.1 // indirect
+ modernc.org/token v1.0.0 // indirect
)
diff --git a/go.sum b/go.sum
index b84717f45b..9a2c5b6a27 100644
--- a/go.sum
+++ b/go.sum
@@ -324,6 +324,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -430,6 +431,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kevinburke/go-bindata v3.21.0+incompatible h1:baK7hwFJDlAHrOqmE9U3u8tow1Uc5ihN9E/b7djcK2g=
github.com/kevinburke/go-bindata v3.21.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -469,9 +472,12 @@ github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHR
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
+github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -629,6 +635,8 @@ github.com/quasilyte/go-ruleguard/dsl v0.3.19 h1:5+KTKb2YREUYiqZFEIuifFyBxlcCUPW
github.com/quasilyte/go-ruleguard/dsl v0.3.19/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -899,6 +907,7 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -917,6 +926,7 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -926,7 +936,9 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211030160813-b3129d9d1021/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -997,6 +1009,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
@@ -1136,6 +1149,113 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU=
+lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
+modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.33.9/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.33.11/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.34.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.4/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.5/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.7/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.8/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.18 h1:rMZhRcWrba0y3nVmdiQ7kxAgOOSq2m2f2VzjHLgEs6U=
+modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60=
+modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw=
+modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI=
+modernc.org/ccgo/v3 v3.11.1/go.mod h1:lWHxfsn13L3f7hgGsGlU28D9eUOf6y3ZYHKoPaKU0ag=
+modernc.org/ccgo/v3 v3.11.3/go.mod h1:0oHunRBMBiXOKdaglfMlRPBALQqsfrCKXgw9okQ3GEw=
+modernc.org/ccgo/v3 v3.12.4/go.mod h1:Bk+m6m2tsooJchP/Yk5ji56cClmN6R1cqc9o/YtbgBQ=
+modernc.org/ccgo/v3 v3.12.6/go.mod h1:0Ji3ruvpFPpz+yu+1m0wk68pdr/LENABhTrDkMDWH6c=
+modernc.org/ccgo/v3 v3.12.8/go.mod h1:Hq9keM4ZfjCDuDXxaHptpv9N24JhgBZmUG5q60iLgUo=
+modernc.org/ccgo/v3 v3.12.11/go.mod h1:0jVcmyDwDKDGWbcrzQ+xwJjbhZruHtouiBEvDfoIsdg=
+modernc.org/ccgo/v3 v3.12.14/go.mod h1:GhTu1k0YCpJSuWwtRAEHAol5W7g1/RRfS4/9hc9vF5I=
+modernc.org/ccgo/v3 v3.12.18/go.mod h1:jvg/xVdWWmZACSgOiAhpWpwHWylbJaSzayCqNOJKIhs=
+modernc.org/ccgo/v3 v3.12.20/go.mod h1:aKEdssiu7gVgSy/jjMastnv/q6wWGRbszbheXgWRHc8=
+modernc.org/ccgo/v3 v3.12.21/go.mod h1:ydgg2tEprnyMn159ZO/N4pLBqpL7NOkJ88GT5zNU2dE=
+modernc.org/ccgo/v3 v3.12.22/go.mod h1:nyDVFMmMWhMsgQw+5JH6B6o4MnZ+UQNw1pp52XYFPRk=
+modernc.org/ccgo/v3 v3.12.25/go.mod h1:UaLyWI26TwyIT4+ZFNjkyTbsPsY3plAEB6E7L/vZV3w=
+modernc.org/ccgo/v3 v3.12.29/go.mod h1:FXVjG7YLf9FetsS2OOYcwNhcdOLGt8S9bQ48+OP75cE=
+modernc.org/ccgo/v3 v3.12.36/go.mod h1:uP3/Fiezp/Ga8onfvMLpREq+KUjUmYMxXPO8tETHtA8=
+modernc.org/ccgo/v3 v3.12.38/go.mod h1:93O0G7baRST1vNj4wnZ49b1kLxt0xCW5Hsa2qRaZPqc=
+modernc.org/ccgo/v3 v3.12.43/go.mod h1:k+DqGXd3o7W+inNujK15S5ZYuPoWYLpF5PYougCmthU=
+modernc.org/ccgo/v3 v3.12.46/go.mod h1:UZe6EvMSqOxaJ4sznY7b23/k13R8XNlyWsO5bAmSgOE=
+modernc.org/ccgo/v3 v3.12.47/go.mod h1:m8d6p0zNps187fhBwzY/ii6gxfjob1VxWb919Nk1HUk=
+modernc.org/ccgo/v3 v3.12.50/go.mod h1:bu9YIwtg+HXQxBhsRDE+cJjQRuINuT9PUK4orOco/JI=
+modernc.org/ccgo/v3 v3.12.51/go.mod h1:gaIIlx4YpmGO2bLye04/yeblmvWEmE4BBBls4aJXFiE=
+modernc.org/ccgo/v3 v3.12.53/go.mod h1:8xWGGTFkdFEWBEsUmi+DBjwu/WLy3SSOrqEmKUjMeEg=
+modernc.org/ccgo/v3 v3.12.54/go.mod h1:yANKFTm9llTFVX1FqNKHE0aMcQb1fuPJx6p8AcUx+74=
+modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7IeU=
+modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU=
+modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc=
+modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM=
+modernc.org/ccgo/v3 v3.12.65/go.mod h1:D6hQtKxPNZiY6wDBtehSGKFKmyXn53F8nGTpH+POmS4=
+modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ=
+modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84=
+modernc.org/ccgo/v3 v3.12.73 h1:AMk4wEpzWjpODXohKvvnlwLob4Xk8tq3we6CwYh88mA=
+modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ=
+modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
+modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
+modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
+modernc.org/libc v1.9.11/go.mod h1:NyF3tsA5ArIjJ83XB0JlqhjTabTCHm9aX4XMPHyQn0Q=
+modernc.org/libc v1.11.0/go.mod h1:2lOfPmj7cz+g1MrPNmX65QCzVxgNq2C5o0jdLY2gAYg=
+modernc.org/libc v1.11.2/go.mod h1:ioIyrl3ETkugDO3SGZ+6EOKvlP3zSOycUETe4XM4n8M=
+modernc.org/libc v1.11.5/go.mod h1:k3HDCP95A6U111Q5TmG3nAyUcp3kR5YFZTeDS9v8vSU=
+modernc.org/libc v1.11.6/go.mod h1:ddqmzR6p5i4jIGK1d/EiSw97LBcE3dK24QEwCFvgNgE=
+modernc.org/libc v1.11.11/go.mod h1:lXEp9QOOk4qAYOtL3BmMve99S5Owz7Qyowzvg6LiZso=
+modernc.org/libc v1.11.13/go.mod h1:ZYawJWlXIzXy2Pzghaf7YfM8OKacP3eZQI81PDLFdY8=
+modernc.org/libc v1.11.16/go.mod h1:+DJquzYi+DMRUtWI1YNxrlQO6TcA5+dRRiq8HWBWRC8=
+modernc.org/libc v1.11.19/go.mod h1:e0dgEame6mkydy19KKaVPBeEnyJB4LGNb0bBH1EtQ3I=
+modernc.org/libc v1.11.24/go.mod h1:FOSzE0UwookyT1TtCJrRkvsOrX2k38HoInhw+cSCUGk=
+modernc.org/libc v1.11.26/go.mod h1:SFjnYi9OSd2W7f4ct622o/PAYqk7KHv6GS8NZULIjKY=
+modernc.org/libc v1.11.27/go.mod h1:zmWm6kcFXt/jpzeCgfvUNswM0qke8qVwxqZrnddlDiE=
+modernc.org/libc v1.11.28/go.mod h1:Ii4V0fTFcbq3qrv3CNn+OGHAvzqMBvC7dBNyC4vHZlg=
+modernc.org/libc v1.11.31/go.mod h1:FpBncUkEAtopRNJj8aRo29qUiyx5AvAlAxzlx9GNaVM=
+modernc.org/libc v1.11.34/go.mod h1:+Tzc4hnb1iaX/SKAutJmfzES6awxfU1BPvrrJO0pYLg=
+modernc.org/libc v1.11.37/go.mod h1:dCQebOwoO1046yTrfUE5nX1f3YpGZQKNcITUYWlrAWo=
+modernc.org/libc v1.11.39/go.mod h1:mV8lJMo2S5A31uD0k1cMu7vrJbSA3J3waQJxpV4iqx8=
+modernc.org/libc v1.11.42/go.mod h1:yzrLDU+sSjLE+D4bIhS7q1L5UwXDOw99PLSX0BlZvSQ=
+modernc.org/libc v1.11.44/go.mod h1:KFq33jsma7F5WXiYelU8quMJasCCTnHK0mkri4yPHgA=
+modernc.org/libc v1.11.45/go.mod h1:Y192orvfVQQYFzCNsn+Xt0Hxt4DiO4USpLNXBlXg/tM=
+modernc.org/libc v1.11.47/go.mod h1:tPkE4PzCTW27E6AIKIR5IwHAQKCAtudEIeAV1/SiyBg=
+modernc.org/libc v1.11.49/go.mod h1:9JrJuK5WTtoTWIFQ7QjX2Mb/bagYdZdscI3xrvHbXjE=
+modernc.org/libc v1.11.51/go.mod h1:R9I8u9TS+meaWLdbfQhq2kFknTW0O3aw3kEMqDDxMaM=
+modernc.org/libc v1.11.53/go.mod h1:5ip5vWYPAoMulkQ5XlSJTy12Sz5U6blOQiYasilVPsU=
+modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw=
+modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M=
+modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18=
+modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8=
+modernc.org/libc v1.11.70/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
+modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
+modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0=
+modernc.org/libc v1.11.82 h1:CSl/6n4odvPYWKKqBtFb8e0ZWVTjxDqwxTjaoee9V7E=
+modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI=
+modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8=
+modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc=
+modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14=
+modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM=
+modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
+modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sqlite v1.14.2-0.20211125151325-d4ed92c0a70f h1:yQwkmqKCIgLzFIfjfPfZAAxLZernckpo7zGTv37Ahv0=
+modernc.org/sqlite v1.14.2-0.20211125151325-d4ed92c0a70f/go.mod h1:YT5XFRKOueohjppHO4cHb54eQlnaUGsZMHoryaCpNo4=
+modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs=
+modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
+modernc.org/tcl v1.8.13 h1:V0sTNBw0Re86PvXZxuCub3oO9WrSTqALgrwNZNvLFGw=
+modernc.org/tcl v1.8.13/go.mod h1:V+q/Ef0IJaNUSECieLU4o+8IScapxnMyFV6i/7uQlAY=
+modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=
+modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/z v1.2.19 h1:BGyRFWhDVn5LFS5OcX4Yd/MlpRTOc7hOPTdcIpCiUao=
+modernc.org/z v1.2.19/go.mod h1:+ZpP0pc4zz97eukOzW3xagV/lS82IpPN9NGG5pNF9vY=
pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=
pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go
index cdeec407d3..3e2614a098 100644
--- a/p2p/discover/v4_udp.go
+++ b/p2p/discover/v4_udp.go
@@ -301,8 +301,14 @@ func (t *UDPv4) newLookup(ctx context.Context, targetKey *ecdsa.PublicKey) *look
return it
}
-// findnode sends a findnode request to the given node and waits until
-// the node has sent up to k neighbors.
+// FindNode sends a "FindNode" request to the given node and waits until
+// the node has sent up to bucketSize neighbors or a respTimeout has passed.
+func (t *UDPv4) FindNode(toNode *enode.Node, targetKey *ecdsa.PublicKey) ([]*enode.Node, error) {
+ targetKeyEnc := v4wire.EncodePubkey(targetKey)
+ nodes, err := t.findnode(toNode.ID(), wrapNode(toNode).addr(), targetKeyEnc)
+ return unwrapNodes(nodes), err
+}
+
func (t *UDPv4) findnode(toid enode.ID, toaddr *net.UDPAddr, target v4wire.Pubkey) ([]*node, error) {
t.ensureBond(toid, toaddr)
--
GitLab