diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 06a68ee0d98..a84f4c23fdb 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -116,8 +116,13 @@ type FileDownloadedStatisticsUpdate struct { } type SegmentPeer struct { - Url string `json:"url"` - DownloadRate uint64 `json:"downloadRate"` + Url string `json:"url"` + DownloadRate uint64 `json:"downloadRate"` + UploadRate uint64 `json:"uploadRate"` + PiecesCount uint64 `json:"piecesCount"` + RemoteAddr string `json:"remoteAddr"` + PeerId [20]byte `json:"peerId"` + TorrentName string `json:"torrentName"` } type SnapshotIndexingStatistics struct { diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 22e40c13eaa..61ac1523eaf 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -1569,7 +1569,7 @@ func (d *Downloader) torrentDownload(t *torrent.Torrent, statusChan chan downloa return case <-t.Complete.On(): downloadTime := time.Since(downloadStarted) - downloaded := t.Stats().BytesReadUsefulData + downloaded := t.Stats().BytesCompleted diagnostics.Send(diagnostics.FileDownloadedStatisticsUpdate{ FileName: t.Name(), @@ -2293,7 +2293,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { d.lock.Unlock() if !stats.Completed { - logger.Debug("[snapshots] downloading", + log.Debug("[snapshots] downloading", "len", len(torrents), "webTransfers", webTransfers, "torrent", torrentInfo, @@ -2305,6 +2305,8 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { "completion-rate", fmt.Sprintf("%s/s", common.ByteCount(stats.CompletionRate)), "flushed", common.ByteCount(stats.BytesFlushed), "flush-rate", fmt.Sprintf("%s/s", common.ByteCount(stats.FlushRate)), + "downloaded", common.ByteCount(stats.BytesDownload), + "download-rate", fmt.Sprintf("%s/s", common.ByteCount(stats.DownloadRate)), "webseed-trips", stats.WebseedTripCount.Load(), "webseed-active", stats.WebseedActiveTrips.Load(), "webseed-max-active", stats.WebseedMaxActiveTrips.Load(), @@ -2374,10 +2376,16 @@ func getWebseedsRatesForlogs(weebseedPeersOfThisFile []*torrent.Peer, fName stri if peerUrl, err := webPeerUrl(peer); err == nil { if shortUrl, err := url.JoinPath(peerUrl.Host, peerUrl.Path); err == nil { rate := uint64(peer.DownloadRate()) + upRate := uint64(peer.UploadRate()) if !finished { seed := diagnostics.SegmentPeer{ Url: peerUrl.Host, DownloadRate: rate, + UploadRate: upRate, + RemoteAddr: peer.RemoteAddr.String(), + PeerId: [20]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + PiecesCount: 0, + TorrentName: fName, } seeds = append(seeds, seed) } @@ -2401,11 +2409,17 @@ func getPeersRatesForlogs(peersOfThisFile []*torrent.PeerConn, fName string) ([] for _, peer := range peersOfThisFile { dr := uint64(peer.DownloadRate()) + ur := uint64(peer.UploadRate()) url := fmt.Sprintf("%v", peer.PeerClientName.Load()) segPeer := diagnostics.SegmentPeer{ Url: url, DownloadRate: dr, + UploadRate: ur, + PiecesCount: peer.PeerPieces().GetCardinality(), + RemoteAddr: peer.RemoteAddr.String(), + PeerId: peer.PeerID, + TorrentName: fName, } peers = append(peers, segPeer) rates = append(rates, url, fmt.Sprintf("%s/s", common.ByteCount(dr))) @@ -2904,22 +2918,22 @@ func (d *Downloader) logProgress() { "completion-rate", fmt.Sprintf("%s/s", common.ByteCount(d.stats.CompletionRate)), "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys)) - - diagnostics.Send(diagnostics.SnapshotDownloadStatistics{ - Downloaded: bytesDone, - Total: d.stats.BytesTotal, - TotalTime: time.Since(d.startTime).Round(time.Second).Seconds(), - DownloadRate: d.stats.DownloadRate, - UploadRate: d.stats.UploadRate, - Peers: d.stats.PeersUnique, - Files: d.stats.FilesTotal, - Connections: d.stats.ConnectionsTotal, - Alloc: m.Alloc, - Sys: m.Sys, - DownloadFinished: d.stats.Completed, - TorrentMetadataReady: d.stats.MetadataReady, - }) } + + diagnostics.Send(diagnostics.SnapshotDownloadStatistics{ + Downloaded: bytesDone, + Total: d.stats.BytesTotal, + TotalTime: time.Since(d.startTime).Round(time.Second).Seconds(), + DownloadRate: d.stats.DownloadRate, + UploadRate: d.stats.UploadRate, + Peers: d.stats.PeersUnique, + Files: d.stats.FilesTotal, + Connections: d.stats.ConnectionsTotal, + Alloc: m.Alloc, + Sys: m.Sys, + DownloadFinished: d.stats.Completed, + TorrentMetadataReady: d.stats.MetadataReady, + }) } func calculateTime(amountLeft, rate uint64) string { diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 2d96fdd3bf9..1d00e35ea69 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -81,10 +81,10 @@ require ( github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/anacrolix/chansync v0.3.0 // indirect github.com/anacrolix/envpprof v1.3.0 // indirect - github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 // indirect + github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab // indirect github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/perf v1.0.0 // indirect - github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 // indirect + github.com/anacrolix/missinggo/v2 v2.7.3 // indirect github.com/anacrolix/mmsg v1.0.0 // indirect github.com/anacrolix/multiless v0.3.1-0.20221221005021-2d12701f83f7 // indirect github.com/anacrolix/stm v0.4.1-0.20221221005312-96d17df0e496 // indirect @@ -106,7 +106,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35 github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect - github.com/go-llsqlite/crawshaw v0.4.0 // indirect + github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 3559d3e1e95..4162cf6669d 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -37,8 +37,8 @@ github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54g github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tcJDk= github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 h1:Kmcl3I9K2+5AdnnR7hvrnVT0TLeFWWMa9bxnm55aVIg= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab h1:MvuAC/UJtcohN6xWc8zYXSZfllh1LVNepQ0R3BCX5I4= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/go-libutp v1.3.1 h1:idJzreNLl+hNjGC3ZnUOjujEaryeOGgkwHLqSGoige0= github.com/anacrolix/go-libutp v1.3.1/go.mod h1:heF41EC8kN0qCLMokLBVkB8NXiLwx3t8R8810MTNI5o= github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= @@ -60,8 +60,8 @@ github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5ur github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY= github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA= github.com/anacrolix/missinggo/v2 v2.5.2/go.mod h1:yNvsLrtZYRYCOI+KRH/JM8TodHjtIE/bjOGhQaLOWIE= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 h1:W/oGeHhYwxueeiDjQfmK9G+X9M2xJgfTtow62v0TWAs= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= +github.com/anacrolix/missinggo/v2 v2.7.3 h1:Ee//CmZBMadeNiYB/hHo9ly2PFOEZ4Fhsbnug3rDAIE= +github.com/anacrolix/missinggo/v2 v2.7.3/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= github.com/anacrolix/mmsg v0.0.0-20180515031531-a4a3ba1fc8bb/go.mod h1:x2/ErsYUmT77kezS63+wzZp8E3byYB0gzirM/WMBLfw= github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg= github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= @@ -182,8 +182,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 h1:OyQmpAN302wAopDgwVjgs2HkFawP9ahIEqkUYz7V7CA= github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= -github.com/go-llsqlite/crawshaw v0.4.0 h1:L02s2jZBBJj80xm1VkkdyB/JlQ/Fi0kLbNHfXA8yrec= -github.com/go-llsqlite/crawshaw v0.4.0/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= +github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568 h1:3EpZo8LxIzF4q3BT+vttQQlRfA6uTtTb/cxVisWa5HM= +github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=