diff --git a/.gitignore b/.gitignore index 7fb352b..f5c4b98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ bin/ -/locode/ tmp/ .idea diff --git a/CHANGELOG.md b/CHANGELOG.md index ffaedad..f7d2104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Changelog for NeoFS Monitor ## [Unreleased] +### Changed +- Usage of Locode DB Go package (#100) + +### Removed +- Locode DB configuration options (#100) + ## [0.9.5] - 2022-12-29 ### Changed diff --git a/Makefile b/Makefile index 0ae3568..ecb871f 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,7 @@ APP = neofs-net-monitor BINARY = ./bin/${APP} SRC = ./cmd/${APP}/ -LOCODE_DIR = ./locode -LOCODE_FILE = locode_db.gz -LOCODE_DB_URL = https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.2.1/locode_db.gz - -.PHONY: bin image up up-testnet up-devenv down down-testnet down-devenv clean locode lint test cover +.PHONY: bin image up up-testnet up-devenv down down-testnet down-devenv clean lint test cover bin: @echo "Build neofs-net-monitor binary" @@ -31,21 +27,13 @@ image: --build-arg VERSION=$(VERSION) \ -t ${REPO}/${APP}:$(HUB_TAG) . -locode: - @mkdir -p ${LOCODE_DIR} - @echo "⇒ Download NeoFS LOCODE database from ${LOCODE_DB_URL}" - @curl \ - -sSL "${LOCODE_DB_URL}" \ - -o ${LOCODE_DIR}/${LOCODE_FILE} - gzip -dfk ${LOCODE_DIR}/${LOCODE_FILE} - -up: locode +up: @docker-compose -f docker/docker-compose.yml --env-file docker/mainnet.env up -d -up-testnet: locode +up-testnet: @docker-compose -f docker/docker-compose.yml --env-file docker/testnet.env up -d -up-devenv: locode +up-devenv: @docker-compose -f docker/docker-compose.devenv.yml up -d down: @@ -59,7 +47,6 @@ down-devenv: clean: rm -f ${BINARY} - rm -rf ${LOCODE_DIR} version: @echo ${VERSION} diff --git a/README.md b/README.md index f89bfa0..9f2255d 100644 --- a/README.md +++ b/README.md @@ -44,14 +44,6 @@ $ neofs-net-monitor --config config.yaml Also, you can provide all options using env variables. -To download actual LOCODE database run `$ make locode`. -Visit LOCODE [repository](https://github.com/nspcc-dev/neofs-locode-db) for more information. - -``` -// Optional path to NeoFS LOCODE database. -NEOFS_NET_MONITOR_LOCODE_DB_PATH=path/to/database -``` - ## Connect to neofs-dev-env After `Jebudo` release monitor can be attached to diff --git a/cmd/neofs-net-monitor/config.go b/cmd/neofs-net-monitor/config.go index 3b90466..d0abb7b 100644 --- a/cmd/neofs-net-monitor/config.go +++ b/cmd/neofs-net-monitor/config.go @@ -26,9 +26,6 @@ const ( cfgMetricsEndpoint = "metrics.endpoint" cfgMetricsInterval = "metrics.interval" - // path to the NeoFS locode database - cfgLocodeDB = "locode.db.path" - // level of logging cfgLoggerLevel = "logger.level" ) @@ -43,8 +40,6 @@ func DefaultConfiguration(cfg *viper.Viper) { cfg.SetDefault(cfgMetricsEndpoint, ":16512") cfg.SetDefault(cfgMetricsInterval, 15*time.Minute) - cfg.SetDefault(cfgLocodeDB, "./locode/db") - cfg.SetDefault(cfgLoggerLevel, "info") } diff --git a/cmd/neofs-net-monitor/monitor.go b/cmd/neofs-net-monitor/monitor.go index a179b41..9c352b9 100644 --- a/cmd/neofs-net-monitor/monitor.go +++ b/cmd/neofs-net-monitor/monitor.go @@ -6,7 +6,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" rpcnns "github.com/nspcc-dev/neofs-contract/rpc/nns" - "github.com/nspcc-dev/neofs-net-monitor/pkg/locode" "github.com/nspcc-dev/neofs-net-monitor/pkg/monitor" "github.com/nspcc-dev/neofs-net-monitor/pkg/morphchain" "github.com/nspcc-dev/neofs-net-monitor/pkg/morphchain/contracts" @@ -124,12 +123,6 @@ func New(ctx context.Context, cfg *viper.Viper) (*monitor.Monitor, error) { logger.Info("neofs contract ignored") } - geoFetcher := locode.New( - locode.Prm{ - Path: cfg.GetString(cfgLocodeDB), - }, - ) - return monitor.New(monitor.Args{ Balance: balance, Proxy: proxy, @@ -137,7 +130,6 @@ func New(ctx context.Context, cfg *viper.Viper) (*monitor.Monitor, error) { Logger: logger, Sleep: cfg.GetDuration(cfgMetricsInterval), MetricsAddress: cfg.GetString(cfgMetricsEndpoint), - GeoFetcher: geoFetcher, AlpFetcher: alphabetFetcher, NmFetcher: nmFetcher, IRFetcher: nmFetcher, diff --git a/config/config.env b/config/config.env index df9e50b..fa48e46 100644 --- a/config/config.env +++ b/config/config.env @@ -17,5 +17,3 @@ NEOFS_NET_MONITOR_METRICS_INTERVAL=15m # NeoFS contract from main chain. Required for asset supply metric. NEOFS_NET_MONITOR_CONTRACTS_NEOFS=3c3f4b84773ef0141576e48c3ff60e5078235891 -# Optional path to NeoFS LOCODE database. -NEOFS_NET_MONITOR_LOCODE_DB_PATH=path/to/database diff --git a/config/config.yaml b/config/config.yaml index 17bc48a..2c952e6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -23,11 +23,6 @@ metrics: interval: 15s endpoint: ":16512" -# Optional path to NeoFS LOCODE database. -locode: - db: - path: locode/db - contracts: # NeoFS contract from main chain. Required for asset supply metric. neofs: 3c3f4b84773ef0141576e48c3ff60e5078235891 diff --git a/docker/docker-compose.devenv.yml b/docker/docker-compose.devenv.yml index 86e8d81..8b052b2 100644 --- a/docker/docker-compose.devenv.yml +++ b/docker/docker-compose.devenv.yml @@ -43,11 +43,8 @@ services: - prometheus environment: - NEOFS_NET_MONITOR_METRICS_INTERVAL=15s - - NEOFS_NET_MONITOR_LOCODE_DB_PATH=locode/db - NEOFS_NET_MONITOR_MAINNET_RPC_ENDPOINT=http://192.168.130.50:30333 - NEOFS_NET_MONITOR_MORPH_RPC_ENDPOINT=http://192.168.130.90:30333 - volumes: - - ./../locode/locode_db:/locode/db networks: - internet restart: on-failure diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 53a164b..48c1cba 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,12 +40,9 @@ services: - prometheus environment: - NEOFS_NET_MONITOR_METRICS_INTERVAL=15s - - NEOFS_NET_MONITOR_LOCODE_DB_PATH=locode/db - NEOFS_NET_MONITOR_MAINNET_RPC_ENDPOINT=${NEOFS_NET_MONITOR_MAINNET_RPC_ENDPOINT} - NEOFS_NET_MONITOR_MORPH_RPC_ENDPOINT=${NEOFS_NET_MONITOR_MORPH_RPC_ENDPOINT} - NEOFS_NET_MONITOR_CONTRACTS_NEOFS=${NEOFS_NET_MONITOR_CONTRACTS_NEOFS} - volumes: - - ./../locode/locode_db:/locode/db networks: - monitor-net restart: on-failure diff --git a/go.mod b/go.mod index 428cf69..5dfda4c 100644 --- a/go.mod +++ b/go.mod @@ -12,22 +12,22 @@ require ( github.com/prometheus/client_golang v1.17.0 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 - go.etcd.io/bbolt v1.3.7 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.12.0 // indirect ) +require ( + github.com/google/uuid v1.3.0 + github.com/nspcc-dev/locode-db v0.4.1-0.20231109111047-07567f7bb52c +) + require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/ipfs/go-cid v0.0.7 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -41,7 +41,6 @@ require ( github.com/multiformats/go-multibase v0.0.3 // indirect github.com/multiformats/go-multihash v0.0.14 // indirect github.com/multiformats/go-varint v0.0.6 // indirect - github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230919154019-f66abd020d6a // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect @@ -55,7 +54,6 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect golang.org/x/net v0.14.0 // indirect diff --git a/go.sum b/go.sum index 0d035f6..3b12a52 100644 --- a/go.sum +++ b/go.sum @@ -190,6 +190,8 @@ github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaF github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= +github.com/nspcc-dev/locode-db v0.4.1-0.20231109111047-07567f7bb52c h1:YOYctsLYN6LniG146qogXx7IPDjIXi3QLfibAC7nWwk= +github.com/nspcc-dev/locode-db v0.4.1-0.20231109111047-07567f7bb52c/go.mod h1:DXkhJM1Wrh2JY5sAv9gI1G8XneGYS8zjlEhwU4f6KPw= github.com/nspcc-dev/neo-go v0.102.0 h1:O2Gt4JPOWmp0c+PnPWwd2wPI74BKSwkaNCEyvyQTWJw= github.com/nspcc-dev/neo-go v0.102.0/go.mod h1:QXxpZxJT2KedwM0Nlj8UO0/fZN2WIe4h/i03uBHKbnc= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230919154019-f66abd020d6a h1:ZF1T2PfRJh0LbQcMQAPMmp94B27o/6DKZV+zcMIUNHM= @@ -259,7 +261,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/pkg/locode/calls.go b/pkg/locode/calls.go deleted file mode 100644 index b569a4f..0000000 --- a/pkg/locode/calls.go +++ /dev/null @@ -1,135 +0,0 @@ -package locode - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "strings" - - "go.etcd.io/bbolt" -) - -var errNotExist = errors.New("database file does not exists") - -// Open opens underlying BoltDB instance. -// -// Timeout of BoltDB opening is 3s (only for Linux or Darwin). -func (db *DB) Open() error { - _, err := os.Stat(db.path) - if os.IsNotExist(err) { - return errNotExist - } - - db.bolt, err = bbolt.Open(db.path, db.mode, db.boltOpts) - if err != nil { - return fmt.Errorf("could not open BoltDB: %w", err) - } - - return nil -} - -// Close closes underlying BoltDB instance. -// -// Must not be called before successful Open call. -func (db *DB) Close() error { - if db.bolt != nil { - return db.bolt.Close() - } - - return nil -} - -type recordJSON struct { - LocationName string `json:"LocationName"` - Lat float64 `json:"Latitude"` - Long float64 `json:"Longitude"` -} - -func (r recordJSON) Latitude() float64 { - return r.Lat -} - -func (r recordJSON) Longitude() float64 { - return r.Long -} - -func (r recordJSON) Location() string { - return r.LocationName -} - -var ( - errRecordNotFound = errors.New("could not get record by provided locode") - errDBNotInitialized = errors.New("DB instance not initialized") -) - -type ( - countryCode = [2]byte - locationCode = [3]byte - - Position interface { - Latitude() float64 - Longitude() float64 - Location() string - } -) - -func (db *DB) Get(locode string) (position Position, err error) { - if db.bolt == nil { - return nil, errDBNotInitialized - } - - cc, lc, err := parseLocode(locode) - if err != nil { - return nil, err - } - - err = db.bolt.View(func(tx *bbolt.Tx) error { - bktCountry := tx.Bucket(cc[:]) - if bktCountry == nil { - return errRecordNotFound - } - - data := bktCountry.Get(lc[:]) - if data == nil { - return errRecordNotFound - } - - rj := new(recordJSON) - - err := json.Unmarshal(data, rj) - if err != nil { - return err - } - - position = rj - - return err - }) - - return -} - -var errIncorrectLocode = errors.New("incorrect locode") - -func parseLocode(l string) (cc countryCode, lc locationCode, err error) { - locParts := strings.Split(l, " ") - - if len(locParts) != 2 { - return cc, lc, errIncorrectLocode - } - - if len(locParts[0]) != 2 { - return cc, lc, errIncorrectLocode - } - - copy(cc[:], locParts[0]) - - if len(locParts[1]) != 3 { - return cc, lc, errIncorrectLocode - } - - copy(lc[:], locParts[1]) - - return -} diff --git a/pkg/locode/db.go b/pkg/locode/db.go deleted file mode 100644 index ccdcb00..0000000 --- a/pkg/locode/db.go +++ /dev/null @@ -1,72 +0,0 @@ -package locode - -import ( - "fmt" - "io/fs" - "os" - "time" - - "go.etcd.io/bbolt" -) - -// Prm groups the required parameters of the DB's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Path to BoltDB file with NeoFS location database. - // - // Must not be empty. - Path string -} - -// DB is a descriptor of the NeoFS BoltDB location database. -// -// For correct operation, DB must be created -// using the constructor (New) based on the required parameters -// and optional components. -// -// After successful creation, -// DB must be opened through Open call. After successful opening, -// DB is ready to work through API (until Close call). -// -// Upon completion of work with the DB, it must be closed -// by Close method. -type DB struct { - path string - - mode fs.FileMode - - boltOpts *bbolt.Options - - bolt *bbolt.DB -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the DB. DB is read-only. -// -// Panics if at least one value of the parameters is invalid. -// -// The created DB requires calling the Open method in order -// to initialize required resources. -func New(prm Prm) *DB { - switch { - case prm.Path == "": - panicOnPrmValue("Path", prm.Path) - } - - return &DB{ - path: prm.Path, - mode: os.ModePerm, - boltOpts: &bbolt.Options{ - Timeout: 3 * time.Second, - ReadOnly: true, - }, - } -} diff --git a/pkg/monitor/monitor.go b/pkg/monitor/monitor.go index 62678f6..7985749 100644 --- a/pkg/monitor/monitor.go +++ b/pkg/monitor/monitor.go @@ -7,11 +7,12 @@ import ( "net/http" "sort" "strconv" + "strings" "time" + "github.com/nspcc-dev/locode-db/pkg/locodedb" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neofs-net-monitor/pkg/locode" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" @@ -69,7 +70,6 @@ type ( logger *zap.Logger sleep time.Duration metricsServer http.Server - geoFetcher *locode.DB alpFetcher AlphabetFetcher nmFetcher NetmapFetcher irFetcher InnerRingFetcher @@ -85,7 +85,6 @@ type ( Logger *zap.Logger Sleep time.Duration MetricsAddress string - GeoFetcher *locode.DB AlpFetcher AlphabetFetcher NmFetcher NetmapFetcher IRFetcher InnerRingFetcher @@ -106,7 +105,6 @@ func New(p Args) *Monitor { Addr: p.MetricsAddress, Handler: promhttp.Handler(), }, - geoFetcher: p.GeoFetcher, alpFetcher: p.AlpFetcher, nmFetcher: p.NmFetcher, irFetcher: p.IRFetcher, @@ -134,10 +132,6 @@ func (m *Monitor) Start(ctx context.Context) { prometheus.MustRegister(alphabetSideDivergence) prometheus.MustRegister(containersNumber) - if err := m.geoFetcher.Open(); err != nil { - m.logger.Warn("geoposition fetching disabled", zap.Error(err)) - } - go func() { err := m.metricsServer.ListenAndServe() if !errors.Is(err, http.ErrServerClosed) { @@ -153,8 +147,6 @@ func (m *Monitor) Stop() { if err != nil { m.logger.Error("stop metrics server error", zap.Error(err)) } - - _ = m.geoFetcher.Close() } func (m *Monitor) Job(ctx context.Context) { @@ -248,14 +240,14 @@ func (m *Monitor) processNetworkMap(nm NetmapInfo, candidates NetmapCandidatesIn exportBalancesGAS[keyHex] = balanceGAS } - pos, err := m.geoFetcher.Get(node.Locode) + record, err := locodedb.Get(strings.ReplaceAll(node.Locode, " ", "")) if err != nil { m.logger.Debug("can't fetch geoposition", zap.String("key", keyHex), zap.Error(err)) } else { nodeLoc := nodeLocation{ - name: pos.Location(), - long: strconv.FormatFloat(pos.Longitude(), 'f', 4, 64), - lat: strconv.FormatFloat(pos.Latitude(), 'f', 4, 64), + name: record.Location, + long: strconv.FormatFloat(record.Point.Longitude(), 'f', 4, 64), + lat: strconv.FormatFloat(record.Point.Latitude(), 'f', 4, 64), } exportCountries[nodeLoc]++