diff --git a/src/index.js b/src/index.js index c07d70c2f6..b4ff32771d 100644 --- a/src/index.js +++ b/src/index.js @@ -328,7 +328,7 @@ class Libp2p extends EventEmitter { async ping (peer) { const peerInfo = await getPeerInfo(peer, this.peerStore) - return ping(this, peerInfo) + return ping(this, peerInfo.id) } /** diff --git a/src/peer-store/README.md b/src/peer-store/README.md index d9d79fe56a..056c4b7010 100644 --- a/src/peer-store/README.md +++ b/src/peer-store/README.md @@ -1,3 +1,156 @@ # Peerstore -WIP \ No newline at end of file +Libp2p's Peerstore is responsible for keeping an updated register with the relevant information of the known peers. It should gather environment changes and be able to take decisions and notice interested parties of relevant changes. The Peerstore comprises four main components: `addressBook`, `keyBook`, `protocolBook` and `metadataBook`. These book components have similar characteristics with the `Javascript Map` implementation. + +The PeerStore needs to manage the high level operations on its inner books, have a job runner to trigger other books runners for data trimming or computations. Moreover, the peerStore should be responsible for noticing interested parties of relevant events, through its Event Emitter. + +## Peers Environment + +#### Sense + +Several libp2p subsystems will perform operations, which will gather relevant information about peers. Some operations might not have this as an end goal, but can also gather important data. + +In a libp2p node life, it will discover peers the existance of peers through its discovery protocols. In a typical discovery protocol, an address of the peer is discovered combined with its peer id. Once this happens, the `PeerStore` should collect this information for future (or immediate) usage by other subsystems. When the information is stored, the `PeerStore` should inform interested parties of the peer discovered (`peer` event). + +Taking into account a different scenario, a peer might perform/receive a dial request to/from a unkwown peer. In such a scenario, the `PeerStore` must store the peer's multiaddr once a connection is established. + +(NOTE: this should be removed later) +(currently we silently put it to the peerStore, without emitting events, as this logic exists in the `onConnected` callback from the upgrader. This way, we are never emitting the `peer` event when inbound connections happen, or a unkwown peer is dialed. Should we differentiate this?) + +After a connection is established with a peer, the Identify Service will act on this connection. A stream is created and peers exchange their information (listenMuldiaddrs and running protocols). Once this information is obtained, the PeerStore can collect the new data. In this specific case, we have a guarantee that this data is complete and updated, so the data stored in the PeerStore should be replaced (older and outdated data should disappear). However, if the recorded `multiaddrs` or `protocols` have changed, interested parties must be informed via `change:multiaddrs` or `change:protocols` events. + +In the background, the Identify Service is also waiting for new protocols to be started by the peer. If a new protocol is started, the `identify-push` message is sent to all the connected peers, so that their PeerStore can be updated with the new protocol and relevant parties are noticed. As the `identify-push` also sends complete and updated information, the data in the PeerStore is replaced. + +On different context, it is also possible to gather relevant information for the peers. For instance, in `dht` operations, nodes can exchanges data of peers they know as part of the `dht` operation. In this case, we can get information from a peer that we already know. As a consequence, the `PeerStore` should act accordingly and not replace the data it owns, but just try to merge it the discovered data is new. For example, discovered a new address of a peer. + +#### Act + +When the `PeerStore` data is updated, this information might be important for different parties. + +`js-libp2p` keeps a topology of peers for each protocol a node is running. This way, once a protocol is supported for a peer, the topology of that protocol should be informed that a new peer may be used and the subsystem can decide if it should open a new stream it that peer or not. + +Every time a peer needs to dial another peer, it is essential that it knows the multiaddrs used by the peer, in order to perform a successful dial to a peer. The same is true for pinging a peer. + +## PeerStore implementation + +(Note: except for highlighting the APIs functionallity, they should be better formally described on `API.md` file) + +#### API: + +Access to its underlying books: + +- `peerStore.protoBook.*` +- `peerStore.addressBook.*` + +High level operations: + +- `peerStore.set(peerId, data, options)` or `events` + +High level set which should be able to identify the type of data received and forward to the appropriate book sets. More than a bridge, this aims to allow the combination of multiple data storage as follows: + +`data = { multiaddrs: [...], protocols: [...] }` + +---- (should be removed / re-written, but important for design decisions) + +One aspect that we need to consider is wether we should add information to every book, even if we don't have any relevant information for it. For instance, if we just discover a peer via a typical discovery service, we will have the `peerId` and an array of `multiaddr`. When we do `peerStore.set()`, should we also do `protoBook.set()` with an empty list of protocols? I don't see any advantage on adding to the remaining ones. + +**IMPORTANT:** This is one of the biggest design decisions to make (set vs events). The programmatic API is the easiest solution but it can provide users an API that they sould not need. If we go on an event-based approach, the `peerStore` should receive all the relevant subsystems (discovery, identifyService, ...) and sense the environment (via events) to gather the information that would need to be sent via the API. Thile the latest seems the best solution, it is the more complex one to implement, as we would ideally have an interface that those subsystems would need to implement and each time we have a new subsystem that needs to add data to the peerStore, we might need to update the `peer-store` codebase (or have a good set of abstractions). + +It is also important pointing out that users would be able to use `peerStore.protoBook.*`, so eventually we should move those into `peerStore._ protoBook.*` if we do not intend them to use it. + +--- + +- `peerStore.get(peerId, options)` + +Get the information of a provided peer. The amount of information that we want can be customized with the following options, which are true by default: + +```js +{ + address: true, + proto: true, + key: true, + metadata: true +} +``` + +- `peerStore.delete(peerId, [data])` + +Deletes the provided peer from every book. If data is provided, just remove the data from the books. The data should be provided as follows: + +```js +{ + address: [], + proto: [], + key: [], + metadata: [] +} +``` + +- `peerStore.peers(options)` + +Get an array of all the peers, as well as their information. The information intended can be customized with the following options, which are true by default: + +```js +{ + address: true, + proto: true, + key: true, + metadata: true +} +``` + +## Address Book + +The `addressBook` keeps the known multiaddrs of a peer. The multiaddrs of each peer are not a constant and the Address book must have this into consideration. + +`Map` + +A `peerId.toString()` identifier mapping to a `multiaddrInfo` object, which should have the following structure: + +```js +{ + multiaddr: , + validity: , + confidence: +} +``` + +**Note:** except for multiaddr namings, the other properties are placeholders for now and might not be as described in the future milestones. + +- `addressBook.set()` +- `addressBook.get()` +- `getMultiaddrsForPeer()` +- `addressBook.has()` +- `addressBook.delete()` +- `addressBook.peers()` + +It is important pointing out that the API methods which return arrays of data (`set`, `get`, `getMultiaddrsForPeer`) shuld return the `multiaddr` property of the `multiaddrInfo` and not the entire `multiaddrInfo` as the remaining data should be used internally. Should we consider having two datastructure instead? + +Further API methods will probably be added in the context of multiaddr `ttl` and multiaddr confidence. + +**Not Yet Implemented**: Multiaddr Confidence + +## Key Book + +The `keyBook` tracks the keys of the peers. + +**Not Yet Implemented** + +## Protocol Book + +The `protoBook` holds the identifiers of the protocols supported by each peer. The protocols supported by each peer are dynamic and will change over time. + +`Map>` + +A `peerId.toString()` identifier mapping to a `Set` of protocol identifier strings. + +- `protoBook.set()` +- `protoBook.get()` +- `protoBook.has()` +- `protoBook.delete()` +- `protoBook.supports()` +- `protoBook.peers()` + +## Metadata Book + +**Not Yet Implemented** diff --git a/src/peer-store/address-book.js b/src/peer-store/address-book.js new file mode 100644 index 0000000000..e246bc9be6 --- /dev/null +++ b/src/peer-store/address-book.js @@ -0,0 +1,269 @@ +'use strict' + +const errcode = require('err-code') +const debug = require('debug') +const log = debug('libp2p:peer-store:address-book') +log.error = debug('libp2p:peer-store:address-book:error') + +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') + +const { + ERR_INVALID_PARAMETERS +} = require('../errors') + +/** + * The AddressBook is responsible for keeping the known multiaddrs + * of a peer. + */ +class AddressBook { + /** + * MultiaddrInfo object + * @typedef {Object} multiaddrInfo + * @property {Multiaddr} multiaddr peer multiaddr. + * @property {number} validity NOT USED YET + * @property {number} confidence NOT USED YET + */ + + /** + * @constructor + * @param {EventEmitter} peerStore + */ + constructor (peerStore) { + /** + * PeerStore Event emitter, used by the AddressBook to emit: + * "peer" - emitted when a peer is discovered by the node. + * "change:multiaddrs" - emitted when the known multiaddrs of a peer change. + */ + this._ps = peerStore + + /** + * Map known peers to their known multiaddrs. + * @type {Map} + */ + this.addressBook = new Map() + } + + /** + * Set known addresses of a provided peer. + * @param {PeerId} peerId + * @param {Array|Multiaddr} addresses + * @param {Object} [options] + * @param {boolean} [options.replace = true] wether addresses received replace stored ones or a unique union is performed. + * @returns {Array} + */ + set (peerId, addresses, { replace = true }) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + if (!addresses) { + throw errcode(new Error('addresses must be provided'), ERR_INVALID_PARAMETERS) + } + + if (!Array.isArray(addresses)) { + addresses = [addresses] + } + + // create multiaddrInfo for each address + const multiaddrInfos = [] + addresses.forEach((addr) => { + if (!multiaddr.isMultiaddr(addr)) { + throw errcode(new Error(`multiaddr ${addr} must be an instance of multiaddr`), ERR_INVALID_PARAMETERS) + } + + multiaddrInfos.push({ + multiaddr: addr + }) + }) + + if (replace) { + return this._replace(peerId, multiaddrInfos) + } + + return this._add(peerId, multiaddrInfos) + } + + /** + * Replace known addresses to a provided peer. + * If the peer is not known, it is set with the given addresses. + * @param {PeerId} peerId + * @param {Array} multiaddrInfos + * @returns {Array} + */ + _replace (peerId, multiaddrInfos) { + const id = peerId.toString() + const rec = this.addressBook.get(id) + + // Already know the peer + if (rec && rec.length === multiaddrInfos.length) { + const intersection = rec.filter((mi) => multiaddrInfos.some((newMi) => mi.multiaddr === newMi.multiaddr)) + + // New addresses equal the old ones? + // If yes, no changes needed! + if (intersection.length === rec.length) { + return [...multiaddrInfos] + } + } + + this.addressBook.set(id, multiaddrInfos) + + this._ps.emit('peer', peerId) + this._ps.emit('change:multiaddrs', { + peerId, + multiaddrs: multiaddrInfos.map((mi) => mi.multiaddr) + }) + + return [...multiaddrInfos] + } + + /** + * Add new known addresses to a provided peer. + * If the peer is not known, it is set with the given addresses. + * @param {PeerId} peerId + * @param {Array} multiaddrInfos + * @returns {Array} + */ + _add (peerId, multiaddrInfos) { + const id = peerId.toString() + const rec = this.addressBook.get(id) || [] + + // Add recorded uniquely to the new array + rec.forEach((mi) => { + if (!multiaddrInfos.find(r => r.multiaddr === mi.multiaddr)) { + multiaddrInfos.push(mi) + } + }) + + // If the recorded length is equal to the new after the uniquely union + // The content is the same, no need to update. + if (rec.length === multiaddrInfos) { + return [...multiaddrInfos] + } + + this.addressBook.set(id, multiaddrInfos) + this._ps.emit('change:multiaddrs', { + peerId, + multiaddrs: multiaddrInfos.map((mi) => mi.multiaddr) + }) + + // Notify the existance of a new peer + // TODO: do we need this? + if (!rec) { + this._ps.emit('peer', peerId) + } + + return [...multiaddrInfos] + } + + /** + * Get known addresses of a provided peer. + * @param {PeerId} peerId + * @returns {Array} + */ + get (peerId) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + const record = this.addressBook.get(peerId.toString()) + + if (!record) { + return undefined + } + + return record.map((multiaddrInfo) => multiaddrInfo.multiaddr) + } + + /** + * Get the known multiaddrs for a given peer. All returned multiaddrs + * will include the encapsulated `PeerId` of the peer. + * @param {PeerId} peer + * @returns {Array} + */ + getMultiaddrsForPeer (peerId) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + const record = this.addressBook.get(peerId.toString()) + + if (!record) { + return undefined + } + + return record.map((multiaddrInfo) => { + const addr = multiaddrInfo.multiaddr + + if (addr.getPeerId()) return addr + return addr.encapsulate(`/p2p/${peerId.toB58String()}`) + }) + } + + /** + * Has known addresses of a provided peer. + * @param {PeerId} peerId + * @returns {boolean} + */ + has (peerId) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + return this.addressBook.has(peerId.toString()) + } + + /** + * Deletes the provided peer from the book. + * If addresses are provided, just remove the provided addresses and keep the peer. + * @param {PeerId} peerId + * @param {Array|multiaddr} [addresses] + * @returns {boolean} + */ + delete (peerId, addresses) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + if (addresses) { + return this._remove(peerId, addresses) + } + + this._ps('change:multiaddrs', { + peerId, + multiaddrs: [] + }) + + return this.addressBook.delete(peerId.toString()) + } + + /** + * Removes the given multiaddrs from the provided peer. + * @param {PeerId} peerId + * @param {Array|multiaddr} addresses + * @returns {boolean} + */ + _remove (peerId, addresses) { + if (!Array.isArray(addresses)) { + addresses = [addresses] + } + + const record = this.addressBook.get(peerId.toString()) + + if (!record) { + return false + } + + record.filter((mi) => addresses.includes(mi.multiaddr)) + // TODO: should we keep it if empty? + + this._ps('change:multiaddrs', { + peerId, + multiaddrs: record.map((multiaddrInfo) => multiaddrInfo.multiaddr) + }) + + return true + } +} + +module.exports = AddressBook diff --git a/src/peer-store/index.js b/src/peer-store/index.js index 3fe4954e04..8bad0fa782 100644 --- a/src/peer-store/index.js +++ b/src/peer-store/index.js @@ -30,11 +30,17 @@ class PeerStore extends EventEmitter { */ this.peers = new Map() - // TODO: Track ourselves. We should split `peerInfo` up into its pieces so we get better - // control and observability. This will be the initial step for removing PeerInfo - // https://github.com/libp2p/go-libp2p-core/blob/master/peerstore/peerstore.go - // this.addressBook = new Map() - // this.protoBook = new Map() + /** + * Map known peers to their known multiaddrs. + * @type {Map} + */ + this.addressBook = new Map() + + /** + * Map known peers to their known supported protocols. + * @type {Map} + */ + this.protoBook = new Map() } /** @@ -82,8 +88,8 @@ class PeerStore extends EventEmitter { peerInfo.multiaddrs.forEach((ma) => newPeerInfo.multiaddrs.add(ma)) peerInfo.protocols.forEach((p) => newPeerInfo.protocols.add(p)) - const connectedMa = peerInfo.isConnected() - connectedMa && newPeerInfo.connect(connectedMa) + // const connectedMa = peerInfo.isConnected() + // connectedMa && newPeerInfo.connect(connectedMa) const peerProxy = new Proxy(newPeerInfo, { set: (obj, prop, value) => { @@ -120,10 +126,10 @@ class PeerStore extends EventEmitter { const recorded = this.peers.get(id) // pass active connection state - const ma = peerInfo.isConnected() - if (ma) { - recorded.connect(ma) - } + // const ma = peerInfo.isConnected() + // if (ma) { + // recorded.connect(ma) + // } // Verify new multiaddrs // TODO: better track added and removed multiaddrs @@ -229,6 +235,7 @@ class PeerStore extends EventEmitter { peerInfo, multiaddrs: peerInfo.multiaddrs.toArray() }) + this.emit('change:protocols', { peerInfo, protocols: Array.from(peerInfo.protocols) diff --git a/src/peer-store/proto-book.js b/src/peer-store/proto-book.js new file mode 100644 index 0000000000..20ecff931f --- /dev/null +++ b/src/peer-store/proto-book.js @@ -0,0 +1,233 @@ +'use strict' + +const errcode = require('err-code') +const debug = require('debug') +const log = debug('libp2p:peer-store:proto-book') +log.error = debug('libp2p:peer-store:proto-book:error') + +const PeerId = require('peer-id') + +const { + ERR_INVALID_PARAMETERS +} = require('../errors') + +/** + * The ProtoBook is responsible for keeping the known suppoerted + * protocols of a peer. + * @fires ProtoBook#change:protocols + */ +class ProtoBook { + /** + * @constructor + * @param {EventEmitter} peerStore + */ + constructor (peerStore) { + /** + * PeerStore Event emitter, used by the ProtoBook to emit: + * "change:protocols" - emitted when the known protocols of a peer change. + */ + this._ps = peerStore + + /** + * Map known peers to their known protocols. + * @type {Map} + */ + this.protoBook = new Map() + } + + /** + * Set known protocols of a provided peer. + * If the peer was not known before, it will be added. + * @param {PeerId} peerId + * @param {Array|string} protocols + * @param {Object} [options] + * @param {boolean} [options.replace = true] wether protocols received replace stored ones or a unique union is performed. + * @returns {Array} + */ + set (peerId, protocols, { replace = true }) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + if (!protocols) { + throw errcode(new Error('protocols must be provided'), ERR_INVALID_PARAMETERS) + } + + if (!Array.isArray(protocols)) { + protocols = [protocols] + } + + if (replace) { + return this._replace(PeerId, protocols) + } + + return this._add(PeerId, protocols) + } + + /** + * Replace known protocols to a provided peer. + * If the peer is not known, it is set with the given protocols. + * @param {PeerId} peerId + * @param {Array} protocols + * @returns {Array} + */ + _replace (peerId, protocols) { + const id = peerId.toString() + const recSet = this.protoBook.get(id) + const newSet = new Set(protocols) + + const isSetEqual = (a, b) => a.size === b.size && [...a].every(value => b.has(value)) + + // Already know the peer and the recorded protocols are the same? + // If yes, no changes needed! + if (recSet && isSetEqual(recSet, newSet)) { + return protocols + } + + this.protoBook.set(id, newSet) + this._ps('change:protocols', { + peerId, + protocols + }) + + return protocols + } + + /** + * Add new known protocols to a provided peer. + * If the peer is not known, it is set with the given protocols. + * @param {PeerId} peerId + * @param {Array|string} protocols + * @returns {Array} + */ + _add (peerId, protocols) { + const id = peerId.toString() + const recSet = this.protoBook.get(id) || new Set() + const newSet = new Set([...recSet, ...protocols]) + + // Any new protocol added? + if (recSet.size === newSet.size) { + return protocols + } + + protocols = [...newSet] + + this.protoBook.set(id, newSet) + this._ps('change:protocols', { + peerId, + protocols + }) + + return protocols + } + + /** + * Get known supported protocols of a provided peer. + * @param {PeerId} peerId + * @returns {Array} + */ + get (peerId) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + const recSet = this.protoBook.get(peerId.toString()) + + return recSet ? [...recSet] : undefined + } + + /** + * Verify if the provided peer supports the given protocols. + * @param {PeerId} peerId + * @param {Array|string} protocols + * @returns {boolean} + */ + supports (peerId, protocols) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + if (!Array.isArray(protocols)) { + protocols = [protocols] + } + + const recSet = this.protoBook.get(peerId.toString()) + + if (!recSet) { + return false + } + + return [...recSet].filter((p) => protocols.includes(p)).length === protocols.length + } + + /** + * Has known protocols of a provided peer. + * @param {PeerId} peerId + * @returns {boolean} + */ + has (peerId) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + return this.protoBook.has(peerId.toString()) + } + + /** + * Deletes the provided peer from the book. + * If protocols are provided, just remove the provided protocols and keep the peer. + * @param {PeerId} peerId + * @param {Array|string} [protocols] + * @returns {boolean} + */ + delete (peerId, protocols) { + if (!PeerId.isPeerId(peerId)) { + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + if (protocols) { + return this._remove(peerId, protocols) + } + + if (!this.protoBook.delete(peerId.toString())) { + return false + } + + this._ps('change:protocols', { + peerId, + protocols: [] + }) + + return true + } + + /** + * Removes the given protocols from the provided peer. + * @param {PeerId} peerId + * @param {Array|string} protocols + * @returns {boolean} + */ + _remove (peerId, protocols) { + if (!Array.isArray(protocols)) { + protocols = [protocols] + } + + const recSet = this.protoBook.get(peerId.toString()) + + if (!recSet) { + return false + } + + protocols.forEach((p) => recSet.delete(p)) + // TODO: should we keep it if empty? + + this._ps('change:protocols', { + peerId, + protocols: [...recSet] + }) + + return true + } +} + +module.exports = ProtoBook diff --git a/src/ping/index.js b/src/ping/index.js index d679a5c102..8590323056 100644 --- a/src/ping/index.js +++ b/src/ping/index.js @@ -15,11 +15,11 @@ const { PROTOCOL, PING_LENGTH } = require('./constants') /** * Ping a given peer and wait for its response, getting the operation latency. * @param {Libp2p} node - * @param {PeerInfo} peer + * @param {PeerId} peer * @returns {Promise} */ async function ping (node, peer) { - log('dialing %s to %s', PROTOCOL, peer.id.toB58String()) + log('dialing %s to %s', PROTOCOL, peer.toB58String()) const { stream } = await node.dialProtocol(peer, PROTOCOL) diff --git a/src/registrar.js b/src/registrar.js index cc24548516..43fa235d03 100644 --- a/src/registrar.js +++ b/src/registrar.js @@ -22,6 +22,8 @@ class Registrar { * @constructor */ constructor ({ peerStore }) { + // Used on topology to listen for protocol changes + // TODO: should we only provide the protobook? this.peerStore = peerStore /**