Skip to content

Commit

Permalink
chore: upgrade libp2p all the way (#2224)
Browse files Browse the repository at this point in the history
## Why is this change needed?

Upgrades libp2p all the way to the latest version ([Release
Notes](https://github.com/libp2p/js-libp2p/releases/tag/libp2p-v1.8.2) |
[Migration
Guide](https://github.com/libp2p/js-libp2p/blob/main/doc/migrations/v0.46-v1.0.0.md))

Upgrades GossipSub along with it.

No major interface changes were required here, just different dependency
references as the libp2p project moved some things around.

## Merge Checklist

_Choose all relevant options below by adding an `x` now or at any time
before submitting for review_

- [x] PR title adheres to the [conventional
commits](https://www.conventionalcommits.org/en/v1.0.0/) standard
- [x] PR has a
[changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets)
- [x] PR has been tagged with a change label(s) (i.e. documentation,
feature, bugfix, or chore)
- [ ] PR includes
[documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs)
if necessary.

<!-- start pr-codex -->

---

## PR-Codex overview
This PR upgrades `libp2p/gossipsub` and dependencies, aligns imports
with `@libp2p/interface`, and adjusts types in various files.

### Detailed summary
- Upgraded `libp2p/gossipsub` and dependencies
- Aligned imports with `@libp2p/interface`
- Adjusted types in multiple files for compatibility

> The following files were skipped due to too many changes:
`apps/hubble/src/hubble.ts`,
`apps/hubble/src/network/p2p/gossipNodeWorker.ts`, `yarn.lock`

> ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your
question}`

<!-- end pr-codex -->
  • Loading branch information
CassOnMars authored Jul 30, 2024
1 parent 939dde8 commit 6195946
Show file tree
Hide file tree
Showing 18 changed files with 7,908 additions and 1,481 deletions.
6 changes: 6 additions & 0 deletions .changeset/tame-seas-raise.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
"@farcaster/core": patch
"@farcaster/hubble": patch
---

upgrade libp2p/gossipsub and dependencies
30 changes: 14 additions & 16 deletions apps/hubble/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
"test:ci": "yarn build:all && yarn test:rust --release && ENVIRONMENT=test NODE_OPTIONS=\"--experimental-vm-modules --max-old-space-size=4096\" jest --ci --forceExit --coverage -w 2"
},
"devDependencies": {
"@libp2p/interface-mocks": "^12.0.0",
"@libp2p/interface-compliance-tests": "5.4.10",
"@types/async-lock": "^1.4.0",
"@types/chance": "^1.1.3",
"@types/cli-progress": "^3.11.0",
Expand Down Expand Up @@ -71,23 +71,24 @@
"@aws-sdk/client-s3": "^3.400.0",
"@aws-sdk/client-sts": "^3.398.0",
"@aws-sdk/lib-storage": "^3.504.0",
"@chainsafe/libp2p-gossipsub": "8.0.1",
"@chainsafe/libp2p-noise": "^12.0.0 ",
"@chainsafe/libp2p-gossipsub": "13.1.0",
"@chainsafe/libp2p-noise": "15.1.0",
"@datastructures-js/priority-queue": "^6.3.1",
"@faker-js/faker": "~7.6.0",
"@farcaster/hub-nodejs": "^0.11.21",
"@fastify/cors": "^8.4.0",
"@figma/hot-shots": "^9.0.0-figma.1",
"@grpc/grpc-js": "~1.11.1",
"@libp2p/interface-connection": "^5.0.0",
"@libp2p/interface-connection-gater": "^3.0.0",
"@libp2p/interface-peer-id": "^2.0.1",
"@libp2p/interface-peer-store": "^2.0.0",
"@libp2p/interface-registrar": "2.0.11",
"@libp2p/mplex": "^8.0.0",
"@libp2p/peer-id-factory": "^2.0.0",
"@libp2p/tcp": "^7.0.1",
"@libp2p/utils": "^3.0.2",
"@libp2p/interface": "1.6.2",
"@libp2p/interface-compliance-tests": "5.4.10",
"@libp2p/autonat": "^1.1.3",
"@libp2p/identify": "^2.1.3",
"@libp2p/ping": "^1.1.3",
"@libp2p/fetch": "^1.1.3",
"@libp2p/mplex": "10.1.3",
"@libp2p/peer-id-factory": "4.2.2",
"@libp2p/tcp": "9.1.3",
"@libp2p/utils": "5.4.7",
"@multiformats/multiaddr": "^12.0.0",
"@noble/curves": "^1.0.0",
"abitype": "^0.8.3",
Expand All @@ -97,7 +98,7 @@
"cli-progress": "^3.12.0",
"commander": "~10.0.0",
"fastify": "^4.22.0",
"libp2p": "0.45.0",
"libp2p": "1.8.2",
"neverthrow": "~6.0.0",
"node-cron": "~3.0.2",
"patch-package": "^8.0.0",
Expand All @@ -110,8 +111,5 @@
"tar": "^6.2.1",
"tiny-typed-emitter": "~2.1.0",
"viem": "^2.17.4"
},
"resolutions": {
"@libp2p/interface-registrar": "2.0.11"
}
}
4 changes: 2 additions & 2 deletions apps/hubble/src/cli.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { FarcasterNetwork, farcasterNetworkFromJSON } from "@farcaster/hub-nodejs";
import { peerIdFromString } from "@libp2p/peer-id";
import { PeerId } from "@libp2p/interface-peer-id";
import { Ed25519PeerId, PeerId, RSAPeerId, Secp256k1PeerId } from "@libp2p/interface";
import { createEd25519PeerId, createFromProtobuf, exportToProtobuf } from "@libp2p/peer-id-factory";
import { AddrInfo } from "@chainsafe/libp2p-gossipsub/types";
import { Command } from "commander";
Expand Down Expand Up @@ -748,7 +748,7 @@ app.addCommand(s3SnapshotURL);
/** Write a given PeerId to a file */
const writePeerId = async (peerId: PeerId, filepath: string) => {
const directory = dirname(filepath);
const proto = exportToProtobuf(peerId);
const proto = exportToProtobuf(peerId as RSAPeerId | Ed25519PeerId | Secp256k1PeerId);
// Handling: using try-catch is more ergonomic than capturing and handling throwable, since we
// want a fast failure back to the CLI
try {
Expand Down
13 changes: 8 additions & 5 deletions apps/hubble/src/hubble.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import {
validations,
} from "@farcaster/hub-nodejs";
import { ClientOptions as StatsDClientOptions } from "@figma/hot-shots";
import { PeerId } from "@libp2p/interface-peer-id";
import { Ed25519PeerId, PeerId, RSAPeerId, Secp256k1PeerId } from "@libp2p/interface";
import { peerIdFromBytes, peerIdFromString } from "@libp2p/peer-id";
import { publicAddressesFirst } from "@libp2p/utils/address-sort";
import { unmarshalPrivateKey, unmarshalPublicKey } from "@libp2p/crypto/keys";
Expand Down Expand Up @@ -96,6 +96,7 @@ import { MerkleTrie } from "./network/sync/merkleTrie.js";
import { DEFAULT_CATCHUP_SYNC_SNAPSHOT_MESSAGE_LIMIT } from "./defaultConfig.js";
import { diagnosticReporter } from "./utils/diagnosticReport.js";
import { startupCheck, StartupCheckStatus } from "./utils/startupCheck.js";
import { AddressInfo } from "node:net";

export type HubSubmitSource = "gossip" | "rpc" | "eth-provider" | "l2-provider" | "sync" | "fname-registry";

Expand Down Expand Up @@ -772,7 +773,9 @@ export class Hub implements HubInterface {

const bootstrapAddrs = this.options.bootstrapAddrs ?? [];

const peerId = this.options.peerId ? exportToProtobuf(this.options.peerId) : undefined;
const peerId = this.options.peerId
? exportToProtobuf(this.options.peerId as RSAPeerId | Ed25519PeerId | Secp256k1PeerId)
: undefined;

// Start the Gossip node
await this.gossipNode.start(bootstrapAddrs, {
Expand Down Expand Up @@ -1115,7 +1118,7 @@ export class Hub implements HubInterface {
const family = nodeMultiAddr?.nodeAddress().family;
const announceIp = this.options.announceIp ?? nodeMultiAddr?.nodeAddress().address;
const gossipPort = nodeMultiAddr?.nodeAddress().port;
const rpcPort = this.rpcServer.address?.map((addr) => addr.port).unwrapOr(0);
const rpcPort = this.rpcServer.address?.map((addr: AddressInfo) => addr.port).unwrapOr(0);

const gossipAddressContactInfo = GossipAddressInfo.create({
address: announceIp,
Expand Down Expand Up @@ -1570,7 +1573,7 @@ export class Hub implements HubInterface {
try {
const sslClientResult = getSSLHubRpcClient(address, options);

sslClientResult.$.waitForReady(Date.now() + 2000, (err) => {
sslClientResult.$.waitForReady(Date.now() + 2000, (err: Error | undefined) => {
if (!err) {
resolve(sslClientResult);
} else {
Expand Down Expand Up @@ -1697,7 +1700,7 @@ export class Hub implements HubInterface {
const peerAddresses = peerInfo.multiaddrs;

// sorts addresses by Public IPs first
const addr = peerAddresses.sort((a, b) =>
const addr = peerAddresses.sort((a: Multiaddr, b: Multiaddr) =>
publicAddressesFirst({ multiaddr: a, isCertified: false }, { multiaddr: b, isCertified: false }),
)[0];
if (addr === undefined) {
Expand Down
4 changes: 2 additions & 2 deletions apps/hubble/src/network/p2p/connectionFilter.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { mockMultiaddrConnPair } from "@libp2p/interface-mocks";
import { PeerId } from "@libp2p/interface-peer-id";
import { mockMultiaddrConnPair } from "@libp2p/interface-compliance-tests/mocks";
import { PeerId } from "@libp2p/interface";
import { createEd25519PeerId } from "@libp2p/peer-id-factory";
import { multiaddr } from "@multiformats/multiaddr";
import { ConnectionFilter } from "./connectionFilter.js";
Expand Down
4 changes: 1 addition & 3 deletions apps/hubble/src/network/p2p/connectionFilter.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import { MultiaddrConnection } from "@libp2p/interface-connection";
import { PeerId } from "@libp2p/interface-peer-id";
import { ConnectionGater, MultiaddrConnection, PeerId } from "@libp2p/interface";
import { Multiaddr } from "@multiformats/multiaddr";
import { logger } from "../../utils/logger.js";
import { ConnectionGater } from "@libp2p/interface-connection-gater";

const log = logger.child({
component: "ConnectionFilter",
Expand Down
2 changes: 1 addition & 1 deletion apps/hubble/src/network/p2p/gossipNode.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import Server from "../../rpc/server.js";
import { jestRocksDB } from "../../storage/db/jestUtils.js";
import { MockHub } from "../../test/mocks.js";
import SyncEngine from "../sync/syncEngine.js";
import { PeerId } from "@libp2p/interface-peer-id";
import { PeerId } from "@libp2p/interface";
import { sleepWhile } from "../../utils/crypto.js";
import { createEd25519PeerId } from "@libp2p/peer-id-factory";
import { LibP2PNode } from "./gossipNodeWorker.js";
Expand Down
38 changes: 27 additions & 11 deletions apps/hubble/src/network/p2p/gossipNode.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
import { Message as GossipSubMessage, PublishResult } from "@libp2p/interface-pubsub";
import { Worker } from "worker_threads";
import { PeerInfo } from "@libp2p/interface-peer-info";
import {
ContactInfoContent,
FarcasterNetwork,
Expand All @@ -13,8 +11,16 @@ import {
Message,
MessageBundle,
} from "@farcaster/hub-nodejs";
import { Connection } from "@libp2p/interface-connection";
import { PeerId } from "@libp2p/interface-peer-id";
import {
Connection,
PeerId,
PeerInfo,
Message as GossipSubMessage,
PublishResult,
RSAPeerId,
Ed25519PeerId,
Secp256k1PeerId,
} from "@libp2p/interface";
import { peerIdFromBytes, peerIdFromString } from "@libp2p/peer-id";
import { multiaddr, Multiaddr } from "@multiformats/multiaddr";
import { err, ok, Result } from "neverthrow";
Expand Down Expand Up @@ -322,7 +328,11 @@ export class GossipNode extends TypedEmitter<NodeEvents> {
}

async addPeerToAddressBook(peerId: PeerId, multiaddr: Multiaddr) {
await this.callMethod("addToAddressBook", exportToProtobuf(peerId), multiaddr.bytes);
await this.callMethod(
"addToAddressBook",
exportToProtobuf(peerId as RSAPeerId | Ed25519PeerId | Secp256k1PeerId),
multiaddr.bytes,
);
}

async peerStoreCount(): Promise<number> {
Expand All @@ -331,18 +341,24 @@ export class GossipNode extends TypedEmitter<NodeEvents> {

/** Removes the peer from the address book and hangs up on them */
async removePeerFromAddressBook(peerId: PeerId) {
await this.callMethod("removeFromAddressBook", exportToProtobuf(peerId));
await this.callMethod(
"removeFromAddressBook",
exportToProtobuf(peerId as RSAPeerId | Ed25519PeerId | Secp256k1PeerId),
);
}

/** Returns the libp2p Peer instance after updating the connections in the AddressBook */
async getPeerAddresses(peerId: PeerId): Promise<Multiaddr[]> {
return (await this.callMethod("getPeerAddresses", exportToProtobuf(peerId))).map((addr: Uint8Array) =>
multiaddr(addr),
);
return (
await this.callMethod("getPeerAddresses", exportToProtobuf(peerId as RSAPeerId | Ed25519PeerId | Secp256k1PeerId))
).map((addr: Uint8Array) => multiaddr(addr));
}

async isPeerAllowed(peerId: PeerId) {
return await this.callMethod("isPeerAllowed", exportToProtobuf(peerId));
return await this.callMethod(
"isPeerAllowed",
exportToProtobuf(peerId as RSAPeerId | Ed25519PeerId | Secp256k1PeerId),
);
}

/**
Expand Down Expand Up @@ -532,7 +548,7 @@ export class GossipNode extends TypedEmitter<NodeEvents> {
{
peer: detail.remotePeer,
addrs: detail.remoteAddr,
type: detail.stat.direction,
type: detail.direction,
},
"P2P Connection established",
);
Expand Down
2 changes: 1 addition & 1 deletion apps/hubble/src/network/p2p/gossipNodeDb.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { GossipNode } from "./gossipNode.js";
import { jestRocksDB } from "../../storage/db/jestUtils.js";
import { PeerId } from "@libp2p/interface-peer-id";
import { PeerId } from "@libp2p/interface";
import { sleepWhile } from "../../utils/crypto.js";
import { ResultAsync } from "neverthrow";

Expand Down
Loading

0 comments on commit 6195946

Please sign in to comment.