From 29cb6a8b1075e3fba78a4c3d5a2bc15f421db1b9 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 9 Aug 2023 19:19:15 +0530 Subject: [PATCH 01/58] set peer-exchange with default bootstrap --- package-lock.json | 2 ++ packages/sdk/package.json | 9 +++++---- packages/sdk/src/create.ts | 17 +++++++++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 80a74f305a..b699b68e18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28004,6 +28004,7 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", + "@waku/peer-exchange": "^0.0.15", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "libp2p": "^0.45.9" @@ -32500,6 +32501,7 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", + "@waku/peer-exchange": "*", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "cspell": "^6.31.1", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 3731f8749a..3eb5c93b0f 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -51,14 +51,16 @@ "@chainsafe/libp2p-noise": "^12.0.1", "@libp2p/mplex": "^8.0.4", "@libp2p/websockets": "^6.0.3", - "@waku/utils": "0.0.10", - "@waku/relay": "0.0.5", "@waku/core": "0.0.22", - "@waku/interfaces": "0.0.17", "@waku/dns-discovery": "0.0.16", + "@waku/interfaces": "0.0.17", + "@waku/peer-exchange": "^0.0.15", + "@waku/relay": "0.0.5", + "@waku/utils": "0.0.10", "libp2p": "^0.45.9" }, "devDependencies": { + "@chainsafe/libp2p-gossipsub": "^9.1.0", "@libp2p/interface-address-manager": "^3.0.1", "@libp2p/interface-connection": "^5.1.1", "@libp2p/interface-connection-manager": "^3.0.1", @@ -77,7 +79,6 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.0.2", "@waku/build-utils": "*", - "@chainsafe/libp2p-gossipsub": "^9.1.0", "cspell": "^6.31.1", "interface-datastore": "^7.0.4", "npm-run-all": "^4.1.5", diff --git a/packages/sdk/src/create.ts b/packages/sdk/src/create.ts index b9eb138bda..93bcde1098 100644 --- a/packages/sdk/src/create.ts +++ b/packages/sdk/src/create.ts @@ -21,6 +21,7 @@ import type { ProtocolCreateOptions, RelayNode, } from "@waku/interfaces"; +import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange"; import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay"; import { createLibp2p, Libp2pOptions } from "libp2p"; import { identifyService } from "libp2p/identify"; @@ -45,7 +46,7 @@ export async function createLightNode( const libp2pOptions = options?.libp2p ?? {}; const peerDiscovery = libp2pOptions.peerDiscovery ?? []; if (options?.defaultBootstrap) { - peerDiscovery.push(defaultPeerDiscovery()); + peerDiscovery.push(...defaultPeerDiscoveries()); Object.assign(libp2pOptions, { peerDiscovery }); } @@ -78,7 +79,7 @@ export async function createRelayNode( const libp2pOptions = options?.libp2p ?? {}; const peerDiscovery = libp2pOptions.peerDiscovery ?? []; if (options?.defaultBootstrap) { - peerDiscovery.push(defaultPeerDiscovery()); + peerDiscovery.push(...defaultPeerDiscoveries()); Object.assign(libp2pOptions, { peerDiscovery }); } @@ -119,7 +120,7 @@ export async function createFullNode( const libp2pOptions = options?.libp2p ?? {}; const peerDiscovery = libp2pOptions.peerDiscovery ?? []; if (options?.defaultBootstrap) { - peerDiscovery.push(defaultPeerDiscovery()); + peerDiscovery.push(...defaultPeerDiscoveries()); Object.assign(libp2pOptions, { peerDiscovery }); } @@ -144,10 +145,14 @@ export async function createFullNode( ) as FullNode; } -export function defaultPeerDiscovery(): ( +export function defaultPeerDiscoveries(): (( components: Libp2pComponents -) => PeerDiscovery { - return wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS); +) => PeerDiscovery)[] { + const discoveries = [ + wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS), + wakuPeerExchangeDiscovery(), + ]; + return discoveries; } type PubsubService = { From 3646fdc5c64374ce867992e6c97d4153b14ec57e Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 11 Aug 2023 14:12:29 +0530 Subject: [PATCH 02/58] only initialise protocols with bootstrap peers --- packages/utils/package.json | 4 ++-- packages/utils/src/libp2p/index.ts | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/utils/package.json b/packages/utils/package.json index 8fe2ea23a0..4f79db073e 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -66,7 +66,8 @@ }, "dependencies": { "debug": "^4.3.4", - "uint8arrays": "^4.0.4" + "uint8arrays": "^4.0.4", + "@waku/interfaces": "^0.0.17" }, "devDependencies": { "@libp2p/interface-connection": "^5.1.1", @@ -76,7 +77,6 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.0.2", "@waku/build-utils": "*", - "@waku/interfaces": "0.0.17", "cspell": "^6.31.1", "npm-run-all": "^4.1.5", "prettier": "^2.8.8", diff --git a/packages/utils/src/libp2p/index.ts b/packages/utils/src/libp2p/index.ts index a0ffefc974..b19fd97a78 100644 --- a/packages/utils/src/libp2p/index.ts +++ b/packages/utils/src/libp2p/index.ts @@ -1,6 +1,7 @@ import type { Connection } from "@libp2p/interface-connection"; import type { PeerId } from "@libp2p/interface-peer-id"; import type { Peer, PeerStore } from "@libp2p/interface-peer-store"; +import { Tags } from "@waku/interfaces"; import debug from "debug"; const log = debug("waku:libp2p-utils"); @@ -26,7 +27,12 @@ export async function getPeersForProtocol( const peers: Peer[] = []; await peerStore.forEach((peer) => { for (let i = 0; i < protocols.length; i++) { - if (peer.protocols.includes(protocols[i])) { + // only add peer if it supports the protocol and is a bootstrap peer + // ref: https://github.com/waku-org/js-waku/issues/1429#issuecomment-1667764036 + if ( + peer.protocols.includes(protocols[i]) && + peer.tags.has(Tags.BOOTSTRAP) + ) { peers.push(peer); break; } From 573fb8d5dc27bf94e92affd3f51bb76ca29424ae Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 11 Aug 2023 14:26:10 +0530 Subject: [PATCH 03/58] update package --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b699b68e18..0a63cc6e0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28330,6 +28330,7 @@ "version": "0.0.10", "license": "MIT OR Apache-2.0", "dependencies": { + "@waku/interfaces": "^0.0.17", "debug": "^4.3.4", "uint8arrays": "^4.0.4" }, @@ -28341,7 +28342,6 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.0.2", "@waku/build-utils": "*", - "@waku/interfaces": "0.0.17", "cspell": "^6.31.1", "npm-run-all": "^4.1.5", "prettier": "^2.8.8", @@ -32501,7 +32501,7 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", - "@waku/peer-exchange": "*", + "@waku/peer-exchange": "^0.0.15", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "cspell": "^6.31.1", @@ -32711,7 +32711,7 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.0.2", "@waku/build-utils": "*", - "@waku/interfaces": "0.0.17", + "@waku/interfaces": "^0.0.17", "cspell": "^6.31.1", "debug": "^4.3.4", "npm-run-all": "^4.1.5", From 6c1ec84306700f215cf51846cdf703c0eb195cf9 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Mon, 14 Aug 2023 02:38:51 +0530 Subject: [PATCH 04/58] update package-lock --- package-lock.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4eced6fe0b..2a639880c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30640,7 +30640,8 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.0.2", "@waku/build-utils": "*", - "cspell": "^6.31.1", + "@waku/interfaces": "0.0.17", + "cspell": "^7.0.0", "npm-run-all": "^4.1.5", "rollup": "^3.21.3", "typescript": "^5.0.4" @@ -35569,8 +35570,8 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.0.2", "@waku/build-utils": "*", - "@waku/interfaces": "^0.0.17", - "cspell": "^6.31.1", + "@waku/interfaces": "0.0.17", + "cspell": "^7.0.0", "debug": "^4.3.4", "npm-run-all": "^4.1.5", "rollup": "^3.21.3", From a0f066821d7dbf3ece833cecfc93961e5703c628 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 15 Aug 2023 10:55:41 +0530 Subject: [PATCH 05/58] refactor `getPeers` while setting up a protocol --- packages/core/src/lib/base_protocol.ts | 43 +++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 420d7058b7..907889589f 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -2,7 +2,7 @@ import type { Stream } from "@libp2p/interface-connection"; import type { Libp2p } from "@libp2p/interface-libp2p"; import type { PeerId } from "@libp2p/interface-peer-id"; import { Peer, PeerStore } from "@libp2p/interface-peer-store"; -import type { IBaseProtocol, Libp2pComponents } from "@waku/interfaces"; +import { IBaseProtocol, Libp2pComponents, Tags } from "@waku/interfaces"; import { getPeersForProtocol, selectConnection, @@ -20,6 +20,7 @@ export class BaseProtocol implements IBaseProtocol { constructor( public multicodec: string, private components: Libp2pComponents, + private log: debug.Debugger, ) { this.addLibp2pEventListener = components.events.addEventListener.bind( components.events, @@ -50,6 +51,46 @@ export class BaseProtocol implements IBaseProtocol { ); return peer; } + + protected async getPeers(peerId?: PeerId): Promise { + const selectedPeers: Peer[] = []; + + if (peerId) { + const { peer } = await selectPeerForProtocol( + this.peerStore, + [this.multicodec], + peerId, + ); + + selectedPeers.push(peer); + } + + const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ + this.multicodec, + ]); + + // Counter to keep track of the number of bootstrap peers + let bootstrapPeerCount = 0; + + allPeersForProtocol.map((peer) => { + // If the peer is a bootstrap peer and we don't have one yet, add it + if (peer.tags.has(Tags.BOOTSTRAP) && bootstrapPeerCount < 1) { + selectedPeers.push(peer); + bootstrapPeerCount++; + } + // If we have less than 3 total peers, add non-bootstrap peers + else if (selectedPeers.length < 3 && !peer.tags.has(Tags.BOOTSTRAP)) { + selectedPeers.push(peer); + } + }); + + if (bootstrapPeerCount === 0) { + this.log(`warning: no bootstrap peers found, using random peers`); + } + + return selectedPeers; + } + protected async newStream(peer: Peer): Promise { const connections = this.components.connectionManager.getConnections( peer.id, From f376334f81f18560aeb3fd41a0f4daa0f6700b65 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 15 Aug 2023 11:02:12 +0530 Subject: [PATCH 06/58] move codecs to `@waku/interfaces` --- packages/core/src/lib/filter/index.ts | 14 +++++--------- packages/core/src/lib/store/index.ts | 5 ++--- packages/interfaces/src/protocols.ts | 7 +++++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index e57c43b64a..79d4bec5d0 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -2,8 +2,9 @@ import { Stream } from "@libp2p/interface-connection"; import type { PeerId } from "@libp2p/interface-peer-id"; import type { Peer } from "@libp2p/interface-peer-store"; import type { IncomingStreamData } from "@libp2p/interface-registrar"; -import type { +import { Callback, + Codecs, ContentTopic, IAsyncIterator, IDecodedMessage, @@ -41,11 +42,6 @@ type SubscriptionCallback = { callback: Callback; }; -const FilterCodecs = { - SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1", - PUSH: "/vac/waku/filter-push/2.0.0-beta1", -}; - class Subscription { private readonly peer: Peer; private readonly pubSubTopic: PubSubTopic; @@ -246,10 +242,10 @@ class Filter extends BaseProtocol implements IReceiver { } constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(FilterCodecs.SUBSCRIBE, libp2p.components); + super(Codecs.FilterSubscribe, libp2p.components, log); - libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => { - log("Failed to register ", FilterCodecs.PUSH, e); + libp2p.handle(Codecs.FilterPush, this.onRequest.bind(this)).catch((e) => { + log("Failed to register ", Codecs.FilterPush, e); }); this.activeSubscriptions = new Map(); diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 0f6aca8db8..eeab33a9bf 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -2,6 +2,7 @@ import type { Stream } from "@libp2p/interface-connection"; import type { PeerId } from "@libp2p/interface-peer-id"; import { sha256 } from "@noble/hashes/sha256"; import { + Codecs, Cursor, IDecodedMessage, IDecoder, @@ -28,8 +29,6 @@ import HistoryError = proto.HistoryResponse.HistoryError; const log = debug("waku:store"); -export const StoreCodec = "/vac/waku/store/2.0.0-beta4"; - export const DefaultPageSize = 10; export { PageDirection }; @@ -82,7 +81,7 @@ class Store extends BaseProtocol implements IStore { options: ProtocolCreateOptions; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(StoreCodec, libp2p.components); + super(Codecs.Store, libp2p.components, log); this.options = options ?? {}; } diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts index 3affc622f7..d82c2112f0 100644 --- a/packages/interfaces/src/protocols.ts +++ b/packages/interfaces/src/protocols.ts @@ -12,6 +12,13 @@ export enum Protocols { Filter = "filter", } +export enum Codecs { + LightPush = "/vac/waku/lightpush/2.0.0-beta1", + Store = "/vac/waku/store/2.0.0-beta4", + FilterSubscribe = "/vac/waku/filter-subscribe/2.0.0-beta1", + FilterPush = "/vac/waku/filter-push/2.0.0-beta1", +} + export interface IBaseProtocol { multicodec: string; peerStore: PeerStore; From f7f383b678cb3ee4155684367befd6a584d683c5 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 15 Aug 2023 11:03:25 +0530 Subject: [PATCH 07/58] lightpush: send messages to multiple peers --- packages/core/src/lib/light_push/index.ts | 118 +++++++++++++--------- packages/utils/src/libp2p/index.ts | 8 +- 2 files changed, 70 insertions(+), 56 deletions(-) diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index a2f91512ae..bcf75879a4 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -1,5 +1,6 @@ import type { PeerId } from "@libp2p/interface-peer-id"; import { + Codecs, IEncoder, ILightPush, IMessage, @@ -24,7 +25,6 @@ import { PushRpc } from "./push_rpc.js"; const log = debug("waku:light-push"); -export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1"; export { PushResponse }; /** @@ -34,7 +34,7 @@ class LightPush extends BaseProtocol implements ILightPush { options: ProtocolCreateOptions; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(LightPushCodec, libp2p.components); + super(Codecs.LightPush, libp2p.components, log); this.options = options || {}; } @@ -45,62 +45,82 @@ class LightPush extends BaseProtocol implements ILightPush { ): Promise { const { pubSubTopic = DefaultPubSubTopic } = this.options; - const peer = await this.getPeer(opts?.peerId); - const stream = await this.newStream(peer); + if (!isSizeValid(message.payload)) { + log("Failed to send waku light push: message is bigger than 1MB"); + return { + recipients: [], + error: SendError.SIZE_TOO_BIG, + }; + } - const recipients: PeerId[] = []; - let error: undefined | SendError = undefined; + const protoMessage = await encoder.toProtoObj(message); + if (!protoMessage) { + log("Failed to encode to protoMessage, aborting push"); + return { + recipients: [], + error: SendError.ENCODE_FAILED, + }; + } - try { - if (!isSizeValid(message.payload)) { - log("Failed to send waku light push: message is bigger that 1MB"); - return { - recipients, - error: SendError.SIZE_TOO_BIG, - }; - } + const query = PushRpc.createRequest(protoMessage, pubSubTopic); + const peers = await this.getPeers(opts?.peerId); - const protoMessage = await encoder.toProtoObj(message); - if (!protoMessage) { - log("Failed to encode to protoMessage, aborting push"); - return { - recipients, - error: SendError.ENCODE_FAILED, - }; - } - const query = PushRpc.createRequest(protoMessage, pubSubTopic); - const res = await pipe( - [query.encode()], - lp.encode, - stream, - lp.decode, - async (source) => await all(source), - ); + const promises = peers.map(async (peer) => { + const recipients: PeerId[] = []; + let error: SendError | undefined; + + const stream = await this.newStream(peer); try { - const bytes = new Uint8ArrayList(); - res.forEach((chunk) => { - bytes.append(chunk); - }); - - const response = PushRpc.decode(bytes).response; - - if (response?.isSuccess) { - recipients.push(peer.id); - } else { - log("No response in PushRPC"); - error = SendError.NO_RPC_RESPONSE; + const res = await pipe( + [query.encode()], + lp.encode, + stream, + lp.decode, + async (source) => await all(source), + ); + try { + const bytes = new Uint8ArrayList(); + res.forEach((chunk) => { + bytes.append(chunk); + }); + + const response = PushRpc.decode(bytes).response; + + if (response?.isSuccess) { + recipients.push(peer.id); + } else { + log("No response in PushRPC"); + error = SendError.NO_RPC_RESPONSE; + } + } catch (err) { + log("Failed to decode push reply", err); + error = SendError.DECODE_FAILED; } } catch (err) { - log("Failed to decode push reply", err); - error = SendError.DECODE_FAILED; + log("Failed to send waku light push request", err); + error = SendError.GENERIC_FAIL; } - } catch (err) { - log("Failed to send waku light push request", err); - error = SendError.GENERIC_FAIL; - } + + return { recipients, error }; + }); + + const results = await Promise.allSettled(promises); + const successfulResults = results.filter( + (result) => result.status === "fulfilled", + ) as PromiseFulfilledResult<{ + recipients: PeerId[]; + error: SendError | undefined; + }>[]; + const recipients = successfulResults.flatMap( + (result) => result.value.recipients, + ); + const errors = successfulResults + .map((result) => result.value.error) + .filter((error) => error !== undefined); + return { - error, recipients, + error: errors.length > 0 ? errors : undefined, }; } } diff --git a/packages/utils/src/libp2p/index.ts b/packages/utils/src/libp2p/index.ts index 773a0fd95d..f684469509 100644 --- a/packages/utils/src/libp2p/index.ts +++ b/packages/utils/src/libp2p/index.ts @@ -1,7 +1,6 @@ import type { Connection } from "@libp2p/interface-connection"; import type { PeerId } from "@libp2p/interface-peer-id"; import type { Peer, PeerStore } from "@libp2p/interface-peer-store"; -import { Tags } from "@waku/interfaces"; import debug from "debug"; const log = debug("waku:libp2p-utils"); @@ -27,12 +26,7 @@ export async function getPeersForProtocol( const peers: Peer[] = []; await peerStore.forEach((peer) => { for (let i = 0; i < protocols.length; i++) { - // only add peer if it supports the protocol and is a bootstrap peer - // ref: https://github.com/waku-org/js-waku/issues/1429#issuecomment-1667764036 - if ( - peer.protocols.includes(protocols[i]) && - peer.tags.has(Tags.BOOTSTRAP) - ) { + if (peer.protocols.includes(protocols[i])) { peers.push(peer); break; } From 6807888ac680c2d849716abe0c2a7cf3e0a762f1 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 15 Aug 2023 11:17:42 +0530 Subject: [PATCH 08/58] only use multiple peers for LP and Filter --- packages/core/src/lib/base_protocol.ts | 33 +++++++++++++++++--------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 907889589f..07311948a0 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -2,7 +2,12 @@ import type { Stream } from "@libp2p/interface-connection"; import type { Libp2p } from "@libp2p/interface-libp2p"; import type { PeerId } from "@libp2p/interface-peer-id"; import { Peer, PeerStore } from "@libp2p/interface-peer-store"; -import { IBaseProtocol, Libp2pComponents, Tags } from "@waku/interfaces"; +import { + Codecs, + IBaseProtocol, + Libp2pComponents, + Tags, +} from "@waku/interfaces"; import { getPeersForProtocol, selectConnection, @@ -18,7 +23,7 @@ export class BaseProtocol implements IBaseProtocol { public readonly removeLibp2pEventListener: Libp2p["removeEventListener"]; constructor( - public multicodec: string, + public multicodec: Codecs, private components: Libp2pComponents, private log: debug.Debugger, ) { @@ -52,18 +57,24 @@ export class BaseProtocol implements IBaseProtocol { return peer; } - protected async getPeers(peerId?: PeerId): Promise { + protected async getPeers(peerId?: PeerId): Promise { const selectedPeers: Peer[] = []; - if (peerId) { - const { peer } = await selectPeerForProtocol( - this.peerStore, - [this.multicodec], - peerId, - ); + const { peer } = await selectPeerForProtocol( + this.peerStore, + [this.multicodec], + peerId, + ); + + selectedPeers.push(peer); - selectedPeers.push(peer); - } + // only return multiple peers for protocols LP and Filter + if ( + ![Codecs.LightPush, Codecs.FilterPush, Codecs.FilterSubscribe].includes( + this.multicodec, + ) + ) + return selectedPeers; const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ this.multicodec, From f956f7f46b5c0b901593a9653a80190caea79dad Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 15 Aug 2023 11:18:07 +0530 Subject: [PATCH 09/58] fix: ts warnings --- packages/core/src/index.ts | 9 ++------- packages/core/src/lib/light_push/index.ts | 8 ++++---- packages/interfaces/src/protocols.ts | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c00ff152de..00d9cb84a5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -15,15 +15,10 @@ export * as waku_filter from "./lib/filter/index.js"; export { wakuFilter } from "./lib/filter/index.js"; export * as waku_light_push from "./lib/light_push/index.js"; -export { wakuLightPush, LightPushCodec } from "./lib/light_push/index.js"; +export { wakuLightPush } from "./lib/light_push/index.js"; export * as waku_store from "./lib/store/index.js"; -export { - PageDirection, - wakuStore, - StoreCodec, - createCursor, -} from "./lib/store/index.js"; +export { PageDirection, wakuStore, createCursor } from "./lib/store/index.js"; export { waitForRemotePeer } from "./lib/wait_for_remote_peer.js"; diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index bcf75879a4..377f101c9e 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -49,7 +49,7 @@ class LightPush extends BaseProtocol implements ILightPush { log("Failed to send waku light push: message is bigger than 1MB"); return { recipients: [], - error: SendError.SIZE_TOO_BIG, + errors: [SendError.SIZE_TOO_BIG], }; } @@ -58,7 +58,7 @@ class LightPush extends BaseProtocol implements ILightPush { log("Failed to encode to protoMessage, aborting push"); return { recipients: [], - error: SendError.ENCODE_FAILED, + errors: [SendError.ENCODE_FAILED], }; } @@ -116,11 +116,11 @@ class LightPush extends BaseProtocol implements ILightPush { ); const errors = successfulResults .map((result) => result.value.error) - .filter((error) => error !== undefined); + .filter((error) => error !== undefined) as SendError[]; return { recipients, - error: errors.length > 0 ? errors : undefined, + errors, }; } } diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts index d82c2112f0..126e22a5c4 100644 --- a/packages/interfaces/src/protocols.ts +++ b/packages/interfaces/src/protocols.ts @@ -81,6 +81,6 @@ export enum SendError { } export interface SendResult { - error?: SendError; + errors?: SendError[]; recipients: PeerId[]; } From b43fe35bcd0f40138dd9a78234087ec3d6c915c2 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 15 Aug 2023 11:50:30 +0530 Subject: [PATCH 10/58] lightpush: tests pass --- packages/core/src/lib/light_push/index.ts | 14 +++++++------- packages/tests/tests/light_push.node.spec.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index 377f101c9e..f75dc23ae7 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -45,10 +45,12 @@ class LightPush extends BaseProtocol implements ILightPush { ): Promise { const { pubSubTopic = DefaultPubSubTopic } = this.options; + const recipients: PeerId[] = []; + if (!isSizeValid(message.payload)) { log("Failed to send waku light push: message is bigger than 1MB"); return { - recipients: [], + recipients, errors: [SendError.SIZE_TOO_BIG], }; } @@ -57,7 +59,7 @@ class LightPush extends BaseProtocol implements ILightPush { if (!protoMessage) { log("Failed to encode to protoMessage, aborting push"); return { - recipients: [], + recipients, errors: [SendError.ENCODE_FAILED], }; } @@ -66,7 +68,6 @@ class LightPush extends BaseProtocol implements ILightPush { const peers = await this.getPeers(opts?.peerId); const promises = peers.map(async (peer) => { - const recipients: PeerId[] = []; let error: SendError | undefined; const stream = await this.newStream(peer); @@ -87,7 +88,8 @@ class LightPush extends BaseProtocol implements ILightPush { const response = PushRpc.decode(bytes).response; if (response?.isSuccess) { - recipients.push(peer.id); + recipients.some((recipient) => recipient.equals(peer.id)) || + recipients.push(peer.id); } else { log("No response in PushRPC"); error = SendError.NO_RPC_RESPONSE; @@ -111,9 +113,7 @@ class LightPush extends BaseProtocol implements ILightPush { recipients: PeerId[]; error: SendError | undefined; }>[]; - const recipients = successfulResults.flatMap( - (result) => result.value.recipients, - ); + const errors = successfulResults .map((result) => result.value.error) .filter((error) => error !== undefined) as SendError[]; diff --git a/packages/tests/tests/light_push.node.spec.ts b/packages/tests/tests/light_push.node.spec.ts index d4afd26009..b3d1fb3969 100644 --- a/packages/tests/tests/light_push.node.spec.ts +++ b/packages/tests/tests/light_push.node.spec.ts @@ -108,7 +108,7 @@ describe("Waku Light Push [node only]", () => { payload: generateRandomUint8Array(MB + 65536), }); expect(pushResponse.recipients.length).to.eq(0); - expect(pushResponse.error).to.eq(SendError.SIZE_TOO_BIG); + expect(pushResponse.errors).to.include(SendError.SIZE_TOO_BIG); }); }); From d712e8ac111e9e6e9c969d06c57934ba24d1a161 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 15 Aug 2023 11:50:44 +0530 Subject: [PATCH 11/58] update breaking changes for new API --- packages/interfaces/src/protocols.ts | 2 ++ packages/peer-exchange/src/waku_peer_exchange.ts | 8 +++----- packages/relay/src/constants.ts | 5 ----- packages/relay/src/index.ts | 16 +++++++++------- packages/tests/tests/relay.node.spec.ts | 4 ++-- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts index 126e22a5c4..223bf5b467 100644 --- a/packages/interfaces/src/protocols.ts +++ b/packages/interfaces/src/protocols.ts @@ -17,6 +17,8 @@ export enum Codecs { Store = "/vac/waku/store/2.0.0-beta4", FilterSubscribe = "/vac/waku/filter-subscribe/2.0.0-beta1", FilterPush = "/vac/waku/filter-push/2.0.0-beta1", + PeerExchange = "/vac/waku/peer-exchange/2.0.0-alpha1", + Relay = "/vac/waku/relay/2.0.0", } export interface IBaseProtocol { diff --git a/packages/peer-exchange/src/waku_peer_exchange.ts b/packages/peer-exchange/src/waku_peer_exchange.ts index 4e13753044..c2e5b712af 100644 --- a/packages/peer-exchange/src/waku_peer_exchange.ts +++ b/packages/peer-exchange/src/waku_peer_exchange.ts @@ -1,6 +1,7 @@ import { BaseProtocol } from "@waku/core/lib/base_protocol"; import { EnrDecoder } from "@waku/enr"; -import type { +import { + Codecs, IPeerExchange, Libp2pComponents, PeerExchangeQueryParams, @@ -23,14 +24,11 @@ const log = debug("waku:peer-exchange"); * Implementation of the Peer Exchange protocol (https://rfc.vac.dev/spec/34/) */ export class WakuPeerExchange extends BaseProtocol implements IPeerExchange { - multicodec: string; - /** * @param components - libp2p components */ constructor(components: Libp2pComponents) { - super(PeerExchangeCodec, components); - this.multicodec = PeerExchangeCodec; + super(Codecs.PeerExchange, components, log); } /** diff --git a/packages/relay/src/constants.ts b/packages/relay/src/constants.ts index 067084d991..2649117cd3 100644 --- a/packages/relay/src/constants.ts +++ b/packages/relay/src/constants.ts @@ -1,11 +1,6 @@ export const second = 1000; export const minute = 60 * second; -/** - * RelayCodec is the libp2p identifier for the waku relay protocol - */ -export const RelayCodecs = ["/vac/waku/relay/2.0.0"]; - /** * RelayGossipFactor affects how many peers we will emit gossip to at each heartbeat. * We will send gossip to RelayGossipFactor * (total number of non-mesh peers), or diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index 1f20668446..e45e3e602f 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -6,12 +6,14 @@ import { } from "@chainsafe/libp2p-gossipsub"; import type { PeerIdStr, TopicStr } from "@chainsafe/libp2p-gossipsub/types"; import { SignaturePolicy } from "@chainsafe/libp2p-gossipsub/types"; +import type { PeerId } from "@libp2p/interface/peer-id"; import type { PubSub } from "@libp2p/interface-pubsub"; import { sha256 } from "@noble/hashes/sha256"; import { DefaultPubSubTopic } from "@waku/core"; import { ActiveSubscriptions, Callback, + Codecs, IAsyncIterator, IDecodedMessage, IDecoder, @@ -27,7 +29,6 @@ import { import { groupByContentTopic, isSizeValid, toAsyncIterator } from "@waku/utils"; import debug from "debug"; -import { RelayCodecs } from "./constants.js"; import { messageValidator } from "./message_validator.js"; import { TopicOnlyDecoder } from "./topic_only_message.js"; @@ -49,7 +50,7 @@ class Relay implements IRelay { private readonly pubSubTopic: string; private defaultDecoder: IDecoder; - public static multicodec: string = RelayCodecs[0]; + public static multicodec: string = Codecs.Relay; public readonly gossipSub: GossipSub; /** @@ -98,11 +99,12 @@ class Relay implements IRelay { * Send Waku message. */ public async send(encoder: IEncoder, message: IMessage): Promise { + const recipients: PeerId[] = []; if (!isSizeValid(message.payload)) { log("Failed to send waku relay: message is bigger that 1MB"); return { - recipients: [], - error: SendError.SIZE_TOO_BIG, + recipients, + errors: [SendError.SIZE_TOO_BIG], }; } @@ -110,8 +112,8 @@ class Relay implements IRelay { if (!msg) { log("Failed to encode message, aborting publish"); return { - recipients: [], - error: SendError.ENCODE_FAILED, + recipients, + errors: [SendError.ENCODE_FAILED], }; } @@ -263,7 +265,7 @@ export function wakuGossipSub( fallbackToFloodsub: false, }; const pubsub = new GossipSub(components, init); - pubsub.multicodecs = RelayCodecs; + pubsub.multicodecs = [Codecs.Relay]; return pubsub; }; } diff --git a/packages/tests/tests/relay.node.spec.ts b/packages/tests/tests/relay.node.spec.ts index afeaf78fcd..b8cc94545e 100644 --- a/packages/tests/tests/relay.node.spec.ts +++ b/packages/tests/tests/relay.node.spec.ts @@ -382,13 +382,13 @@ describe("Waku Relay [node only]", () => { payload: generateRandomUint8Array(1 * MB + 65536), }); expect(sendResult.recipients.length).to.eq(0); - expect(sendResult.error).to.eq(SendError.SIZE_TOO_BIG); + expect(sendResult.errors).to.include(SendError.SIZE_TOO_BIG); sendResult = await waku1.relay.send(TestEncoder, { payload: generateRandomUint8Array(2 * MB), }); expect(sendResult.recipients.length).to.eq(0); - expect(sendResult.error).to.eq(SendError.SIZE_TOO_BIG); + expect(sendResult.errors).to.include(SendError.SIZE_TOO_BIG); const waku2ReceivedMsg = await waku2ReceivedMsgPromise; expect(waku2ReceivedMsg?.payload?.length).to.eq(0); From 8bf969a457d1b13af4c2b239441bba1391a1e729 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 16:33:04 +0530 Subject: [PATCH 12/58] move codecs back into protocol files --- packages/core/src/lib/filter/index.ts | 12 ++++++++---- packages/core/src/lib/store/index.ts | 5 +++-- packages/interfaces/src/protocols.ts | 9 --------- packages/peer-exchange/src/waku_peer_exchange.ts | 7 +++---- packages/relay/src/index.ts | 7 ++++--- 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 79d4bec5d0..7aa801a0b8 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -4,7 +4,6 @@ import type { Peer } from "@libp2p/interface-peer-store"; import type { IncomingStreamData } from "@libp2p/interface-registrar"; import { Callback, - Codecs, ContentTopic, IAsyncIterator, IDecodedMessage, @@ -37,6 +36,11 @@ import { const log = debug("waku:filter:v2"); +const FilterCodec = { + SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1", + PUSH: "/vac/waku/filter-push/2.0.0-beta1", +}; + type SubscriptionCallback = { decoders: IDecoder[]; callback: Callback; @@ -242,10 +246,10 @@ class Filter extends BaseProtocol implements IReceiver { } constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(Codecs.FilterSubscribe, libp2p.components, log); + super(FilterCodec.SUBSCRIBE, libp2p.components, log); - libp2p.handle(Codecs.FilterPush, this.onRequest.bind(this)).catch((e) => { - log("Failed to register ", Codecs.FilterPush, e); + libp2p.handle(FilterCodec.PUSH, this.onRequest.bind(this)).catch((e) => { + log("Failed to register ", FilterCodec.PUSH, e); }); this.activeSubscriptions = new Map(); diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index eeab33a9bf..186550cada 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -2,7 +2,6 @@ import type { Stream } from "@libp2p/interface-connection"; import type { PeerId } from "@libp2p/interface-peer-id"; import { sha256 } from "@noble/hashes/sha256"; import { - Codecs, Cursor, IDecodedMessage, IDecoder, @@ -33,6 +32,8 @@ export const DefaultPageSize = 10; export { PageDirection }; +const StoreCodec = "/vac/waku/store/2.0.0-beta4"; + export interface TimeFilter { startTime: Date; endTime: Date; @@ -81,7 +82,7 @@ class Store extends BaseProtocol implements IStore { options: ProtocolCreateOptions; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(Codecs.Store, libp2p.components, log); + super(StoreCodec, libp2p.components, log); this.options = options ?? {}; } diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts index 223bf5b467..84c3a4522c 100644 --- a/packages/interfaces/src/protocols.ts +++ b/packages/interfaces/src/protocols.ts @@ -12,15 +12,6 @@ export enum Protocols { Filter = "filter", } -export enum Codecs { - LightPush = "/vac/waku/lightpush/2.0.0-beta1", - Store = "/vac/waku/store/2.0.0-beta4", - FilterSubscribe = "/vac/waku/filter-subscribe/2.0.0-beta1", - FilterPush = "/vac/waku/filter-push/2.0.0-beta1", - PeerExchange = "/vac/waku/peer-exchange/2.0.0-alpha1", - Relay = "/vac/waku/relay/2.0.0", -} - export interface IBaseProtocol { multicodec: string; peerStore: PeerStore; diff --git a/packages/peer-exchange/src/waku_peer_exchange.ts b/packages/peer-exchange/src/waku_peer_exchange.ts index c2e5b712af..deacb06fc2 100644 --- a/packages/peer-exchange/src/waku_peer_exchange.ts +++ b/packages/peer-exchange/src/waku_peer_exchange.ts @@ -1,7 +1,6 @@ import { BaseProtocol } from "@waku/core/lib/base_protocol"; import { EnrDecoder } from "@waku/enr"; import { - Codecs, IPeerExchange, Libp2pComponents, PeerExchangeQueryParams, @@ -16,7 +15,7 @@ import { Uint8ArrayList } from "uint8arraylist"; import { PeerExchangeRPC } from "./rpc.js"; -export const PeerExchangeCodec = "/vac/waku/peer-exchange/2.0.0-alpha1"; +const PeerExchangeCodec = "/vac/waku/peer-exchange/2.0.0-alpha1"; const log = debug("waku:peer-exchange"); @@ -28,7 +27,7 @@ export class WakuPeerExchange extends BaseProtocol implements IPeerExchange { * @param components - libp2p components */ constructor(components: Libp2pComponents) { - super(Codecs.PeerExchange, components, log); + super(PeerExchangeCodec, components, log); } /** @@ -43,7 +42,7 @@ export class WakuPeerExchange extends BaseProtocol implements IPeerExchange { numPeers: BigInt(numPeers), }); - const peer = await this.getPeer(params.peerId); + const peer = await this.getPeers(params.peerId && [params.peerId]); const stream = await this.newStream(peer); diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index e45e3e602f..aa74a8845c 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -13,7 +13,6 @@ import { DefaultPubSubTopic } from "@waku/core"; import { ActiveSubscriptions, Callback, - Codecs, IAsyncIterator, IDecodedMessage, IDecoder, @@ -42,6 +41,8 @@ export type Observer = { export type RelayCreateOptions = ProtocolCreateOptions & GossipsubOpts; export type ContentTopic = string; +const RelayCodec = "/vac/waku/relay/2.0.0"; + /** * Implements the [Waku v2 Relay protocol](https://rfc.vac.dev/spec/11/). * Throws if libp2p.pubsub does not support Waku Relay @@ -50,7 +51,7 @@ class Relay implements IRelay { private readonly pubSubTopic: string; private defaultDecoder: IDecoder; - public static multicodec: string = Codecs.Relay; + public static multicodec: string = RelayCodec; public readonly gossipSub: GossipSub; /** @@ -265,7 +266,7 @@ export function wakuGossipSub( fallbackToFloodsub: false, }; const pubsub = new GossipSub(components, init); - pubsub.multicodecs = [Codecs.Relay]; + pubsub.multicodecs = [RelayCodec]; return pubsub; }; } From 32042678106a56c7c88f5024c167321b075364bc Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 16:37:56 +0530 Subject: [PATCH 13/58] refactor: `getPeers()` --- packages/core/src/lib/base_protocol.ts | 80 ++++++++++++-------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 07311948a0..14bf54e333 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -2,12 +2,7 @@ import type { Stream } from "@libp2p/interface-connection"; import type { Libp2p } from "@libp2p/interface-libp2p"; import type { PeerId } from "@libp2p/interface-peer-id"; import { Peer, PeerStore } from "@libp2p/interface-peer-store"; -import { - Codecs, - IBaseProtocol, - Libp2pComponents, - Tags, -} from "@waku/interfaces"; +import { IBaseProtocol, Libp2pComponents, Tags } from "@waku/interfaces"; import { getPeersForProtocol, selectConnection, @@ -23,9 +18,8 @@ export class BaseProtocol implements IBaseProtocol { public readonly removeLibp2pEventListener: Libp2p["removeEventListener"]; constructor( - public multicodec: Codecs, + public multicodec: string, private components: Libp2pComponents, - private log: debug.Debugger, ) { this.addLibp2pEventListener = components.events.addEventListener.bind( components.events, @@ -57,49 +51,49 @@ export class BaseProtocol implements IBaseProtocol { return peer; } - protected async getPeers(peerId?: PeerId): Promise { - const selectedPeers: Peer[] = []; + protected async getPeers( + numPeers: number, + includeBootstrap: boolean, + peerIds?: PeerId[], + ): Promise { + // Retrieve all peers that support the protocol + const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ + this.multicodec, + ]); - const { peer } = await selectPeerForProtocol( - this.peerStore, - [this.multicodec], - peerId, + // Filter the bootstrap peer if required to include + const bootstrapPeer = includeBootstrap + ? allPeersForProtocol.find((peer) => peer.tags.has(Tags.BOOTSTRAP)) + : undefined; + + // Filter the peers that match the specified peerIds + const matchingPeers = peerIds + ? allPeersForProtocol.filter((peer) => peerIds.includes(peer.id)) + : []; + + // Filter remaining peers excluding bootstrap and specified peerIds + const remainingPeers = allPeersForProtocol.filter( + (peer) => peer !== bootstrapPeer && !matchingPeers.includes(peer), ); - selectedPeers.push(peer); + // Initialize the list of selected peers + const selectedPeers: Peer[] = []; - // only return multiple peers for protocols LP and Filter - if ( - ![Codecs.LightPush, Codecs.FilterPush, Codecs.FilterSubscribe].includes( - this.multicodec, - ) - ) - return selectedPeers; + // Add the bootstrap peer if available and required + if (bootstrapPeer) { + selectedPeers.push(bootstrapPeer); + } - const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ - this.multicodec, - ]); + // Add the specified peerIds if available + selectedPeers.push(...matchingPeers); - // Counter to keep track of the number of bootstrap peers - let bootstrapPeerCount = 0; - - allPeersForProtocol.map((peer) => { - // If the peer is a bootstrap peer and we don't have one yet, add it - if (peer.tags.has(Tags.BOOTSTRAP) && bootstrapPeerCount < 1) { - selectedPeers.push(peer); - bootstrapPeerCount++; - } - // If we have less than 3 total peers, add non-bootstrap peers - else if (selectedPeers.length < 3 && !peer.tags.has(Tags.BOOTSTRAP)) { - selectedPeers.push(peer); - } - }); - - if (bootstrapPeerCount === 0) { - this.log(`warning: no bootstrap peers found, using random peers`); + // Fill up to numPeers with remaining random peers if needed + while (selectedPeers.length < numPeers && remainingPeers.length > 0) { + selectedPeers.push(remainingPeers.shift()!); } - return selectedPeers; + // Trim the result to the specified number of peers if more were added + return selectedPeers.slice(0, numPeers); } protected async newStream(peer: Peer): Promise { From 645d7c122c7029cb2939b6d29e991990cc4fb3b2 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 16:38:08 +0530 Subject: [PATCH 14/58] rm: log as an arg --- packages/core/src/lib/filter/index.ts | 2 +- packages/core/src/lib/light_push/index.ts | 8 +++++--- packages/core/src/lib/store/index.ts | 2 +- packages/peer-exchange/src/index.ts | 2 +- packages/peer-exchange/src/waku_peer_exchange.ts | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 7aa801a0b8..de138c22e2 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -246,7 +246,7 @@ class Filter extends BaseProtocol implements IReceiver { } constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(FilterCodec.SUBSCRIBE, libp2p.components, log); + super(FilterCodec.SUBSCRIBE, libp2p.components); libp2p.handle(FilterCodec.PUSH, this.onRequest.bind(this)).catch((e) => { log("Failed to register ", FilterCodec.PUSH, e); diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index f75dc23ae7..112c638e8c 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -1,6 +1,5 @@ import type { PeerId } from "@libp2p/interface-peer-id"; import { - Codecs, IEncoder, ILightPush, IMessage, @@ -27,6 +26,8 @@ const log = debug("waku:light-push"); export { PushResponse }; +const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1"; + /** * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/). */ @@ -34,7 +35,7 @@ class LightPush extends BaseProtocol implements ILightPush { options: ProtocolCreateOptions; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(Codecs.LightPush, libp2p.components, log); + super(LightPushCodec, libp2p.components); this.options = options || {}; } @@ -65,7 +66,8 @@ class LightPush extends BaseProtocol implements ILightPush { } const query = PushRpc.createRequest(protoMessage, pubSubTopic); - const peers = await this.getPeers(opts?.peerId); + + const peers = await this.getPeers(3, true, opts?.peerId && [opts?.peerId]); const promises = peers.map(async (peer) => { let error: SendError | undefined; diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 186550cada..5b97397ad1 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -82,7 +82,7 @@ class Store extends BaseProtocol implements IStore { options: ProtocolCreateOptions; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(StoreCodec, libp2p.components, log); + super(StoreCodec, libp2p.components); this.options = options ?? {}; } diff --git a/packages/peer-exchange/src/index.ts b/packages/peer-exchange/src/index.ts index c06f8db0b7..fe6ee7e021 100644 --- a/packages/peer-exchange/src/index.ts +++ b/packages/peer-exchange/src/index.ts @@ -1,7 +1,7 @@ export { wakuPeerExchange, - PeerExchangeCodec, WakuPeerExchange, + PeerExchangeCodec, } from "./waku_peer_exchange.js"; export { wakuPeerExchangeDiscovery, diff --git a/packages/peer-exchange/src/waku_peer_exchange.ts b/packages/peer-exchange/src/waku_peer_exchange.ts index deacb06fc2..ccd8e4611e 100644 --- a/packages/peer-exchange/src/waku_peer_exchange.ts +++ b/packages/peer-exchange/src/waku_peer_exchange.ts @@ -15,7 +15,7 @@ import { Uint8ArrayList } from "uint8arraylist"; import { PeerExchangeRPC } from "./rpc.js"; -const PeerExchangeCodec = "/vac/waku/peer-exchange/2.0.0-alpha1"; +export const PeerExchangeCodec = "/vac/waku/peer-exchange/2.0.0-alpha1"; const log = debug("waku:peer-exchange"); @@ -27,7 +27,7 @@ export class WakuPeerExchange extends BaseProtocol implements IPeerExchange { * @param components - libp2p components */ constructor(components: Libp2pComponents) { - super(PeerExchangeCodec, components, log); + super(PeerExchangeCodec, components); } /** @@ -42,7 +42,7 @@ export class WakuPeerExchange extends BaseProtocol implements IPeerExchange { numPeers: BigInt(numPeers), }); - const peer = await this.getPeers(params.peerId && [params.peerId]); + const peer = await this.getPeer(params.peerId); const stream = await this.newStream(peer); From 9d70d55f0892b032964b8bd67ec18e1ab5b0e5e0 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 16:41:43 +0530 Subject: [PATCH 15/58] add tsdoc for getPeers --- packages/core/src/lib/base_protocol.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 14bf54e333..d12c9638b0 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -51,6 +51,14 @@ export class BaseProtocol implements IBaseProtocol { return peer; } + /** + * Retrieves a list of peers based on the specified criteria. + * + * @param numPeers - The number of peers to retrieve. If you need all available peers, specify a large number. + * @param includeBootstrap - If true, includes a bootstrap peer in the result. Useful for protocols like Filter and Store that require only one peer for now. + * @param peerIds - Optional list of specific peer IDs to include in the result. This allows for the inclusion of specific peers if needed. + * @returns A Promise that resolves to an array of peers based on the specified criteria. + */ protected async getPeers( numPeers: number, includeBootstrap: boolean, From 773d1ef8c2e753474453bb693bd999680b4d976d Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 17:03:24 +0530 Subject: [PATCH 16/58] add import --- package-lock.json | 2 -- packages/core/src/lib/base_protocol.ts | 9 ++++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5ef1ed9841..f26d502014 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30810,7 +30810,6 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", - "@waku/peer-exchange": "^0.0.15", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "libp2p": "^0.46.3" @@ -35752,7 +35751,6 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", - "@waku/peer-exchange": "^0.0.15", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "cspell": "^7.0.0", diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 6d01ccaa7e..398c4edd2f 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -2,8 +2,7 @@ import type { Libp2p } from "@libp2p/interface"; import type { Stream } from "@libp2p/interface/connection"; import type { PeerId } from "@libp2p/interface/peer-id"; import { Peer, PeerStore } from "@libp2p/interface/peer-store"; -import type { IBaseProtocol, Libp2pComponents } from "@waku/interfaces"; - +import { IBaseProtocol, Libp2pComponents, Tags } from "@waku/interfaces"; import { getPeersForProtocol, selectConnection, @@ -63,11 +62,11 @@ export class BaseProtocol implements IBaseProtocol { protected async getPeers( numPeers: number, includeBootstrap: boolean, - peerIds?: PeerId[], + peerIds?: PeerId[] ): Promise { // Retrieve all peers that support the protocol const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ - this.multicodec, + this.multicodec ]); // Filter the bootstrap peer if required to include @@ -82,7 +81,7 @@ export class BaseProtocol implements IBaseProtocol { // Filter remaining peers excluding bootstrap and specified peerIds const remainingPeers = allPeersForProtocol.filter( - (peer) => peer !== bootstrapPeer && !matchingPeers.includes(peer), + (peer) => peer !== bootstrapPeer && !matchingPeers.includes(peer) ); // Initialize the list of selected peers From f34549614cac01513ef88cb727e089072e0ccdfe Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 17:12:45 +0530 Subject: [PATCH 17/58] add prettier rule to eslint --- .eslintrc.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.eslintrc.json b/.eslintrc.json index 4e17958d1b..ecfa1a0907 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -16,6 +16,12 @@ ], "globals": { "BigInt": true, "console": true, "WebAssembly": true }, "rules": { + "prettier/prettier": [ + "error", + { + "trailingComma": "none" + } + ], "comma-dangle": ["error", "never"], "@typescript-eslint/explicit-function-return-type": "off", "@typescript-eslint/explicit-module-boundary-types": "off", From efef7a2303cd4a23cb241d75e8bbd0ac845fda27 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 17:14:58 +0530 Subject: [PATCH 18/58] add: peer exchange to sdk as a dep --- package-lock.json | 2 ++ packages/sdk/package.json | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index f26d502014..c22c27b078 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30810,6 +30810,7 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", + "@waku/peer-exchange": "^0.0.15", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "libp2p": "^0.46.3" @@ -35751,6 +35752,7 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", + "@waku/peer-exchange": "*", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "cspell": "^7.0.0", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 9c9a7b6a12..356a0e9605 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -51,11 +51,12 @@ "@chainsafe/libp2p-noise": "^13.0.0", "@libp2p/mplex": "^9.0.2", "@libp2p/websockets": "^7.0.3", - "@waku/utils": "0.0.10", - "@waku/relay": "0.0.5", "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", + "@waku/peer-exchange": "^0.0.15", + "@waku/relay": "0.0.5", + "@waku/utils": "0.0.10", "libp2p": "^0.46.3" }, "devDependencies": { From 6a17cc3646d66435bc27075ac34ce3622b578f68 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 17:15:08 +0530 Subject: [PATCH 19/58] fix eslint error --- packages/core/src/lib/filter/index.ts | 2 +- packages/core/src/lib/light_push/index.ts | 12 ++++++------ packages/relay/src/index.ts | 4 ++-- packages/sdk/src/create.ts | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 852ec22684..792810a2b1 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -38,7 +38,7 @@ const log = debug("waku:filter:v2"); const FilterCodec = { SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1", - PUSH: "/vac/waku/filter-push/2.0.0-beta1", + PUSH: "/vac/waku/filter-push/2.0.0-beta1" }; type SubscriptionCallback = { diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index 0781d1e3b3..e3c6d2da5c 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -52,7 +52,7 @@ class LightPush extends BaseProtocol implements ILightPush { log("Failed to send waku light push: message is bigger than 1MB"); return { recipients, - errors: [SendError.SIZE_TOO_BIG], + errors: [SendError.SIZE_TOO_BIG] }; } @@ -61,7 +61,7 @@ class LightPush extends BaseProtocol implements ILightPush { log("Failed to encode to protoMessage, aborting push"); return { recipients, - errors: [SendError.ENCODE_FAILED], + errors: [SendError.ENCODE_FAILED] }; } @@ -73,14 +73,14 @@ class LightPush extends BaseProtocol implements ILightPush { let error: SendError | undefined; const stream = await this.newStream(peer); - + try { const res = await pipe( [query.encode()], lp.encode, stream, lp.decode, - async (source) => await all(source), + async (source) => await all(source) ); try { const bytes = new Uint8ArrayList(); @@ -111,7 +111,7 @@ class LightPush extends BaseProtocol implements ILightPush { const results = await Promise.allSettled(promises); const successfulResults = results.filter( - (result) => result.status === "fulfilled", + (result) => result.status === "fulfilled" ) as PromiseFulfilledResult<{ recipients: PeerId[]; error: SendError | undefined; @@ -123,7 +123,7 @@ class LightPush extends BaseProtocol implements ILightPush { return { recipients, - errors, + errors }; } } diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index 045f71ac00..372a3274e5 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -105,7 +105,7 @@ class Relay implements IRelay { log("Failed to send waku relay: message is bigger that 1MB"); return { recipients, - errors: [SendError.SIZE_TOO_BIG], + errors: [SendError.SIZE_TOO_BIG] }; } @@ -114,7 +114,7 @@ class Relay implements IRelay { log("Failed to encode message, aborting publish"); return { recipients, - errors: [SendError.ENCODE_FAILED], + errors: [SendError.ENCODE_FAILED] }; } diff --git a/packages/sdk/src/create.ts b/packages/sdk/src/create.ts index 2963951682..0b78fd39fc 100644 --- a/packages/sdk/src/create.ts +++ b/packages/sdk/src/create.ts @@ -150,7 +150,7 @@ export function defaultPeerDiscoveries(): (( ) => PeerDiscovery)[] { const discoveries = [ wakuDnsDiscovery([enrTree["PROD"]], DEFAULT_NODE_REQUIREMENTS), - wakuPeerExchangeDiscovery(), + wakuPeerExchangeDiscovery() ]; return discoveries; } From befbd37b6ff24033299fcb9a8a69fcb66dbbf03e Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 17 Aug 2023 18:09:32 +0530 Subject: [PATCH 20/58] add try catch --- package-lock.json | 2 +- packages/core/src/lib/light_push/index.ts | 33 ++++++++++++++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index c22c27b078..5ef1ed9841 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35752,7 +35752,7 @@ "@waku/core": "0.0.22", "@waku/dns-discovery": "0.0.16", "@waku/interfaces": "0.0.17", - "@waku/peer-exchange": "*", + "@waku/peer-exchange": "^0.0.15", "@waku/relay": "0.0.5", "@waku/utils": "0.0.10", "cspell": "^7.0.0", diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index e3c6d2da5c..e5ba7b4f04 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -47,26 +47,35 @@ class LightPush extends BaseProtocol implements ILightPush { const { pubSubTopic = DefaultPubSubTopic } = this.options; const recipients: PeerId[] = []; + let query: PushRpc; + + try { + if (!isSizeValid(message.payload)) { + log("Failed to send waku light push: message is bigger than 1MB"); + return { + recipients, + errors: [SendError.SIZE_TOO_BIG] + }; + } - if (!isSizeValid(message.payload)) { - log("Failed to send waku light push: message is bigger than 1MB"); - return { - recipients, - errors: [SendError.SIZE_TOO_BIG] - }; - } + const protoMessage = await encoder.toProtoObj(message); + if (!protoMessage) { + log("Failed to encode to protoMessage, aborting push"); + return { + recipients, + errors: [SendError.ENCODE_FAILED] + }; + } - const protoMessage = await encoder.toProtoObj(message); - if (!protoMessage) { - log("Failed to encode to protoMessage, aborting push"); + query = PushRpc.createRequest(protoMessage, pubSubTopic); + } catch (error) { + log("Failed to encode to protoMessage", error); return { recipients, errors: [SendError.ENCODE_FAILED] }; } - const query = PushRpc.createRequest(protoMessage, pubSubTopic); - const peers = await this.getPeers(3, true, opts?.peerId && [opts?.peerId]); const promises = peers.map(async (peer) => { From 83d5103d116196d63bbba1e9d6b4821283c40496 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Mon, 21 Aug 2023 14:05:53 +0530 Subject: [PATCH 21/58] revert unecessary diff --- packages/core/src/lib/filter/index.ts | 16 ++++++++-------- packages/core/src/lib/light_push/index.ts | 4 ++-- packages/core/src/lib/store/index.ts | 4 ++-- packages/peer-exchange/src/waku_peer_exchange.ts | 2 +- packages/relay/src/constants.ts | 5 +++++ packages/relay/src/index.ts | 3 +-- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 792810a2b1..85cd87be92 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -36,16 +36,16 @@ import { const log = debug("waku:filter:v2"); -const FilterCodec = { - SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1", - PUSH: "/vac/waku/filter-push/2.0.0-beta1" -}; - type SubscriptionCallback = { decoders: IDecoder[]; callback: Callback; }; +const FilterCodecs = { + SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1", + PUSH: "/vac/waku/filter-push/2.0.0-beta1" +}; + class Subscription { private readonly peer: Peer; private readonly pubSubTopic: PubSubTopic; @@ -246,10 +246,10 @@ class Filter extends BaseProtocol implements IReceiver { } constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(FilterCodec.SUBSCRIBE, libp2p.components); + super(FilterCodecs.SUBSCRIBE, libp2p.components); - libp2p.handle(FilterCodec.PUSH, this.onRequest.bind(this)).catch((e) => { - log("Failed to register ", FilterCodec.PUSH, e); + libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => { + log("Failed to register ", FilterCodecs.PUSH, e); }); this.activeSubscriptions = new Map(); diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index e5ba7b4f04..7bf32fe55b 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -24,10 +24,10 @@ import { PushRpc } from "./push_rpc.js"; const log = debug("waku:light-push"); -export { PushResponse }; - const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1"; +export { PushResponse }; + /** * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/). */ diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index b54d7c2c74..af4c87c079 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -28,12 +28,12 @@ import HistoryError = proto.HistoryResponse.HistoryError; const log = debug("waku:store"); +const StoreCodec = "/vac/waku/store/2.0.0-beta4"; + export const DefaultPageSize = 10; export { PageDirection }; -const StoreCodec = "/vac/waku/store/2.0.0-beta4"; - export interface TimeFilter { startTime: Date; endTime: Date; diff --git a/packages/peer-exchange/src/waku_peer_exchange.ts b/packages/peer-exchange/src/waku_peer_exchange.ts index e4fbdcc045..7916faba2e 100644 --- a/packages/peer-exchange/src/waku_peer_exchange.ts +++ b/packages/peer-exchange/src/waku_peer_exchange.ts @@ -1,6 +1,6 @@ import { BaseProtocol } from "@waku/core/lib/base_protocol"; import { EnrDecoder } from "@waku/enr"; -import { +import type { IPeerExchange, Libp2pComponents, PeerExchangeQueryParams, diff --git a/packages/relay/src/constants.ts b/packages/relay/src/constants.ts index 2649117cd3..6c02e4d59b 100644 --- a/packages/relay/src/constants.ts +++ b/packages/relay/src/constants.ts @@ -1,6 +1,11 @@ export const second = 1000; export const minute = 60 * second; +/** + * RelayCodec is the libp2p identifier for the waku relay protocol + */ +export const RelayCodec = "/vac/waku/relay/2.0.0"; + /** * RelayGossipFactor affects how many peers we will emit gossip to at each heartbeat. * We will send gossip to RelayGossipFactor * (total number of non-mesh peers), or diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index 372a3274e5..5203640f97 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -28,6 +28,7 @@ import { import { groupByContentTopic, isSizeValid, toAsyncIterator } from "@waku/utils"; import debug from "debug"; +import { RelayCodec } from "./constants.js"; import { messageValidator } from "./message_validator.js"; import { TopicOnlyDecoder } from "./topic_only_message.js"; @@ -41,8 +42,6 @@ export type Observer = { export type RelayCreateOptions = ProtocolCreateOptions & GossipsubOpts; export type ContentTopic = string; -const RelayCodec = "/vac/waku/relay/2.0.0"; - /** * Implements the [Waku v2 Relay protocol](https://rfc.vac.dev/spec/11/). * Throws if libp2p.pubsub does not support Waku Relay From 9583bda0e97a5ed9a9a31133b5b4e2a97a4db47e Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Mon, 21 Aug 2023 14:08:47 +0530 Subject: [PATCH 22/58] revert unecessary diff --- packages/core/src/lib/filter/index.ts | 16 ++++++++-------- packages/core/src/lib/light_push/index.ts | 4 ++-- packages/core/src/lib/store/index.ts | 4 ++-- packages/peer-exchange/src/waku_peer_exchange.ts | 2 +- packages/relay/src/constants.ts | 5 +++++ packages/relay/src/index.ts | 3 +-- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 792810a2b1..85cd87be92 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -36,16 +36,16 @@ import { const log = debug("waku:filter:v2"); -const FilterCodec = { - SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1", - PUSH: "/vac/waku/filter-push/2.0.0-beta1" -}; - type SubscriptionCallback = { decoders: IDecoder[]; callback: Callback; }; +const FilterCodecs = { + SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1", + PUSH: "/vac/waku/filter-push/2.0.0-beta1" +}; + class Subscription { private readonly peer: Peer; private readonly pubSubTopic: PubSubTopic; @@ -246,10 +246,10 @@ class Filter extends BaseProtocol implements IReceiver { } constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - super(FilterCodec.SUBSCRIBE, libp2p.components); + super(FilterCodecs.SUBSCRIBE, libp2p.components); - libp2p.handle(FilterCodec.PUSH, this.onRequest.bind(this)).catch((e) => { - log("Failed to register ", FilterCodec.PUSH, e); + libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => { + log("Failed to register ", FilterCodecs.PUSH, e); }); this.activeSubscriptions = new Map(); diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index e5ba7b4f04..4da902729b 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -24,9 +24,9 @@ import { PushRpc } from "./push_rpc.js"; const log = debug("waku:light-push"); -export { PushResponse }; +export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1"; -const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1"; +export { PushResponse }; /** * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/). diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index b54d7c2c74..7583caac72 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -28,12 +28,12 @@ import HistoryError = proto.HistoryResponse.HistoryError; const log = debug("waku:store"); +export const StoreCodec = "/vac/waku/store/2.0.0-beta4"; + export const DefaultPageSize = 10; export { PageDirection }; -const StoreCodec = "/vac/waku/store/2.0.0-beta4"; - export interface TimeFilter { startTime: Date; endTime: Date; diff --git a/packages/peer-exchange/src/waku_peer_exchange.ts b/packages/peer-exchange/src/waku_peer_exchange.ts index e4fbdcc045..7916faba2e 100644 --- a/packages/peer-exchange/src/waku_peer_exchange.ts +++ b/packages/peer-exchange/src/waku_peer_exchange.ts @@ -1,6 +1,6 @@ import { BaseProtocol } from "@waku/core/lib/base_protocol"; import { EnrDecoder } from "@waku/enr"; -import { +import type { IPeerExchange, Libp2pComponents, PeerExchangeQueryParams, diff --git a/packages/relay/src/constants.ts b/packages/relay/src/constants.ts index 2649117cd3..4f6a3d9cb0 100644 --- a/packages/relay/src/constants.ts +++ b/packages/relay/src/constants.ts @@ -1,6 +1,11 @@ export const second = 1000; export const minute = 60 * second; +/** + * RelayCodec is the libp2p identifier for the waku relay protocol + */ +export const RelayCodecs = "/vac/waku/relay/2.0.0"; + /** * RelayGossipFactor affects how many peers we will emit gossip to at each heartbeat. * We will send gossip to RelayGossipFactor * (total number of non-mesh peers), or diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index 372a3274e5..5203640f97 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -28,6 +28,7 @@ import { import { groupByContentTopic, isSizeValid, toAsyncIterator } from "@waku/utils"; import debug from "debug"; +import { RelayCodec } from "./constants.js"; import { messageValidator } from "./message_validator.js"; import { TopicOnlyDecoder } from "./topic_only_message.js"; @@ -41,8 +42,6 @@ export type Observer = { export type RelayCreateOptions = ProtocolCreateOptions & GossipsubOpts; export type ContentTopic = string; -const RelayCodec = "/vac/waku/relay/2.0.0"; - /** * Implements the [Waku v2 Relay protocol](https://rfc.vac.dev/spec/11/). * Throws if libp2p.pubsub does not support Waku Relay From c63d14654ee6705cd5876bda54d02d7a3bee5c8d Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Mon, 21 Aug 2023 15:33:05 +0530 Subject: [PATCH 23/58] fix imports --- packages/relay/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index 5203640f97..0ebc512834 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -28,7 +28,7 @@ import { import { groupByContentTopic, isSizeValid, toAsyncIterator } from "@waku/utils"; import debug from "debug"; -import { RelayCodec } from "./constants.js"; +import { RelayCodecs } from "./constants.js"; import { messageValidator } from "./message_validator.js"; import { TopicOnlyDecoder } from "./topic_only_message.js"; @@ -50,7 +50,7 @@ class Relay implements IRelay { private readonly pubSubTopic: string; private defaultDecoder: IDecoder; - public static multicodec: string = RelayCodec; + public static multicodec: string = RelayCodecs; public readonly gossipSub: GossipSub; /** @@ -265,7 +265,7 @@ export function wakuGossipSub( fallbackToFloodsub: false }; const pubsub = new GossipSub(components, init); - pubsub.multicodecs = [RelayCodec]; + pubsub.multicodecs = [RelayCodecs]; return pubsub; }; } From 6fdcf61c34fa0def3a8bb2929b0ffd4a779a7444 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 22 Aug 2023 13:25:10 +0530 Subject: [PATCH 24/58] convert relaycodecs to array --- packages/relay/src/constants.ts | 2 +- packages/relay/src/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/relay/src/constants.ts b/packages/relay/src/constants.ts index 4f6a3d9cb0..067084d991 100644 --- a/packages/relay/src/constants.ts +++ b/packages/relay/src/constants.ts @@ -4,7 +4,7 @@ export const minute = 60 * second; /** * RelayCodec is the libp2p identifier for the waku relay protocol */ -export const RelayCodecs = "/vac/waku/relay/2.0.0"; +export const RelayCodecs = ["/vac/waku/relay/2.0.0"]; /** * RelayGossipFactor affects how many peers we will emit gossip to at each heartbeat. diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index 0ebc512834..52a57823ad 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -50,7 +50,7 @@ class Relay implements IRelay { private readonly pubSubTopic: string; private defaultDecoder: IDecoder; - public static multicodec: string = RelayCodecs; + public static multicodec: string = RelayCodecs[0]; public readonly gossipSub: GossipSub; /** @@ -265,7 +265,7 @@ export function wakuGossipSub( fallbackToFloodsub: false }; const pubsub = new GossipSub(components, init); - pubsub.multicodecs = [RelayCodecs]; + pubsub.multicodecs = RelayCodecs; return pubsub; }; } From 5dff8646520a6cc7b15b1243bf74c23a16e97114 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 08:55:10 +0530 Subject: [PATCH 25/58] remove: peerId as an arg for protocol methods --- packages/core/src/lib/filter/index.ts | 13 ++++--------- packages/core/src/lib/light_push/index.ts | 9 ++------- packages/core/src/lib/store/index.ts | 12 ++---------- packages/interfaces/src/peer_exchange.ts | 2 -- packages/interfaces/src/protocols.ts | 5 +---- packages/peer-exchange/src/waku_peer_exchange.ts | 2 +- 6 files changed, 10 insertions(+), 33 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 85cd87be92..edf0e51d2f 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -1,5 +1,4 @@ import { Stream } from "@libp2p/interface/connection"; -import type { PeerId } from "@libp2p/interface/peer-id"; import type { Peer } from "@libp2p/interface/peer-store"; import type { IncomingStreamData } from "@libp2p/interface-internal/registrar"; import type { @@ -257,14 +256,11 @@ class Filter extends BaseProtocol implements IReceiver { this.options = options ?? {}; } - async createSubscription( - pubSubTopic?: string, - peerId?: PeerId - ): Promise { + async createSubscription(pubSubTopic?: string): Promise { const _pubSubTopic = pubSubTopic ?? this.options.pubSubTopic ?? DefaultPubSubTopic; - const peer = await this.getPeer(peerId); + const peer = await this.getPeer(); const subscription = this.getActiveSubscription(_pubSubTopic, peer.id.toString()) ?? @@ -301,10 +297,9 @@ class Filter extends BaseProtocol implements IReceiver { */ async subscribe( decoders: IDecoder | IDecoder[], - callback: Callback, - opts?: ProtocolOptions + callback: Callback ): Promise { - const subscription = await this.createSubscription(undefined, opts?.peerId); + const subscription = await this.createSubscription(); await subscription.subscribe(decoders, callback); diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index 4da902729b..44bc82dfe3 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -5,7 +5,6 @@ import { IMessage, Libp2p, ProtocolCreateOptions, - ProtocolOptions, SendError, SendResult } from "@waku/interfaces"; @@ -39,11 +38,7 @@ class LightPush extends BaseProtocol implements ILightPush { this.options = options || {}; } - async send( - encoder: IEncoder, - message: IMessage, - opts?: ProtocolOptions - ): Promise { + async send(encoder: IEncoder, message: IMessage): Promise { const { pubSubTopic = DefaultPubSubTopic } = this.options; const recipients: PeerId[] = []; @@ -76,7 +71,7 @@ class LightPush extends BaseProtocol implements ILightPush { }; } - const peers = await this.getPeers(3, true, opts?.peerId && [opts?.peerId]); + const peers = await this.getPeers(3, true); const promises = peers.map(async (peer) => { let error: SendError | undefined; diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 7583caac72..aba9ab1efd 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -1,5 +1,4 @@ import type { Stream } from "@libp2p/interface/connection"; -import type { PeerId } from "@libp2p/interface/peer-id"; import { sha256 } from "@noble/hashes/sha256"; import { Cursor, @@ -40,10 +39,6 @@ export interface TimeFilter { } export interface QueryOptions { - /** - * The peer to query. If undefined, a pseudo-random peer is selected from the connected Waku Store peers. - */ - peerId?: PeerId; /** * The direction in which pages are retrieved: * - { @link PageDirection.BACKWARD }: Most recent page first. @@ -225,12 +220,9 @@ class Store extends BaseProtocol implements IStore { { contentTopics, startTime, endTime } ); - log("Querying history with the following options", { - ...options, - peerId: options?.peerId?.toString() - }); + log("Querying history with the following options", options); - const peer = await this.getPeer(options?.peerId); + const peer = await this.getPeer(); for await (const messages of paginate( this.newStream.bind(this, peer), diff --git a/packages/interfaces/src/peer_exchange.ts b/packages/interfaces/src/peer_exchange.ts index 8183d6fe13..37559f6b98 100644 --- a/packages/interfaces/src/peer_exchange.ts +++ b/packages/interfaces/src/peer_exchange.ts @@ -1,4 +1,3 @@ -import type { PeerId } from "@libp2p/interface/peer-id"; import type { PeerStore } from "@libp2p/interface/peer-store"; import type { ConnectionManager } from "@libp2p/interface-internal/connection-manager"; @@ -11,7 +10,6 @@ export interface IPeerExchange extends IBaseProtocol { export interface PeerExchangeQueryParams { numPeers: number; - peerId?: PeerId; } export interface PeerExchangeResponse { diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts index 57f9b19865..a8ddc00b91 100644 --- a/packages/interfaces/src/protocols.ts +++ b/packages/interfaces/src/protocols.ts @@ -55,10 +55,7 @@ export type ProtocolCreateOptions = { }; export type ProtocolOptions = { - /** - * Optionally specify an PeerId for the protocol request. If not included, will use a random peer. - */ - peerId?: PeerId; + // }; export type Callback = ( diff --git a/packages/peer-exchange/src/waku_peer_exchange.ts b/packages/peer-exchange/src/waku_peer_exchange.ts index 7916faba2e..e823696844 100644 --- a/packages/peer-exchange/src/waku_peer_exchange.ts +++ b/packages/peer-exchange/src/waku_peer_exchange.ts @@ -42,7 +42,7 @@ export class WakuPeerExchange extends BaseProtocol implements IPeerExchange { numPeers: BigInt(numPeers) }); - const peer = await this.getPeer(params.peerId); + const peer = await this.getPeer(); const stream = await this.newStream(peer); From f75b2d9bbf67ced2c4c17e60bbe679d715dcee3a Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 09:02:10 +0530 Subject: [PATCH 26/58] keep peerId as an arg for peer-exchange --- packages/interfaces/src/peer_exchange.ts | 2 ++ packages/peer-exchange/src/waku_peer_exchange.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/interfaces/src/peer_exchange.ts b/packages/interfaces/src/peer_exchange.ts index 37559f6b98..8183d6fe13 100644 --- a/packages/interfaces/src/peer_exchange.ts +++ b/packages/interfaces/src/peer_exchange.ts @@ -1,3 +1,4 @@ +import type { PeerId } from "@libp2p/interface/peer-id"; import type { PeerStore } from "@libp2p/interface/peer-store"; import type { ConnectionManager } from "@libp2p/interface-internal/connection-manager"; @@ -10,6 +11,7 @@ export interface IPeerExchange extends IBaseProtocol { export interface PeerExchangeQueryParams { numPeers: number; + peerId?: PeerId; } export interface PeerExchangeResponse { diff --git a/packages/peer-exchange/src/waku_peer_exchange.ts b/packages/peer-exchange/src/waku_peer_exchange.ts index e823696844..7916faba2e 100644 --- a/packages/peer-exchange/src/waku_peer_exchange.ts +++ b/packages/peer-exchange/src/waku_peer_exchange.ts @@ -42,7 +42,7 @@ export class WakuPeerExchange extends BaseProtocol implements IPeerExchange { numPeers: BigInt(numPeers) }); - const peer = await this.getPeer(); + const peer = await this.getPeer(params.peerId); const stream = await this.newStream(peer); From 7d14956118c0c844d60ce87a5d7ab7ce292bc17e Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 09:02:33 +0530 Subject: [PATCH 27/58] remove: peerId from getPeers() --- packages/core/src/lib/base_protocol.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 398c4edd2f..ce49a70183 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -56,13 +56,11 @@ export class BaseProtocol implements IBaseProtocol { * * @param numPeers - The number of peers to retrieve. If you need all available peers, specify a large number. * @param includeBootstrap - If true, includes a bootstrap peer in the result. Useful for protocols like Filter and Store that require only one peer for now. - * @param peerIds - Optional list of specific peer IDs to include in the result. This allows for the inclusion of specific peers if needed. * @returns A Promise that resolves to an array of peers based on the specified criteria. */ protected async getPeers( numPeers: number, - includeBootstrap: boolean, - peerIds?: PeerId[] + includeBootstrap: boolean ): Promise { // Retrieve all peers that support the protocol const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ @@ -74,14 +72,9 @@ export class BaseProtocol implements IBaseProtocol { ? allPeersForProtocol.find((peer) => peer.tags.has(Tags.BOOTSTRAP)) : undefined; - // Filter the peers that match the specified peerIds - const matchingPeers = peerIds - ? allPeersForProtocol.filter((peer) => peerIds.includes(peer.id)) - : []; - - // Filter remaining peers excluding bootstrap and specified peerIds + // Filter remaining peers excluding bootstrap const remainingPeers = allPeersForProtocol.filter( - (peer) => peer !== bootstrapPeer && !matchingPeers.includes(peer) + (peer) => peer !== bootstrapPeer ); // Initialize the list of selected peers @@ -92,9 +85,6 @@ export class BaseProtocol implements IBaseProtocol { selectedPeers.push(bootstrapPeer); } - // Add the specified peerIds if available - selectedPeers.push(...matchingPeers); - // Fill up to numPeers with remaining random peers if needed while (selectedPeers.length < numPeers && remainingPeers.length > 0) { selectedPeers.push(remainingPeers.shift()!); From 1c93c128539f2f15cd37bcd862c54b4e5e1e98dc Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 09:04:25 +0530 Subject: [PATCH 28/58] lightpush: extract hardcoded numPeers as a constant --- packages/core/src/lib/light_push/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index 44bc82dfe3..ee5e44c839 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -27,6 +27,8 @@ export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1"; export { PushResponse }; +export const DEFAULT_NUM_PEERS_PROTOCOL = 3; + /** * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/). */ @@ -71,7 +73,7 @@ class LightPush extends BaseProtocol implements ILightPush { }; } - const peers = await this.getPeers(3, true); + const peers = await this.getPeers(DEFAULT_NUM_PEERS_PROTOCOL, true); const promises = peers.map(async (peer) => { let error: SendError | undefined; From 04b9bb56b56ed07edabdbb4def1ba1ca0907d123 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 16:57:44 +0530 Subject: [PATCH 29/58] return all peers if numPeers is 0 and increase readability for random peers --- packages/core/src/lib/base_protocol.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index ce49a70183..ebd739a32d 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -54,7 +54,7 @@ export class BaseProtocol implements IBaseProtocol { /** * Retrieves a list of peers based on the specified criteria. * - * @param numPeers - The number of peers to retrieve. If you need all available peers, specify a large number. + * @param numPeers - The number of peers to retrieve. If 0, all peers are returned. * @param includeBootstrap - If true, includes a bootstrap peer in the result. Useful for protocols like Filter and Store that require only one peer for now. * @returns A Promise that resolves to an array of peers based on the specified criteria. */ @@ -67,6 +67,10 @@ export class BaseProtocol implements IBaseProtocol { this.multicodec ]); + if (numPeers === 0) { + return allPeersForProtocol; + } + // Filter the bootstrap peer if required to include const bootstrapPeer = includeBootstrap ? allPeersForProtocol.find((peer) => peer.tags.has(Tags.BOOTSTRAP)) @@ -87,11 +91,12 @@ export class BaseProtocol implements IBaseProtocol { // Fill up to numPeers with remaining random peers if needed while (selectedPeers.length < numPeers && remainingPeers.length > 0) { - selectedPeers.push(remainingPeers.shift()!); + const randomIndex = Math.floor(Math.random() * remainingPeers.length); + const randomPeer = remainingPeers.splice(randomIndex, 1)[0]; + selectedPeers.push(randomPeer); } - // Trim the result to the specified number of peers if more were added - return selectedPeers.slice(0, numPeers); + return selectedPeers; } protected async newStream(peer: Peer): Promise { From 1f81ae18a2aed5a689fbfe6f3de11defcebc1dcb Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 17:00:08 +0530 Subject: [PATCH 30/58] refactor considering more than 1 bootstrap peers can exist --- packages/core/src/lib/base_protocol.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index ebd739a32d..1baa4af04a 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -71,23 +71,21 @@ export class BaseProtocol implements IBaseProtocol { return allPeersForProtocol; } - // Filter the bootstrap peer if required to include - const bootstrapPeer = includeBootstrap - ? allPeersForProtocol.find((peer) => peer.tags.has(Tags.BOOTSTRAP)) - : undefined; + // Filter the bootstrap peers if required to include + const bootstrapPeers = includeBootstrap + ? allPeersForProtocol.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)) + : []; - // Filter remaining peers excluding bootstrap + // Filter remaining peers excluding bootstrap peers const remainingPeers = allPeersForProtocol.filter( - (peer) => peer !== bootstrapPeer + (peer) => !bootstrapPeers.includes(peer) ); // Initialize the list of selected peers const selectedPeers: Peer[] = []; - // Add the bootstrap peer if available and required - if (bootstrapPeer) { - selectedPeers.push(bootstrapPeer); - } + // Add the bootstrap peers if available and required + selectedPeers.push(...bootstrapPeers); // Fill up to numPeers with remaining random peers if needed while (selectedPeers.length < numPeers && remainingPeers.length > 0) { From dce7b3bfd94b69a2ed7a8f9de6c3997be87cbd0c Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 17:06:51 +0530 Subject: [PATCH 31/58] use `getPeers` --- packages/core/src/lib/filter/index.ts | 2 +- packages/core/src/lib/store/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index edf0e51d2f..361e28d70e 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -260,7 +260,7 @@ class Filter extends BaseProtocol implements IReceiver { const _pubSubTopic = pubSubTopic ?? this.options.pubSubTopic ?? DefaultPubSubTopic; - const peer = await this.getPeer(); + const peer = (await this.getPeers(1, true))[0]; const subscription = this.getActiveSubscription(_pubSubTopic, peer.id.toString()) ?? diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 82eb3ad1cc..323593e436 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -243,7 +243,7 @@ class Store extends BaseProtocol implements IStore { log("Querying history with the following options", options); - const peer = await this.getPeer(); + const peer = (await this.getPeers(0, true))[0]; for await (const messages of paginate( this.newStream.bind(this, peer), From a376bddd3136048f2dfd905336a2a8fe500b7219 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 29 Aug 2023 17:09:59 +0530 Subject: [PATCH 32/58] change arg for `getPeers` to object --- packages/core/src/lib/base_protocol.ts | 11 +++++++---- packages/core/src/lib/filter/index.ts | 4 +++- packages/core/src/lib/light_push/index.ts | 5 ++++- packages/core/src/lib/store/index.ts | 4 +++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 1baa4af04a..08d641eb05 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -58,10 +58,13 @@ export class BaseProtocol implements IBaseProtocol { * @param includeBootstrap - If true, includes a bootstrap peer in the result. Useful for protocols like Filter and Store that require only one peer for now. * @returns A Promise that resolves to an array of peers based on the specified criteria. */ - protected async getPeers( - numPeers: number, - includeBootstrap: boolean - ): Promise { + protected async getPeers({ + numPeers, + includeBootstrap + }: { + numPeers: number; + includeBootstrap: boolean; + }): Promise { // Retrieve all peers that support the protocol const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ this.multicodec diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 361e28d70e..5da7b2b664 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -260,7 +260,9 @@ class Filter extends BaseProtocol implements IReceiver { const _pubSubTopic = pubSubTopic ?? this.options.pubSubTopic ?? DefaultPubSubTopic; - const peer = (await this.getPeers(1, true))[0]; + const peer = ( + await this.getPeers({ includeBootstrap: true, numPeers: 1 }) + )[0]; const subscription = this.getActiveSubscription(_pubSubTopic, peer.id.toString()) ?? diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index 56b86502bc..91f19dc6d4 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -70,7 +70,10 @@ class LightPush extends BaseProtocol implements ILightPush { }; } - const peers = await this.getPeers(DEFAULT_NUM_PEERS_PROTOCOL, true); + const peers = await this.getPeers({ + includeBootstrap: true, + numPeers: DEFAULT_NUM_PEERS_PROTOCOL + }); const promises = peers.map(async (peer) => { let error: SendError | undefined; diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 323593e436..f58670d0d5 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -243,7 +243,9 @@ class Store extends BaseProtocol implements IStore { log("Querying history with the following options", options); - const peer = (await this.getPeers(0, true))[0]; + const peer = ( + await this.getPeers({ includeBootstrap: true, numPeers: 1 }) + )[0]; for await (const messages of paginate( this.newStream.bind(this, peer), From ec3bcc805e750c7bc6a261b1a410474c1076ad73 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 31 Aug 2023 21:59:21 +0530 Subject: [PATCH 33/58] address comments --- packages/core/src/lib/base_protocol.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 08d641eb05..7adcda2536 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -70,20 +70,24 @@ export class BaseProtocol implements IBaseProtocol { this.multicodec ]); - if (numPeers === 0) { - return allPeersForProtocol; - } - - // Filter the bootstrap peers if required to include + // Collect the bootstrap peers if required to include const bootstrapPeers = includeBootstrap ? allPeersForProtocol.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)) : []; - // Filter remaining peers excluding bootstrap peers + // Collect non-bootstrap peers const remainingPeers = allPeersForProtocol.filter( (peer) => !bootstrapPeers.includes(peer) ); + if (numPeers === 0) { + if (includeBootstrap) { + return allPeersForProtocol; + } else { + return remainingPeers; + } + } + // Initialize the list of selected peers const selectedPeers: Peer[] = []; From 1b0e1feabfd5ee3b44365384b85e6bb8a3e687cd Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 1 Sep 2023 00:00:31 +0530 Subject: [PATCH 34/58] refactor tests for new API --- packages/tests/tests/store.node.spec.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/tests/tests/store.node.spec.ts b/packages/tests/tests/store.node.spec.ts index cde93d098c..53736af4bd 100644 --- a/packages/tests/tests/store.node.spec.ts +++ b/packages/tests/tests/store.node.spec.ts @@ -488,8 +488,6 @@ describe("Waku Store", () => { await waku.dial(await nwaku.getMultiaddrWithId()); await waitForRemotePeer(waku, [Protocols.Store]); - const nwakuPeerId = await nwaku.getPeerId(); - const firstMessages: IMessage[] = []; await waku.store.queryWithOrderedCallback( [TestDecoder], @@ -499,7 +497,6 @@ describe("Waku Store", () => { } }, { - peerId: nwakuPeerId, timeFilter: { startTime, endTime: message1Timestamp } } ); @@ -511,7 +508,6 @@ describe("Waku Store", () => { bothMessages.push(msg); }, { - peerId: nwakuPeerId, timeFilter: { startTime, endTime From c7fca3eace25cba2682cb24376bccfc12148734f Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 1 Sep 2023 00:02:55 +0530 Subject: [PATCH 35/58] lightpush: make constant the class variable --- packages/core/src/lib/light_push/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index b82cb58b6a..5fa4c1b524 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -25,7 +25,6 @@ const log = debug("waku:light-push"); export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1"; export { PushResponse }; -export const DEFAULT_NUM_PEERS_PROTOCOL = 3; type PreparePushMessageResult = | { @@ -42,6 +41,7 @@ type PreparePushMessageResult = */ class LightPush extends BaseProtocol implements ILightPush { options: ProtocolCreateOptions; + private readonly NUM_PEERS_PROTOCOL = 1; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { super(LightPushCodec, libp2p.components); @@ -99,7 +99,7 @@ class LightPush extends BaseProtocol implements ILightPush { const peers = await this.getPeers({ includeBootstrap: true, - numPeers: DEFAULT_NUM_PEERS_PROTOCOL + numPeers: this.NUM_PEERS_PROTOCOL }); const promises = peers.map(async (peer) => { From 191355bb1dd77c2b74e297c8afd81246cb82d7aa Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 1 Sep 2023 20:26:23 +0530 Subject: [PATCH 36/58] use `maxBootstrapPeers` instead of `includeBootstrap` --- packages/core/src/lib/base_protocol.ts | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 7adcda2536..b464ed285f 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -54,45 +54,39 @@ export class BaseProtocol implements IBaseProtocol { /** * Retrieves a list of peers based on the specified criteria. * - * @param numPeers - The number of peers to retrieve. If 0, all peers are returned. - * @param includeBootstrap - If true, includes a bootstrap peer in the result. Useful for protocols like Filter and Store that require only one peer for now. + * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned. + * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. * @returns A Promise that resolves to an array of peers based on the specified criteria. */ protected async getPeers({ numPeers, - includeBootstrap + maxBootstrapPeers }: { numPeers: number; - includeBootstrap: boolean; + maxBootstrapPeers: number; }): Promise { // Retrieve all peers that support the protocol const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ this.multicodec ]); - // Collect the bootstrap peers if required to include - const bootstrapPeers = includeBootstrap - ? allPeersForProtocol.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)) - : []; + // Collect the bootstrap peers up to the specified maximum + const bootstrapPeers = allPeersForProtocol + .filter((peer) => peer.tags.has(Tags.BOOTSTRAP)) + .slice(0, maxBootstrapPeers); // Collect non-bootstrap peers const remainingPeers = allPeersForProtocol.filter( (peer) => !bootstrapPeers.includes(peer) ); + // If numPeers is 0, return all peers if (numPeers === 0) { - if (includeBootstrap) { - return allPeersForProtocol; - } else { - return remainingPeers; - } + return [...bootstrapPeers, ...remainingPeers]; } - // Initialize the list of selected peers - const selectedPeers: Peer[] = []; - - // Add the bootstrap peers if available and required - selectedPeers.push(...bootstrapPeers); + // Initialize the list of selected peers with the bootstrap peers + const selectedPeers: Peer[] = [...bootstrapPeers]; // Fill up to numPeers with remaining random peers if needed while (selectedPeers.length < numPeers && remainingPeers.length > 0) { From 6820ebe49f8eda17b590b570334565eb52814297 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 1 Sep 2023 20:54:18 +0530 Subject: [PATCH 37/58] refactor protocols for new API --- packages/core/src/lib/filter/index.ts | 6 +++++- packages/core/src/lib/light_push/index.ts | 2 +- packages/core/src/lib/store/index.ts | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 5da7b2b664..4b2738e899 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -227,6 +227,7 @@ class Subscription { class Filter extends BaseProtocol implements IReceiver { private readonly options: ProtocolCreateOptions; private activeSubscriptions = new Map(); + private readonly NUM_PEERS_PROTOCOL = 1; private getActiveSubscription( pubSubTopic: PubSubTopic, @@ -261,7 +262,10 @@ class Filter extends BaseProtocol implements IReceiver { pubSubTopic ?? this.options.pubSubTopic ?? DefaultPubSubTopic; const peer = ( - await this.getPeers({ includeBootstrap: true, numPeers: 1 }) + await this.getPeers({ + maxBootstrapPeers: 1, + numPeers: this.NUM_PEERS_PROTOCOL + }) )[0]; const subscription = diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index 5fa4c1b524..6a34cc9396 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -98,7 +98,7 @@ class LightPush extends BaseProtocol implements ILightPush { } const peers = await this.getPeers({ - includeBootstrap: true, + maxBootstrapPeers: 1, numPeers: this.NUM_PEERS_PROTOCOL }); diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index f58670d0d5..5e73fb48dc 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -75,6 +75,7 @@ export interface QueryOptions { */ class Store extends BaseProtocol implements IStore { options: ProtocolCreateOptions; + private readonly NUM_PEERS_PROTOCOL = 1; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { super(StoreCodec, libp2p.components); @@ -244,7 +245,10 @@ class Store extends BaseProtocol implements IStore { log("Querying history with the following options", options); const peer = ( - await this.getPeers({ includeBootstrap: true, numPeers: 1 }) + await this.getPeers({ + numPeers: this.NUM_PEERS_PROTOCOL, + maxBootstrapPeers: 1 + }) )[0]; for await (const messages of paginate( From e968d4d442139ce62c8ccf10ad97b635fc896655 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 1 Sep 2023 21:12:04 +0530 Subject: [PATCH 38/58] add tests for `getPeers` --- packages/tests/tests/utils.spec.ts | 95 +++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/packages/tests/tests/utils.spec.ts b/packages/tests/tests/utils.spec.ts index 006170de95..b7c769e8c1 100644 --- a/packages/tests/tests/utils.spec.ts +++ b/packages/tests/tests/utils.spec.ts @@ -1,15 +1,19 @@ +import { Peer } from "@libp2p/interface/peer-store"; +import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; import { createDecoder, createEncoder, DefaultPubSubTopic, waitForRemotePeer } from "@waku/core"; -import type { LightNode } from "@waku/interfaces"; +import { LightNode, Tags } from "@waku/interfaces"; import { Protocols } from "@waku/interfaces"; import { createLightNode } from "@waku/sdk"; import { toAsyncIterator } from "@waku/utils"; import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; +import * as libp2pUtils from "@waku/utils/libp2p"; import { expect } from "chai"; +import sinon, { SinonStub } from "sinon"; import { makeLogFileName, NOISE_KEY_1 } from "../src/index.js"; import { NimGoNode } from "../src/node/node.js"; @@ -115,3 +119,92 @@ describe("Util: toAsyncIterator: Filter", () => { expect(result.done).to.eq(true); }); }); + +describe("getPeers function", function () { + let getPeersForProtocolStub: SinonStub; + let waku: LightNode | undefined; + + beforeEach(async function () { + waku = await createLightNode(); + getPeersForProtocolStub = sinon.stub(libp2pUtils, "getPeersForProtocol"); + }); + + afterEach(function () { + sinon.restore(); + }); + + it.only("should return all peers when numPeers is 0", async function () { + const peer1 = await createSecp256k1PeerId(); + const peer2 = await createSecp256k1PeerId(); + const peer3 = await createSecp256k1PeerId(); + + const mockPeers = [ + { id: peer1, tags: [Tags.BOOTSTRAP] }, + { id: peer2, tags: [Tags.BOOTSTRAP] }, + { id: peer3, tags: [Tags.BOOTSTRAP] } + ] as unknown as Peer[]; + + getPeersForProtocolStub.resolves(mockPeers); + + const result = await (waku?.lightPush as any).getPeers({ + numPeers: 0 + }); + expect(result).to.deep.equal(mockPeers); + }); + + it("should return all peers, except bootstrap, when numPeers is 0 & maxBootstrap is defined", async function () { + const peer1 = await createSecp256k1PeerId(); + const peer2 = await createSecp256k1PeerId(); + const peer3 = await createSecp256k1PeerId(); + const peer4 = await createSecp256k1PeerId(); + const peer5 = await createSecp256k1PeerId(); + + const mockPeers = [ + { id: peer1, tags: [Tags.BOOTSTRAP] }, + { id: peer2, tags: [Tags.BOOTSTRAP] }, + { id: peer3, tags: [Tags.PEER_EXCHANGE] }, + { id: peer4, tags: [Tags.PEER_EXCHANGE] }, + { id: peer5, tags: [Tags.PEER_EXCHANGE] } + ] as unknown as Peer[]; + + getPeersForProtocolStub.resolves(mockPeers); + + const result = await (waku?.lightPush as any).getPeers({ + numPeers: 0, + maxBootstrap: 1 + }); + + // result should have 1 bootstrap peers, and a total of 4 peers + expect(result.length).to.equal(4); + expect( + result.filter((peer: Peer) => peer.tags.has(Tags.BOOTSTRAP)).length + ).to.equal(1); + }); + + it("should return only bootstrap peers up to maxBootstrapPeers", async function () { + const peer1 = await createSecp256k1PeerId(); + const peer2 = await createSecp256k1PeerId(); + const peer3 = await createSecp256k1PeerId(); + const peer4 = await createSecp256k1PeerId(); + const peer5 = await createSecp256k1PeerId(); + const mockPeers = [ + { id: peer1, tags: [Tags.BOOTSTRAP] }, + { id: peer2, tags: [Tags.BOOTSTRAP] }, + { id: peer3, tags: [Tags.BOOTSTRAP] }, + { id: peer4, tags: [Tags.PEER_EXCHANGE] }, + { id: peer5, tags: [Tags.PEER_EXCHANGE] } + ] as unknown as Peer[]; + + getPeersForProtocolStub.resolves(mockPeers); + + const result = await (waku?.lightPush as any).getPeers({ + numPeers: 5, + maxBootstrapPeers: 2 + }); + + // check that result has at least 2 bootstrap peers and no more than 5 peers + expect(result.length).to.be.at.least(2); + expect(result.length).to.be.at.most(5); + expect(result.filter((peer: Peer) => peer.tags.has(Tags.BOOTSTRAP)).length); + }); +}); From 401265bc68bb32553fb82ffc79c28174bb4371dc Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Sat, 2 Sep 2023 15:41:22 +0530 Subject: [PATCH 39/58] skip getPeers test --- packages/tests/tests/utils.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/tests/tests/utils.spec.ts b/packages/tests/tests/utils.spec.ts index b7c769e8c1..3151a606c5 100644 --- a/packages/tests/tests/utils.spec.ts +++ b/packages/tests/tests/utils.spec.ts @@ -120,7 +120,12 @@ describe("Util: toAsyncIterator: Filter", () => { }); }); -describe("getPeers function", function () { +// these tests are skipped until we can figure out how to mock the standalone functions +// sinon doesn't seem to work with the standalone functions +// some helper utilities like proxyquire and rewire were also tried, but they don't seem to work either +// possible solution is the upgrade to jest, which has better mocking capabilities +// https://github.com/waku-org/js-waku/issues/1144 +describe.skip("getPeers function", function () { let getPeersForProtocolStub: SinonStub; let waku: LightNode | undefined; From 1b912789082411a2af090902b599642de5c5eba9 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Sat, 2 Sep 2023 15:47:04 +0530 Subject: [PATCH 40/58] rm: only from test --- packages/tests/tests/utils.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tests/tests/utils.spec.ts b/packages/tests/tests/utils.spec.ts index 3151a606c5..e9445a8376 100644 --- a/packages/tests/tests/utils.spec.ts +++ b/packages/tests/tests/utils.spec.ts @@ -138,7 +138,7 @@ describe.skip("getPeers function", function () { sinon.restore(); }); - it.only("should return all peers when numPeers is 0", async function () { + it("should return all peers when numPeers is 0", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); const peer3 = await createSecp256k1PeerId(); From 56d8b84a2efe793282826073e2f7476414eef921 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 11:14:15 +0530 Subject: [PATCH 41/58] move tests to `base_protocol.spec.ts` --- packages/tests/tests/base_protoco.spec.ts | 101 ++++++++++++++++++++++ packages/tests/tests/utils.spec.ts | 101 +--------------------- 2 files changed, 102 insertions(+), 100 deletions(-) create mode 100644 packages/tests/tests/base_protoco.spec.ts diff --git a/packages/tests/tests/base_protoco.spec.ts b/packages/tests/tests/base_protoco.spec.ts new file mode 100644 index 0000000000..58a6e16d97 --- /dev/null +++ b/packages/tests/tests/base_protoco.spec.ts @@ -0,0 +1,101 @@ +import { Peer } from "@libp2p/interface/peer-store"; +import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; +import { LightNode, Tags } from "@waku/interfaces"; +import { createLightNode } from "@waku/sdk"; +import * as libp2pUtils from "@waku/utils/libp2p"; +import { expect } from "chai"; +import Sinon, { SinonStub } from "sinon"; + +// these tests are skipped until we can figure out how to mock the standalone functions +// sinon doesn't seem to work with the standalone functions +// some helper utilities like proxyquire and rewire were also tried, but they don't seem to work either +// possible solution is the upgrade to jest, which has better mocking capabilities +// https://github.com/waku-org/js-waku/issues/1144 +describe.skip("getPeers function", function () { + let getPeersForProtocolStub: SinonStub; + let waku: LightNode | undefined; + + beforeEach(async function () { + waku = await createLightNode(); + getPeersForProtocolStub = Sinon.stub(libp2pUtils, "getPeersForProtocol"); + }); + + afterEach(function () { + Sinon.restore(); + }); + + it("should return all peers when numPeers is 0", async function () { + const peer1 = await createSecp256k1PeerId(); + const peer2 = await createSecp256k1PeerId(); + const peer3 = await createSecp256k1PeerId(); + + const mockPeers = [ + { id: peer1, tags: [Tags.BOOTSTRAP] }, + { id: peer2, tags: [Tags.BOOTSTRAP] }, + { id: peer3, tags: [Tags.BOOTSTRAP] } + ] as unknown as Peer[]; + + getPeersForProtocolStub.resolves(mockPeers); + + const result = await (waku?.lightPush as any).getPeers({ + numPeers: 0 + }); + expect(result).to.deep.equal(mockPeers); + }); + + it("should return all peers, except bootstrap, when numPeers is 0 & maxBootstrap is defined", async function () { + const peer1 = await createSecp256k1PeerId(); + const peer2 = await createSecp256k1PeerId(); + const peer3 = await createSecp256k1PeerId(); + const peer4 = await createSecp256k1PeerId(); + const peer5 = await createSecp256k1PeerId(); + + const mockPeers = [ + { id: peer1, tags: [Tags.BOOTSTRAP] }, + { id: peer2, tags: [Tags.BOOTSTRAP] }, + { id: peer3, tags: [Tags.PEER_EXCHANGE] }, + { id: peer4, tags: [Tags.PEER_EXCHANGE] }, + { id: peer5, tags: [Tags.PEER_EXCHANGE] } + ] as unknown as Peer[]; + + getPeersForProtocolStub.resolves(mockPeers); + + const result = await (waku?.lightPush as any).getPeers({ + numPeers: 0, + maxBootstrap: 1 + }); + + // result should have 1 bootstrap peers, and a total of 4 peers + expect(result.length).to.equal(4); + expect( + result.filter((peer: Peer) => peer.tags.has(Tags.BOOTSTRAP)).length + ).to.equal(1); + }); + + it("should return only bootstrap peers up to maxBootstrapPeers", async function () { + const peer1 = await createSecp256k1PeerId(); + const peer2 = await createSecp256k1PeerId(); + const peer3 = await createSecp256k1PeerId(); + const peer4 = await createSecp256k1PeerId(); + const peer5 = await createSecp256k1PeerId(); + const mockPeers = [ + { id: peer1, tags: [Tags.BOOTSTRAP] }, + { id: peer2, tags: [Tags.BOOTSTRAP] }, + { id: peer3, tags: [Tags.BOOTSTRAP] }, + { id: peer4, tags: [Tags.PEER_EXCHANGE] }, + { id: peer5, tags: [Tags.PEER_EXCHANGE] } + ] as unknown as Peer[]; + + getPeersForProtocolStub.resolves(mockPeers); + + const result = await (waku?.lightPush as any).getPeers({ + numPeers: 5, + maxBootstrapPeers: 2 + }); + + // check that result has at least 2 bootstrap peers and no more than 5 peers + expect(result.length).to.be.at.least(2); + expect(result.length).to.be.at.most(5); + expect(result.filter((peer: Peer) => peer.tags.has(Tags.BOOTSTRAP)).length); + }); +}); diff --git a/packages/tests/tests/utils.spec.ts b/packages/tests/tests/utils.spec.ts index e9445a8376..98a4198ffa 100644 --- a/packages/tests/tests/utils.spec.ts +++ b/packages/tests/tests/utils.spec.ts @@ -1,19 +1,14 @@ -import { Peer } from "@libp2p/interface/peer-store"; -import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; import { createDecoder, createEncoder, DefaultPubSubTopic, waitForRemotePeer } from "@waku/core"; -import { LightNode, Tags } from "@waku/interfaces"; +import { LightNode } from "@waku/interfaces"; import { Protocols } from "@waku/interfaces"; import { createLightNode } from "@waku/sdk"; import { toAsyncIterator } from "@waku/utils"; import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; -import * as libp2pUtils from "@waku/utils/libp2p"; -import { expect } from "chai"; -import sinon, { SinonStub } from "sinon"; import { makeLogFileName, NOISE_KEY_1 } from "../src/index.js"; import { NimGoNode } from "../src/node/node.js"; @@ -119,97 +114,3 @@ describe("Util: toAsyncIterator: Filter", () => { expect(result.done).to.eq(true); }); }); - -// these tests are skipped until we can figure out how to mock the standalone functions -// sinon doesn't seem to work with the standalone functions -// some helper utilities like proxyquire and rewire were also tried, but they don't seem to work either -// possible solution is the upgrade to jest, which has better mocking capabilities -// https://github.com/waku-org/js-waku/issues/1144 -describe.skip("getPeers function", function () { - let getPeersForProtocolStub: SinonStub; - let waku: LightNode | undefined; - - beforeEach(async function () { - waku = await createLightNode(); - getPeersForProtocolStub = sinon.stub(libp2pUtils, "getPeersForProtocol"); - }); - - afterEach(function () { - sinon.restore(); - }); - - it("should return all peers when numPeers is 0", async function () { - const peer1 = await createSecp256k1PeerId(); - const peer2 = await createSecp256k1PeerId(); - const peer3 = await createSecp256k1PeerId(); - - const mockPeers = [ - { id: peer1, tags: [Tags.BOOTSTRAP] }, - { id: peer2, tags: [Tags.BOOTSTRAP] }, - { id: peer3, tags: [Tags.BOOTSTRAP] } - ] as unknown as Peer[]; - - getPeersForProtocolStub.resolves(mockPeers); - - const result = await (waku?.lightPush as any).getPeers({ - numPeers: 0 - }); - expect(result).to.deep.equal(mockPeers); - }); - - it("should return all peers, except bootstrap, when numPeers is 0 & maxBootstrap is defined", async function () { - const peer1 = await createSecp256k1PeerId(); - const peer2 = await createSecp256k1PeerId(); - const peer3 = await createSecp256k1PeerId(); - const peer4 = await createSecp256k1PeerId(); - const peer5 = await createSecp256k1PeerId(); - - const mockPeers = [ - { id: peer1, tags: [Tags.BOOTSTRAP] }, - { id: peer2, tags: [Tags.BOOTSTRAP] }, - { id: peer3, tags: [Tags.PEER_EXCHANGE] }, - { id: peer4, tags: [Tags.PEER_EXCHANGE] }, - { id: peer5, tags: [Tags.PEER_EXCHANGE] } - ] as unknown as Peer[]; - - getPeersForProtocolStub.resolves(mockPeers); - - const result = await (waku?.lightPush as any).getPeers({ - numPeers: 0, - maxBootstrap: 1 - }); - - // result should have 1 bootstrap peers, and a total of 4 peers - expect(result.length).to.equal(4); - expect( - result.filter((peer: Peer) => peer.tags.has(Tags.BOOTSTRAP)).length - ).to.equal(1); - }); - - it("should return only bootstrap peers up to maxBootstrapPeers", async function () { - const peer1 = await createSecp256k1PeerId(); - const peer2 = await createSecp256k1PeerId(); - const peer3 = await createSecp256k1PeerId(); - const peer4 = await createSecp256k1PeerId(); - const peer5 = await createSecp256k1PeerId(); - const mockPeers = [ - { id: peer1, tags: [Tags.BOOTSTRAP] }, - { id: peer2, tags: [Tags.BOOTSTRAP] }, - { id: peer3, tags: [Tags.BOOTSTRAP] }, - { id: peer4, tags: [Tags.PEER_EXCHANGE] }, - { id: peer5, tags: [Tags.PEER_EXCHANGE] } - ] as unknown as Peer[]; - - getPeersForProtocolStub.resolves(mockPeers); - - const result = await (waku?.lightPush as any).getPeers({ - numPeers: 5, - maxBootstrapPeers: 2 - }); - - // check that result has at least 2 bootstrap peers and no more than 5 peers - expect(result.length).to.be.at.least(2); - expect(result.length).to.be.at.most(5); - expect(result.filter((peer: Peer) => peer.tags.has(Tags.BOOTSTRAP)).length); - }); -}); From dcb4fbff478ff703415627af0abbd424cbcd40ac Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 11:14:46 +0530 Subject: [PATCH 42/58] break down `getPeers` into a `filter` method --- packages/core/src/lib/base_protocol.ts | 55 +++++++++++++++++--------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 7de4a87421..5952d91e9d 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -69,50 +69,67 @@ export class BaseProtocol implements IBaseProtocol { /** * Retrieves a list of peers based on the specified criteria. * + * @param peers - The list of peers to filter from. * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned. * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. * @returns A Promise that resolves to an array of peers based on the specified criteria. */ - protected async getPeers({ - numPeers, - maxBootstrapPeers - }: { - numPeers: number; - maxBootstrapPeers: number; - }): Promise { - // Retrieve all peers that support the protocol - const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ - this.multicodec - ]); - + private async filterPeers( + peers: Peer[], + numPeers: number, + maxBootstrapPeers: number + ): Promise { // Collect the bootstrap peers up to the specified maximum - const bootstrapPeers = allPeersForProtocol + const bootstrapPeers = peers .filter((peer) => peer.tags.has(Tags.BOOTSTRAP)) .slice(0, maxBootstrapPeers); // Collect non-bootstrap peers - const remainingPeers = allPeersForProtocol.filter( - (peer) => !bootstrapPeers.includes(peer) + const nonBootstrapPeers = peers.filter( + (peer) => !peer.tags.has(Tags.BOOTSTRAP) ); // If numPeers is 0, return all peers if (numPeers === 0) { - return [...bootstrapPeers, ...remainingPeers]; + return [...bootstrapPeers, ...nonBootstrapPeers]; } // Initialize the list of selected peers with the bootstrap peers const selectedPeers: Peer[] = [...bootstrapPeers]; // Fill up to numPeers with remaining random peers if needed - while (selectedPeers.length < numPeers && remainingPeers.length > 0) { - const randomIndex = Math.floor(Math.random() * remainingPeers.length); - const randomPeer = remainingPeers.splice(randomIndex, 1)[0]; + while (selectedPeers.length < numPeers && nonBootstrapPeers.length > 0) { + const randomIndex = Math.floor(Math.random() * nonBootstrapPeers.length); + const randomPeer = nonBootstrapPeers.splice(randomIndex, 1)[0]; selectedPeers.push(randomPeer); } return selectedPeers; } + /** + * Retrieves a list of peers based on the specified criteria. + * + * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned. + * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. + * @returns A Promise that resolves to an array of peers based on the specified criteria. + */ + protected async getPeers({ + numPeers, + maxBootstrapPeers + }: { + numPeers: number; + maxBootstrapPeers: number; + }): Promise { + // Retrieve all peers that support the protocol + const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ + this.multicodec + ]); + + // Filter the peers based on the specified criteria + return this.filterPeers(allPeersForProtocol, numPeers, maxBootstrapPeers); + } + protected async newStream(peer: Peer): Promise { const connections = this.components.connectionManager.getConnections( peer.id From 2cd8fa0a63ab8ff0758207635e909f5ff91183cf Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 11:33:16 +0530 Subject: [PATCH 43/58] return all bootstrap peers if arg is 0 --- packages/core/src/lib/base_protocol.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 5952d91e9d..127d16517f 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -80,9 +80,10 @@ export class BaseProtocol implements IBaseProtocol { maxBootstrapPeers: number ): Promise { // Collect the bootstrap peers up to the specified maximum - const bootstrapPeers = peers - .filter((peer) => peer.tags.has(Tags.BOOTSTRAP)) - .slice(0, maxBootstrapPeers); + let bootstrapPeers = peers.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)); + if (maxBootstrapPeers > 0) { + bootstrapPeers = bootstrapPeers.slice(0, maxBootstrapPeers); + } // Collect non-bootstrap peers const nonBootstrapPeers = peers.filter( From ad518afb50642bce9f4000c5f718b308c7c4e955 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 11:33:48 +0530 Subject: [PATCH 44/58] refactor test without stubbing --- packages/tests/tests/base_protoco.spec.ts | 104 ++++++++++++---------- packages/tests/tests/utils.spec.ts | 1 + 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/packages/tests/tests/base_protoco.spec.ts b/packages/tests/tests/base_protoco.spec.ts index 58a6e16d97..69177d8808 100644 --- a/packages/tests/tests/base_protoco.spec.ts +++ b/packages/tests/tests/base_protoco.spec.ts @@ -1,27 +1,15 @@ import { Peer } from "@libp2p/interface/peer-store"; +import type { Tag } from "@libp2p/interface/peer-store"; import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; import { LightNode, Tags } from "@waku/interfaces"; import { createLightNode } from "@waku/sdk"; -import * as libp2pUtils from "@waku/utils/libp2p"; import { expect } from "chai"; -import Sinon, { SinonStub } from "sinon"; -// these tests are skipped until we can figure out how to mock the standalone functions -// sinon doesn't seem to work with the standalone functions -// some helper utilities like proxyquire and rewire were also tried, but they don't seem to work either -// possible solution is the upgrade to jest, which has better mocking capabilities -// https://github.com/waku-org/js-waku/issues/1144 -describe.skip("getPeers function", function () { - let getPeersForProtocolStub: SinonStub; +describe("getPeers function", function () { let waku: LightNode | undefined; beforeEach(async function () { waku = await createLightNode(); - getPeersForProtocolStub = Sinon.stub(libp2pUtils, "getPeersForProtocol"); - }); - - afterEach(function () { - Sinon.restore(); }); it("should return all peers when numPeers is 0", async function () { @@ -30,17 +18,22 @@ describe.skip("getPeers function", function () { const peer3 = await createSecp256k1PeerId(); const mockPeers = [ - { id: peer1, tags: [Tags.BOOTSTRAP] }, - { id: peer2, tags: [Tags.BOOTSTRAP] }, - { id: peer3, tags: [Tags.BOOTSTRAP] } + { + id: peer1, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer2, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer3, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + } ] as unknown as Peer[]; - getPeersForProtocolStub.resolves(mockPeers); - - const result = await (waku?.lightPush as any).getPeers({ - numPeers: 0 - }); - expect(result).to.deep.equal(mockPeers); + const result = await (waku?.lightPush as any).filterPeers(mockPeers, 0, 0); + expect(result.length).to.deep.equal(mockPeers.length); }); it("should return all peers, except bootstrap, when numPeers is 0 & maxBootstrap is defined", async function () { @@ -51,19 +44,29 @@ describe.skip("getPeers function", function () { const peer5 = await createSecp256k1PeerId(); const mockPeers = [ - { id: peer1, tags: [Tags.BOOTSTRAP] }, - { id: peer2, tags: [Tags.BOOTSTRAP] }, - { id: peer3, tags: [Tags.PEER_EXCHANGE] }, - { id: peer4, tags: [Tags.PEER_EXCHANGE] }, - { id: peer5, tags: [Tags.PEER_EXCHANGE] } + { + id: peer1, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer2, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer3, + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) + }, + { + id: peer4, + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) + }, + { + id: peer5, + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) + } ] as unknown as Peer[]; - getPeersForProtocolStub.resolves(mockPeers); - - const result = await (waku?.lightPush as any).getPeers({ - numPeers: 0, - maxBootstrap: 1 - }); + const result = await (waku?.lightPush as any).filterPeers(mockPeers, 0, 1); // result should have 1 bootstrap peers, and a total of 4 peers expect(result.length).to.equal(4); @@ -78,20 +81,31 @@ describe.skip("getPeers function", function () { const peer3 = await createSecp256k1PeerId(); const peer4 = await createSecp256k1PeerId(); const peer5 = await createSecp256k1PeerId(); + const mockPeers = [ - { id: peer1, tags: [Tags.BOOTSTRAP] }, - { id: peer2, tags: [Tags.BOOTSTRAP] }, - { id: peer3, tags: [Tags.BOOTSTRAP] }, - { id: peer4, tags: [Tags.PEER_EXCHANGE] }, - { id: peer5, tags: [Tags.PEER_EXCHANGE] } + { + id: peer1, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer2, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer3, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer4, + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) + }, + { + id: peer5, + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) + } ] as unknown as Peer[]; - getPeersForProtocolStub.resolves(mockPeers); - - const result = await (waku?.lightPush as any).getPeers({ - numPeers: 5, - maxBootstrapPeers: 2 - }); + const result = await (waku?.lightPush as any).filterPeers(mockPeers, 5, 2); // check that result has at least 2 bootstrap peers and no more than 5 peers expect(result.length).to.be.at.least(2); diff --git a/packages/tests/tests/utils.spec.ts b/packages/tests/tests/utils.spec.ts index 98a4198ffa..25c81cc730 100644 --- a/packages/tests/tests/utils.spec.ts +++ b/packages/tests/tests/utils.spec.ts @@ -9,6 +9,7 @@ import { Protocols } from "@waku/interfaces"; import { createLightNode } from "@waku/sdk"; import { toAsyncIterator } from "@waku/utils"; import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes"; +import { expect } from "chai"; import { makeLogFileName, NOISE_KEY_1 } from "../src/index.js"; import { NimGoNode } from "../src/node/node.js"; From f24e8c08431d8dd171b6ce9292eb67ec4cb027e2 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 11:55:50 +0530 Subject: [PATCH 45/58] address comments --- .../src/lib/base_protocol.spec.ts} | 16 ++---- packages/core/src/lib/base_protocol.ts | 46 +---------------- packages/utils/src/common/index.ts | 49 +++++++++++++++++-- 3 files changed, 53 insertions(+), 58 deletions(-) rename packages/{tests/tests/base_protoco.spec.ts => core/src/lib/base_protocol.spec.ts} (87%) diff --git a/packages/tests/tests/base_protoco.spec.ts b/packages/core/src/lib/base_protocol.spec.ts similarity index 87% rename from packages/tests/tests/base_protoco.spec.ts rename to packages/core/src/lib/base_protocol.spec.ts index 69177d8808..98c3f08b09 100644 --- a/packages/tests/tests/base_protoco.spec.ts +++ b/packages/core/src/lib/base_protocol.spec.ts @@ -1,17 +1,11 @@ import { Peer } from "@libp2p/interface/peer-store"; import type { Tag } from "@libp2p/interface/peer-store"; import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; -import { LightNode, Tags } from "@waku/interfaces"; -import { createLightNode } from "@waku/sdk"; +import { Tags } from "@waku/interfaces"; +import { filterPeers } from "@waku/utils/common"; import { expect } from "chai"; describe("getPeers function", function () { - let waku: LightNode | undefined; - - beforeEach(async function () { - waku = await createLightNode(); - }); - it("should return all peers when numPeers is 0", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); @@ -32,7 +26,7 @@ describe("getPeers function", function () { } ] as unknown as Peer[]; - const result = await (waku?.lightPush as any).filterPeers(mockPeers, 0, 0); + const result = await filterPeers(mockPeers, 0, 0); expect(result.length).to.deep.equal(mockPeers.length); }); @@ -66,7 +60,7 @@ describe("getPeers function", function () { } ] as unknown as Peer[]; - const result = await (waku?.lightPush as any).filterPeers(mockPeers, 0, 1); + const result = await filterPeers(mockPeers, 0, 1); // result should have 1 bootstrap peers, and a total of 4 peers expect(result.length).to.equal(4); @@ -105,7 +99,7 @@ describe("getPeers function", function () { } ] as unknown as Peer[]; - const result = await (waku?.lightPush as any).filterPeers(mockPeers, 5, 2); + const result = await filterPeers(mockPeers, 5, 2); // check that result has at least 2 bootstrap peers and no more than 5 peers expect(result.length).to.be.at.least(2); diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 127d16517f..a06c30357e 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -3,7 +3,7 @@ import type { Stream } from "@libp2p/interface/connection"; import type { PeerId } from "@libp2p/interface/peer-id"; import { Peer, PeerStore } from "@libp2p/interface/peer-store"; import type { IBaseProtocol, Libp2pComponents } from "@waku/interfaces"; -import { Tags } from "@waku/interfaces"; +import { filterPeers } from "@waku/utils"; import { getPeersForProtocol, selectConnection, @@ -66,48 +66,6 @@ export class BaseProtocol implements IBaseProtocol { return peer; } - /** - * Retrieves a list of peers based on the specified criteria. - * - * @param peers - The list of peers to filter from. - * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned. - * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. - * @returns A Promise that resolves to an array of peers based on the specified criteria. - */ - private async filterPeers( - peers: Peer[], - numPeers: number, - maxBootstrapPeers: number - ): Promise { - // Collect the bootstrap peers up to the specified maximum - let bootstrapPeers = peers.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)); - if (maxBootstrapPeers > 0) { - bootstrapPeers = bootstrapPeers.slice(0, maxBootstrapPeers); - } - - // Collect non-bootstrap peers - const nonBootstrapPeers = peers.filter( - (peer) => !peer.tags.has(Tags.BOOTSTRAP) - ); - - // If numPeers is 0, return all peers - if (numPeers === 0) { - return [...bootstrapPeers, ...nonBootstrapPeers]; - } - - // Initialize the list of selected peers with the bootstrap peers - const selectedPeers: Peer[] = [...bootstrapPeers]; - - // Fill up to numPeers with remaining random peers if needed - while (selectedPeers.length < numPeers && nonBootstrapPeers.length > 0) { - const randomIndex = Math.floor(Math.random() * nonBootstrapPeers.length); - const randomPeer = nonBootstrapPeers.splice(randomIndex, 1)[0]; - selectedPeers.push(randomPeer); - } - - return selectedPeers; - } - /** * Retrieves a list of peers based on the specified criteria. * @@ -128,7 +86,7 @@ export class BaseProtocol implements IBaseProtocol { ]); // Filter the peers based on the specified criteria - return this.filterPeers(allPeersForProtocol, numPeers, maxBootstrapPeers); + return filterPeers(allPeersForProtocol, numPeers, maxBootstrapPeers); } protected async newStream(peer: Peer): Promise { diff --git a/packages/utils/src/common/index.ts b/packages/utils/src/common/index.ts index 5429451207..9ddb33f0f4 100644 --- a/packages/utils/src/common/index.ts +++ b/packages/utils/src/common/index.ts @@ -1,5 +1,10 @@ export * from "./is_defined.js"; export * from "./random_subset.js"; +export * from "./group_by.js"; +export * from "./to_async_iterator.js"; +export * from "./is_size_valid.js"; +import { Peer } from "@libp2p/interface/peer-store"; +import { Tags } from "@waku/interfaces"; export function removeItemFromArray(arr: unknown[], value: unknown): unknown[] { const index = arr.indexOf(value); @@ -8,6 +13,44 @@ export function removeItemFromArray(arr: unknown[], value: unknown): unknown[] { } return arr; } -export * from "./group_by.js"; -export * from "./to_async_iterator.js"; -export * from "./is_size_valid.js"; +/** + * Retrieves a list of peers based on the specified criteria. + * + * @param peers - The list of peers to filter from. + * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned. + * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. + * @returns A Promise that resolves to an array of peers based on the specified criteria. + */ +export async function filterPeers( + peers: Peer[], + numPeers: number, + maxBootstrapPeers: number +): Promise { + // Collect the bootstrap peers up to the specified maximum + let bootstrapPeers = peers.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)); + if (maxBootstrapPeers > 0) { + bootstrapPeers = bootstrapPeers.slice(0, maxBootstrapPeers); + } + + // Collect non-bootstrap peers + const nonBootstrapPeers = peers.filter( + (peer) => !peer.tags.has(Tags.BOOTSTRAP) + ); + + // If numPeers is 0, return all peers + if (numPeers === 0) { + return [...bootstrapPeers, ...nonBootstrapPeers]; + } + + // Initialize the list of selected peers with the bootstrap peers + const selectedPeers: Peer[] = [...bootstrapPeers]; + + // Fill up to numPeers with remaining random peers if needed + while (selectedPeers.length < numPeers && nonBootstrapPeers.length > 0) { + const randomIndex = Math.floor(Math.random() * nonBootstrapPeers.length); + const randomPeer = nonBootstrapPeers.splice(randomIndex, 1)[0]; + selectedPeers.push(randomPeer); + } + + return selectedPeers; +} From 7354b0921358fbf0c5d50b359a70f798496e1b5c Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 11:58:13 +0530 Subject: [PATCH 46/58] update test title --- packages/core/src/lib/base_protocol.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/lib/base_protocol.spec.ts b/packages/core/src/lib/base_protocol.spec.ts index 98c3f08b09..c8639cc4c9 100644 --- a/packages/core/src/lib/base_protocol.spec.ts +++ b/packages/core/src/lib/base_protocol.spec.ts @@ -30,7 +30,7 @@ describe("getPeers function", function () { expect(result.length).to.deep.equal(mockPeers.length); }); - it("should return all peers, except bootstrap, when numPeers is 0 & maxBootstrap is defined", async function () { + it("should return one bootstrap peer, and all non-boostrap peers, when numPeers is 0 & maxBootstrap is defined", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); const peer3 = await createSecp256k1PeerId(); From c1d2e1ab9172704610c7646daa40c3872997effa Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 11:59:23 +0530 Subject: [PATCH 47/58] move `filterPeers` to a separate file --- packages/utils/src/common/filterPeers.ts | 44 ++++++++++++++++++++++++ packages/utils/src/common/index.ts | 44 +----------------------- 2 files changed, 45 insertions(+), 43 deletions(-) create mode 100644 packages/utils/src/common/filterPeers.ts diff --git a/packages/utils/src/common/filterPeers.ts b/packages/utils/src/common/filterPeers.ts new file mode 100644 index 0000000000..3823a29b73 --- /dev/null +++ b/packages/utils/src/common/filterPeers.ts @@ -0,0 +1,44 @@ +import { Peer } from "@libp2p/interface/peer-store"; +import { Tags } from "@waku/interfaces"; + +/** + * Retrieves a list of peers based on the specified criteria. + * + * @param peers - The list of peers to filter from. + * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned. + * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. + * @returns A Promise that resolves to an array of peers based on the specified criteria. + */ +export async function filterPeers( + peers: Peer[], + numPeers: number, + maxBootstrapPeers: number +): Promise { + // Collect the bootstrap peers up to the specified maximum + let bootstrapPeers = peers.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)); + if (maxBootstrapPeers > 0) { + bootstrapPeers = bootstrapPeers.slice(0, maxBootstrapPeers); + } + + // Collect non-bootstrap peers + const nonBootstrapPeers = peers.filter( + (peer) => !peer.tags.has(Tags.BOOTSTRAP) + ); + + // If numPeers is 0, return all peers + if (numPeers === 0) { + return [...bootstrapPeers, ...nonBootstrapPeers]; + } + + // Initialize the list of selected peers with the bootstrap peers + const selectedPeers: Peer[] = [...bootstrapPeers]; + + // Fill up to numPeers with remaining random peers if needed + while (selectedPeers.length < numPeers && nonBootstrapPeers.length > 0) { + const randomIndex = Math.floor(Math.random() * nonBootstrapPeers.length); + const randomPeer = nonBootstrapPeers.splice(randomIndex, 1)[0]; + selectedPeers.push(randomPeer); + } + + return selectedPeers; +} diff --git a/packages/utils/src/common/index.ts b/packages/utils/src/common/index.ts index 9ddb33f0f4..b9e8188e1c 100644 --- a/packages/utils/src/common/index.ts +++ b/packages/utils/src/common/index.ts @@ -3,8 +3,7 @@ export * from "./random_subset.js"; export * from "./group_by.js"; export * from "./to_async_iterator.js"; export * from "./is_size_valid.js"; -import { Peer } from "@libp2p/interface/peer-store"; -import { Tags } from "@waku/interfaces"; +export * from "./filterPeers.js"; export function removeItemFromArray(arr: unknown[], value: unknown): unknown[] { const index = arr.indexOf(value); @@ -13,44 +12,3 @@ export function removeItemFromArray(arr: unknown[], value: unknown): unknown[] { } return arr; } -/** - * Retrieves a list of peers based on the specified criteria. - * - * @param peers - The list of peers to filter from. - * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned. - * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. - * @returns A Promise that resolves to an array of peers based on the specified criteria. - */ -export async function filterPeers( - peers: Peer[], - numPeers: number, - maxBootstrapPeers: number -): Promise { - // Collect the bootstrap peers up to the specified maximum - let bootstrapPeers = peers.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)); - if (maxBootstrapPeers > 0) { - bootstrapPeers = bootstrapPeers.slice(0, maxBootstrapPeers); - } - - // Collect non-bootstrap peers - const nonBootstrapPeers = peers.filter( - (peer) => !peer.tags.has(Tags.BOOTSTRAP) - ); - - // If numPeers is 0, return all peers - if (numPeers === 0) { - return [...bootstrapPeers, ...nonBootstrapPeers]; - } - - // Initialize the list of selected peers with the bootstrap peers - const selectedPeers: Peer[] = [...bootstrapPeers]; - - // Fill up to numPeers with remaining random peers if needed - while (selectedPeers.length < numPeers && nonBootstrapPeers.length > 0) { - const randomIndex = Math.floor(Math.random() * nonBootstrapPeers.length); - const randomPeer = nonBootstrapPeers.splice(randomIndex, 1)[0]; - selectedPeers.push(randomPeer); - } - - return selectedPeers; -} From 919f10b306cba8935bc9d9a8dc8cc6d8e454cdb7 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 12:03:20 +0530 Subject: [PATCH 48/58] address comments & add more test --- packages/core/src/lib/base_protocol.spec.ts | 40 +++++++++++++++++++-- packages/utils/src/common/filterPeers.ts | 7 ++-- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/packages/core/src/lib/base_protocol.spec.ts b/packages/core/src/lib/base_protocol.spec.ts index c8639cc4c9..449440a2cb 100644 --- a/packages/core/src/lib/base_protocol.spec.ts +++ b/packages/core/src/lib/base_protocol.spec.ts @@ -2,10 +2,10 @@ import { Peer } from "@libp2p/interface/peer-store"; import type { Tag } from "@libp2p/interface/peer-store"; import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; import { Tags } from "@waku/interfaces"; -import { filterPeers } from "@waku/utils/common"; +import { filterPeers } from "@waku/utils"; import { expect } from "chai"; -describe("getPeers function", function () { +describe.only("getPeers function", function () { it("should return all peers when numPeers is 0", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); @@ -26,10 +26,44 @@ describe("getPeers function", function () { } ] as unknown as Peer[]; - const result = await filterPeers(mockPeers, 0, 0); + const result = await filterPeers(mockPeers, 0, 10); expect(result.length).to.deep.equal(mockPeers.length); }); + it("should return all non-bootstrap peers and no bootstrap peer when numPeers is 0 and maxBootstrapPeers is 0", async function () { + const peer1 = await createSecp256k1PeerId(); + const peer2 = await createSecp256k1PeerId(); + const peer3 = await createSecp256k1PeerId(); + const peer4 = await createSecp256k1PeerId(); + + const mockPeers = [ + { + id: peer1, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer2, + tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + }, + { + id: peer3, + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) + }, + { + id: peer4, + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) + } + ] as unknown as Peer[]; + + const result = await filterPeers(mockPeers, 0, 0); + + // result should have no bootstrap peers, and a total of 2 peers + expect(result.length).to.equal(2); + expect( + result.filter((peer: Peer) => peer.tags.has(Tags.BOOTSTRAP)).length + ).to.equal(0); + }); + it("should return one bootstrap peer, and all non-boostrap peers, when numPeers is 0 & maxBootstrap is defined", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); diff --git a/packages/utils/src/common/filterPeers.ts b/packages/utils/src/common/filterPeers.ts index 3823a29b73..298c9f15bc 100644 --- a/packages/utils/src/common/filterPeers.ts +++ b/packages/utils/src/common/filterPeers.ts @@ -15,10 +15,9 @@ export async function filterPeers( maxBootstrapPeers: number ): Promise { // Collect the bootstrap peers up to the specified maximum - let bootstrapPeers = peers.filter((peer) => peer.tags.has(Tags.BOOTSTRAP)); - if (maxBootstrapPeers > 0) { - bootstrapPeers = bootstrapPeers.slice(0, maxBootstrapPeers); - } + const bootstrapPeers = peers + .filter((peer) => peer.tags.has(Tags.BOOTSTRAP)) + .slice(0, maxBootstrapPeers); // Collect non-bootstrap peers const nonBootstrapPeers = peers.filter( From 6a1a4ad9b958726d3030338adb1d9dd21a0c0a6c Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 12:06:07 +0530 Subject: [PATCH 49/58] make test title more verbose --- packages/core/src/lib/base_protocol.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/lib/base_protocol.spec.ts b/packages/core/src/lib/base_protocol.spec.ts index 449440a2cb..7c195a98d5 100644 --- a/packages/core/src/lib/base_protocol.spec.ts +++ b/packages/core/src/lib/base_protocol.spec.ts @@ -64,7 +64,7 @@ describe.only("getPeers function", function () { ).to.equal(0); }); - it("should return one bootstrap peer, and all non-boostrap peers, when numPeers is 0 & maxBootstrap is defined", async function () { + it("should return one bootstrap peer, and all non-boostrap peers, when numPeers is 0 & maxBootstrap is 1", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); const peer3 = await createSecp256k1PeerId(); From 0f629cb5cb896596081f3244cea1a1975194f52b Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 15:33:26 +0530 Subject: [PATCH 50/58] address comments --- packages/core/src/lib/base_protocol.spec.ts | 2 +- packages/core/src/lib/base_protocol.ts | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/packages/core/src/lib/base_protocol.spec.ts b/packages/core/src/lib/base_protocol.spec.ts index 7c195a98d5..961b83d9d5 100644 --- a/packages/core/src/lib/base_protocol.spec.ts +++ b/packages/core/src/lib/base_protocol.spec.ts @@ -5,7 +5,7 @@ import { Tags } from "@waku/interfaces"; import { filterPeers } from "@waku/utils"; import { expect } from "chai"; -describe.only("getPeers function", function () { +describe("getPeers function", function () { it("should return all peers when numPeers is 0", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index a06c30357e..6bd291f4a7 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -4,11 +4,7 @@ import type { PeerId } from "@libp2p/interface/peer-id"; import { Peer, PeerStore } from "@libp2p/interface/peer-store"; import type { IBaseProtocol, Libp2pComponents } from "@waku/interfaces"; import { filterPeers } from "@waku/utils"; -import { - getPeersForProtocol, - selectConnection, - selectPeerForProtocol -} from "@waku/utils/libp2p"; +import { getPeersForProtocol, selectPeerForProtocol } from "@waku/utils/libp2p"; import { StreamManager } from "./stream_manager.js"; @@ -88,16 +84,4 @@ export class BaseProtocol implements IBaseProtocol { // Filter the peers based on the specified criteria return filterPeers(allPeersForProtocol, numPeers, maxBootstrapPeers); } - - protected async newStream(peer: Peer): Promise { - const connections = this.components.connectionManager.getConnections( - peer.id - ); - const connection = selectConnection(connections); - if (!connection) { - throw new Error("Failed to get a connection to the peer"); - } - - return connection.newStream(this.multicodec); - } } From 9302a86efd0221c5433eda304c395127fb00a727 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 17:25:48 +0530 Subject: [PATCH 51/58] remove ProtocolOptions --- packages/core/src/lib/filter/index.ts | 6 ++---- packages/interfaces/src/protocols.ts | 4 ---- packages/interfaces/src/receiver.ts | 8 +++----- packages/interfaces/src/sender.ts | 8 ++------ packages/interfaces/src/store.ts | 4 ++-- packages/relay/src/index.ts | 6 ++---- packages/utils/src/common/to_async_iterator.ts | 12 +++--------- 7 files changed, 14 insertions(+), 34 deletions(-) diff --git a/packages/core/src/lib/filter/index.ts b/packages/core/src/lib/filter/index.ts index 912b44b91d..9f9decbaf4 100644 --- a/packages/core/src/lib/filter/index.ts +++ b/packages/core/src/lib/filter/index.ts @@ -13,7 +13,6 @@ import type { Libp2p, PeerIdStr, ProtocolCreateOptions, - ProtocolOptions, PubSubTopic, Unsubscribe } from "@waku/interfaces"; @@ -280,10 +279,9 @@ class Filter extends BaseProtocol implements IReceiver { } public toSubscriptionIterator( - decoders: IDecoder | IDecoder[], - opts?: ProtocolOptions | undefined + decoders: IDecoder | IDecoder[] ): Promise> { - return toAsyncIterator(this, decoders, opts); + return toAsyncIterator(this, decoders); } /** diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts index a8ddc00b91..c354b1b740 100644 --- a/packages/interfaces/src/protocols.ts +++ b/packages/interfaces/src/protocols.ts @@ -54,10 +54,6 @@ export type ProtocolCreateOptions = { defaultBootstrap?: boolean; }; -export type ProtocolOptions = { - // -}; - export type Callback = ( msg: T ) => void | Promise; diff --git a/packages/interfaces/src/receiver.ts b/packages/interfaces/src/receiver.ts index 64cd4f810a..9dbcc2fde4 100644 --- a/packages/interfaces/src/receiver.ts +++ b/packages/interfaces/src/receiver.ts @@ -1,6 +1,6 @@ import type { IDecodedMessage, IDecoder } from "./message.js"; import type { IAsyncIterator, PubSubTopic, Unsubscribe } from "./misc.js"; -import type { Callback, ProtocolOptions } from "./protocols.js"; +import type { Callback } from "./protocols.js"; type ContentTopic = string; @@ -8,12 +8,10 @@ export type ActiveSubscriptions = Map; export interface IReceiver { toSubscriptionIterator: ( - decoders: IDecoder | IDecoder[], - opts?: ProtocolOptions + decoders: IDecoder | IDecoder[] ) => Promise>; subscribe: ( decoders: IDecoder | IDecoder[], - callback: Callback, - opts?: ProtocolOptions + callback: Callback ) => Unsubscribe | Promise; } diff --git a/packages/interfaces/src/sender.ts b/packages/interfaces/src/sender.ts index efdc2f4531..ab7e6d1b8c 100644 --- a/packages/interfaces/src/sender.ts +++ b/packages/interfaces/src/sender.ts @@ -1,10 +1,6 @@ import type { IEncoder, IMessage } from "./message.js"; -import type { ProtocolOptions, SendResult } from "./protocols.js"; +import type { SendResult } from "./protocols.js"; export interface ISender { - send: ( - encoder: IEncoder, - message: IMessage, - opts?: ProtocolOptions - ) => Promise; + send: (encoder: IEncoder, message: IMessage) => Promise; } diff --git a/packages/interfaces/src/store.ts b/packages/interfaces/src/store.ts index aea825b3b2..9127d35d4d 100644 --- a/packages/interfaces/src/store.ts +++ b/packages/interfaces/src/store.ts @@ -1,5 +1,5 @@ import type { IDecodedMessage, IDecoder } from "./message.js"; -import type { IBaseProtocol, ProtocolOptions } from "./protocols.js"; +import type { IBaseProtocol } from "./protocols.js"; export enum PageDirection { BACKWARD = "backward", @@ -43,7 +43,7 @@ export type StoreQueryOptions = { * Message. */ cursor?: Cursor; -} & ProtocolOptions; +}; export interface IStore extends IBaseProtocol { queryWithOrderedCallback: ( diff --git a/packages/relay/src/index.ts b/packages/relay/src/index.ts index 52a57823ad..20dc7d2f50 100644 --- a/packages/relay/src/index.ts +++ b/packages/relay/src/index.ts @@ -21,7 +21,6 @@ import { IRelay, Libp2p, ProtocolCreateOptions, - ProtocolOptions, SendError, SendResult } from "@waku/interfaces"; @@ -162,10 +161,9 @@ class Relay implements IRelay { } public toSubscriptionIterator( - decoders: IDecoder | IDecoder[], - opts?: ProtocolOptions | undefined + decoders: IDecoder | IDecoder[] ): Promise> { - return toAsyncIterator(this, decoders, opts); + return toAsyncIterator(this, decoders); } public getActiveSubscriptions(): ActiveSubscriptions { diff --git a/packages/utils/src/common/to_async_iterator.ts b/packages/utils/src/common/to_async_iterator.ts index 3dd9df8d51..489ef234ef 100644 --- a/packages/utils/src/common/to_async_iterator.ts +++ b/packages/utils/src/common/to_async_iterator.ts @@ -3,7 +3,6 @@ import type { IDecodedMessage, IDecoder, IReceiver, - ProtocolOptions, Unsubscribe } from "@waku/interfaces"; @@ -31,7 +30,6 @@ const FRAME_RATE = 60; export async function toAsyncIterator( receiver: IReceiver, decoder: IDecoder | IDecoder[], - options?: ProtocolOptions, iteratorOptions?: IteratorOptions ): Promise> { const iteratorDelay = iteratorOptions?.iteratorDelay ?? FRAME_RATE; @@ -39,13 +37,9 @@ export async function toAsyncIterator( const messages: T[] = []; let unsubscribe: undefined | Unsubscribe; - unsubscribe = await receiver.subscribe( - decoder, - (message: T) => { - messages.push(message); - }, - options - ); + unsubscribe = await receiver.subscribe(decoder, (message: T) => { + messages.push(message); + }); const isWithTimeout = Number.isInteger(iteratorOptions?.timeoutMs); const timeoutMs = iteratorOptions?.timeoutMs ?? 0; From 053c9da1fd2f458e7818fff75ef4ebd943d9680e Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 17:34:04 +0530 Subject: [PATCH 52/58] chore: refactor tests for new API --- packages/tests/tests/light_push.node.spec.ts | 10 +++----- packages/tests/tests/utils.spec.ts | 27 +++++++------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/packages/tests/tests/light_push.node.spec.ts b/packages/tests/tests/light_push.node.spec.ts index afcee53c63..5b14f78676 100644 --- a/packages/tests/tests/light_push.node.spec.ts +++ b/packages/tests/tests/light_push.node.spec.ts @@ -138,13 +138,9 @@ describe("Waku Light Push [node only] - custom pubsub topic", () => { const messageText = "Light Push works!"; log("Send message via lightpush"); - const pushResponse = await waku.lightPush.send( - TestEncoder, - { payload: utf8ToBytes(messageText) }, - { - peerId: nimPeerId - } - ); + const pushResponse = await waku.lightPush.send(TestEncoder, { + payload: utf8ToBytes(messageText) + }); log("Ack received", pushResponse); expect(pushResponse.recipients[0].toString()).to.eq(nimPeerId.toString()); diff --git a/packages/tests/tests/utils.spec.ts b/packages/tests/tests/utils.spec.ts index 25c81cc730..42935d60ec 100644 --- a/packages/tests/tests/utils.spec.ts +++ b/packages/tests/tests/utils.spec.ts @@ -49,12 +49,9 @@ describe("Util: toAsyncIterator: Filter", () => { const messageText = "hey, what's up?"; const sent = { payload: utf8ToBytes(messageText) }; - const { iterator } = await toAsyncIterator( - waku.filter, - TestDecoder, - {}, - { timeoutMs: 1000 } - ); + const { iterator } = await toAsyncIterator(waku.filter, TestDecoder, { + timeoutMs: 1000 + }); await waku.lightPush.send(TestEncoder, sent); const { value } = await iterator.next(); @@ -66,12 +63,9 @@ describe("Util: toAsyncIterator: Filter", () => { it("handles multiple messages", async function () { this.timeout(10000); - const { iterator } = await toAsyncIterator( - waku.filter, - TestDecoder, - {}, - { timeoutMs: 1000 } - ); + const { iterator } = await toAsyncIterator(waku.filter, TestDecoder, { + timeoutMs: 1000 + }); await waku.lightPush.send(TestEncoder, { payload: utf8ToBytes("Filtering works!") @@ -89,12 +83,9 @@ describe("Util: toAsyncIterator: Filter", () => { it("unsubscribes", async function () { this.timeout(10000); - const { iterator, stop } = await toAsyncIterator( - waku.filter, - TestDecoder, - {}, - { timeoutMs: 1000 } - ); + const { iterator, stop } = await toAsyncIterator(waku.filter, TestDecoder, { + timeoutMs: 1000 + }); await waku.lightPush.send(TestEncoder, { payload: utf8ToBytes("This should be received") From 5aa4fcbc2d5a9a799334a6b10ef5310f34dfbaef Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 18:02:12 +0530 Subject: [PATCH 53/58] add defaults for getPeers --- packages/core/src/lib/base_protocol.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 6bd291f4a7..43b1ea6654 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -69,13 +69,18 @@ export class BaseProtocol implements IBaseProtocol { * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve. * @returns A Promise that resolves to an array of peers based on the specified criteria. */ - protected async getPeers({ - numPeers, - maxBootstrapPeers - }: { - numPeers: number; - maxBootstrapPeers: number; - }): Promise { + protected async getPeers( + { + numPeers, + maxBootstrapPeers + }: { + numPeers: number; + maxBootstrapPeers: number; + } = { + maxBootstrapPeers: 1, + numPeers: 0 + } + ): Promise { // Retrieve all peers that support the protocol const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [ this.multicodec From afbcaaee89a3b5c72c7098b60b4d63274c395a37 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Tue, 5 Sep 2023 18:18:01 +0530 Subject: [PATCH 54/58] address comments --- packages/core/src/lib/light_push/index.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index fa55d0907c..819b19dc16 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -102,6 +102,7 @@ class LightPush extends BaseProtocol implements ILightPush { numPeers: this.NUM_PEERS_PROTOCOL }); + // This map returns only fulfilled promises const promises = peers.map(async (peer) => { let error: SendError | undefined; const stream = await this.getStream(peer); @@ -142,14 +143,15 @@ class LightPush extends BaseProtocol implements ILightPush { }); const results = await Promise.allSettled(promises); - const successfulResults = results.filter( - (result) => result.status === "fulfilled" - ) as PromiseFulfilledResult<{ - recipients: PeerId[]; - error: SendError | undefined; - }>[]; - - const errors = successfulResults + const errors = results + .filter( + ( + result + ): result is PromiseFulfilledResult<{ + recipients: PeerId[]; + error: SendError | undefined; + }> => result.status === "fulfilled" + ) .map((result) => result.value.error) .filter((error) => error !== undefined) as SendError[]; From d44670d0d1a0ade263d1c148aa60d9538c6e050e Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 6 Sep 2023 02:14:53 +0530 Subject: [PATCH 55/58] rm unneeded comment --- packages/core/src/lib/light_push/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/lib/light_push/index.ts b/packages/core/src/lib/light_push/index.ts index 819b19dc16..d9379c1690 100644 --- a/packages/core/src/lib/light_push/index.ts +++ b/packages/core/src/lib/light_push/index.ts @@ -102,7 +102,6 @@ class LightPush extends BaseProtocol implements ILightPush { numPeers: this.NUM_PEERS_PROTOCOL }); - // This map returns only fulfilled promises const promises = peers.map(async (peer) => { let error: SendError | undefined; const stream = await this.getStream(peer); From 9439bb3c149b1347b7ff8b6d31fd34e782812fc3 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 7 Sep 2023 11:59:52 +0530 Subject: [PATCH 56/58] address comment: add diversity of node tags to test --- packages/core/src/lib/base_protocol.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/lib/base_protocol.spec.ts b/packages/core/src/lib/base_protocol.spec.ts index 961b83d9d5..ac3a8ddd6f 100644 --- a/packages/core/src/lib/base_protocol.spec.ts +++ b/packages/core/src/lib/base_protocol.spec.ts @@ -22,7 +22,7 @@ describe("getPeers function", function () { }, { id: peer3, - tags: new Map([[Tags.BOOTSTRAP, { value: 100 }]]) + tags: new Map([[Tags.PEER_EXCHANGE, { value: 100 }]]) } ] as unknown as Peer[]; From 1e2a9d2064ace2aa9bb2bd62b4255c591c96454e Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 7 Sep 2023 12:52:43 +0530 Subject: [PATCH 57/58] address comments --- packages/core/src/lib/base_protocol.ts | 2 +- .../core/src/lib/{base_protocol.spec.ts => filterPeers.spec.ts} | 2 +- packages/{utils/src/common => core/src/lib}/filterPeers.ts | 0 packages/utils/src/common/index.ts | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) rename packages/core/src/lib/{base_protocol.spec.ts => filterPeers.spec.ts} (98%) rename packages/{utils/src/common => core/src/lib}/filterPeers.ts (100%) diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts index 43b1ea6654..1546fda480 100644 --- a/packages/core/src/lib/base_protocol.ts +++ b/packages/core/src/lib/base_protocol.ts @@ -3,9 +3,9 @@ import type { Stream } from "@libp2p/interface/connection"; import type { PeerId } from "@libp2p/interface/peer-id"; import { Peer, PeerStore } from "@libp2p/interface/peer-store"; import type { IBaseProtocol, Libp2pComponents } from "@waku/interfaces"; -import { filterPeers } from "@waku/utils"; import { getPeersForProtocol, selectPeerForProtocol } from "@waku/utils/libp2p"; +import { filterPeers } from "./filterPeers.js"; import { StreamManager } from "./stream_manager.js"; /** diff --git a/packages/core/src/lib/base_protocol.spec.ts b/packages/core/src/lib/filterPeers.spec.ts similarity index 98% rename from packages/core/src/lib/base_protocol.spec.ts rename to packages/core/src/lib/filterPeers.spec.ts index ac3a8ddd6f..5f83207235 100644 --- a/packages/core/src/lib/base_protocol.spec.ts +++ b/packages/core/src/lib/filterPeers.spec.ts @@ -5,7 +5,7 @@ import { Tags } from "@waku/interfaces"; import { filterPeers } from "@waku/utils"; import { expect } from "chai"; -describe("getPeers function", function () { +describe("filterPeers function", function () { it("should return all peers when numPeers is 0", async function () { const peer1 = await createSecp256k1PeerId(); const peer2 = await createSecp256k1PeerId(); diff --git a/packages/utils/src/common/filterPeers.ts b/packages/core/src/lib/filterPeers.ts similarity index 100% rename from packages/utils/src/common/filterPeers.ts rename to packages/core/src/lib/filterPeers.ts diff --git a/packages/utils/src/common/index.ts b/packages/utils/src/common/index.ts index b9e8188e1c..f73240fbeb 100644 --- a/packages/utils/src/common/index.ts +++ b/packages/utils/src/common/index.ts @@ -3,7 +3,6 @@ export * from "./random_subset.js"; export * from "./group_by.js"; export * from "./to_async_iterator.js"; export * from "./is_size_valid.js"; -export * from "./filterPeers.js"; export function removeItemFromArray(arr: unknown[], value: unknown): unknown[] { const index = arr.indexOf(value); From c90f88ba128baeef7c4d4abd7de1fb4396db5164 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 7 Sep 2023 13:03:48 +0530 Subject: [PATCH 58/58] fix: imports --- packages/core/src/lib/filterPeers.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/lib/filterPeers.spec.ts b/packages/core/src/lib/filterPeers.spec.ts index 5f83207235..de51da5593 100644 --- a/packages/core/src/lib/filterPeers.spec.ts +++ b/packages/core/src/lib/filterPeers.spec.ts @@ -2,9 +2,10 @@ import { Peer } from "@libp2p/interface/peer-store"; import type { Tag } from "@libp2p/interface/peer-store"; import { createSecp256k1PeerId } from "@libp2p/peer-id-factory"; import { Tags } from "@waku/interfaces"; -import { filterPeers } from "@waku/utils"; import { expect } from "chai"; +import { filterPeers } from "./filterPeers.js"; + describe("filterPeers function", function () { it("should return all peers when numPeers is 0", async function () { const peer1 = await createSecp256k1PeerId();