Merge branch 'master' of github.com:waku-org/js-waku into feat/shard-…

…peer-selection

.release-please-manifest.json

@@ -1,13 +1,13 @@
 {
-  "packages/utils": "0.0.13",
-  "packages/proto": "0.0.5",
-  "packages/interfaces": "0.0.20",
-  "packages/message-hash": "0.1.9",
-  "packages/enr": "0.0.19",
-  "packages/peer-exchange": "0.0.18",
-  "packages/core": "0.0.25",
-  "packages/dns-discovery": "0.0.19",
-  "packages/message-encryption": "0.0.23",
-  "packages/relay": "0.0.8",
-  "packages/sdk": "0.0.21"
+  "packages/utils": "0.0.14",
+  "packages/proto": "0.0.6",
+  "packages/interfaces": "0.0.21",
+  "packages/message-hash": "0.1.10",
+  "packages/enr": "0.0.20",
+  "packages/peer-exchange": "0.0.19",
+  "packages/core": "0.0.26",
+  "packages/dns-discovery": "0.0.20",
+  "packages/message-encryption": "0.0.24",
+  "packages/relay": "0.0.9",
+  "packages/sdk": "0.0.22"
 }

packages/core/CHANGELOG.md

@@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file.
 The file is maintained by [Release Please](https://github.com/googleapis/release-please) based on [Conventional Commits](https://www.conventionalcommits.org) specification,
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.0.26](https://github.com/waku-org/js-waku/compare/core-v0.0.25...core-v0.0.26) (2024-01-10)
+
+
+### ⚠ BREAKING CHANGES
+
+* add support for sharded pubsub topics & remove support for named pubsub topics ([#1697](https://github.com/waku-org/js-waku/issues/1697))
+* change all instances of `PubSubTopic` to `PubsubTopic` ([#1703](https://github.com/waku-org/js-waku/issues/1703))
+
+### Features
+
+* Add support for autosharded pubsub topics ([2bc3735](https://github.com/waku-org/js-waku/commit/2bc3735e4dcf85f06b3dee542024d7f20a40fac2))
+* Add support for sharded pubsub topics & remove support for named pubsub topics ([#1697](https://github.com/waku-org/js-waku/issues/1697)) ([4cf2ffe](https://github.com/waku-org/js-waku/commit/4cf2ffefa75e0571805036b71644d2cdd4fe3192))
+* Metadata protocol ([#1732](https://github.com/waku-org/js-waku/issues/1732)) ([9ac2a3f](https://github.com/waku-org/js-waku/commit/9ac2a3f36352523b79fcd8f8a94bd6e0e109fc30))
+* Track node connection state ([#1719](https://github.com/waku-org/js-waku/issues/1719)) ([1d0e2ac](https://github.com/waku-org/js-waku/commit/1d0e2ace7fa5b44ab192505c7ebce01a7ce343e0))
+
+
+### Miscellaneous Chores
+
+* Change all instances of `PubSubTopic` to `PubsubTopic` ([#1703](https://github.com/waku-org/js-waku/issues/1703)) ([3166a51](https://github.com/waku-org/js-waku/commit/3166a5135e77583da4fa722ee2aa47c785854a38))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @waku/enr bumped from ^0.0.19 to ^0.0.20
+    * @waku/interfaces bumped from 0.0.20 to 0.0.21
+    * @waku/proto bumped from 0.0.5 to 0.0.6
+    * @waku/utils bumped from 0.0.13 to 0.0.14
+
 ## [0.0.25](https://github.com/waku-org/js-waku/compare/core-v0.0.24...core-v0.0.25) (2023-11-01)
 
 

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@waku/core",
-  "version": "0.0.25",
+  "version": "0.0.26",
   "description": "TypeScript implementation of the Waku v2 protocol",
   "types": "./dist/index.d.ts",
   "module": "./dist/index.js",
@@ -73,10 +73,10 @@
   },
   "dependencies": {
     "@noble/hashes": "^1.3.2",
-    "@waku/enr": "^0.0.19",
-    "@waku/interfaces": "0.0.20",
-    "@waku/proto": "0.0.5",
-    "@waku/utils": "0.0.13",
+    "@waku/enr": "^0.0.20",
+    "@waku/interfaces": "0.0.21",
+    "@waku/proto": "0.0.6",
+    "@waku/utils": "0.0.14",
     "debug": "^4.3.4",
     "it-all": "^3.0.4",
     "it-length-prefixed": "^9.0.1",

packages/core/src/lib/base_protocol.ts

@@ -1,6 +1,5 @@
 import type { Libp2p } from "@libp2p/interface";
 import type { Stream } from "@libp2p/interface/connection";
-import type { PeerId } from "@libp2p/interface/peer-id";
 import { Peer, PeerStore } from "@libp2p/interface/peer-store";
 import type {
   IBaseProtocol,
@@ -9,14 +8,14 @@ import type {
   PubsubTopic
 } from "@waku/interfaces";
 import { DefaultPubsubTopic } from "@waku/interfaces";
-import { shardInfoToPubsubTopics } from "@waku/utils";
+import { Logger, shardInfoToPubsubTopics } from "@waku/utils";
 import {
   getConnectedPeersForProtocolAndShard,
   getPeersForProtocol,
-  selectPeerForProtocol
+  sortPeersByLatency
 } from "@waku/utils/libp2p";
 
-import { filterPeers } from "./filterPeers.js";
+import { filterPeersByDiscovery } from "./filterPeers.js";
 import { StreamManager } from "./stream_manager.js";
 
 /**
@@ -32,7 +31,8 @@ export class BaseProtocol implements IBaseProtocol {
   constructor(
     public multicodec: string,
     private components: Libp2pComponents,
-    public options?: ProtocolCreateOptions
+    private log: Logger,
+    private options?: ProtocolCreateOptions
   ) {
     this.pubsubTopics = this.initializePubsubTopic(options);
 
@@ -77,22 +77,14 @@ export class BaseProtocol implements IBaseProtocol {
     });
   }
 
-  protected async getPeer(peerId?: PeerId): Promise<Peer> {
-    const { peer } = await selectPeerForProtocol(
-      this.peerStore,
-      [this.multicodec],
-      peerId
-    );
-    return peer;
-  }
-
   /**
-   * Retrieves a list of connected peers based on the specified criteria.
+   * Retrieves a list of connected peers that support the protocol. The list is sorted by latency.
    *
    * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned.
    * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
-   * @returns A Promise that resolves to an array of peers based on the specified criteria.
-   */
+   
+  * @returns A list of peers that support the protocol sorted by latency.
+  */
   protected async getPeers(
     {
       numPeers,
@@ -110,15 +102,30 @@ export class BaseProtocol implements IBaseProtocol {
       await getConnectedPeersForProtocolAndShard(
         this.components.connectionManager.getConnections(),
         this.peerStore,
-        [this.multicodec]
+        [this.multicodec],
+        this.options?.shardInfo
       );
 
-    // Filter the peers based on the specified criteria
-    return filterPeers(
+    // Filter the peers based on discovery & number of peers requested
+    const filteredPeers = filterPeersByDiscovery(
       connectedPeersForProtocolAndShard,
       numPeers,
       maxBootstrapPeers
     );
+
+    // Sort the peers by latency
+    const sortedFilteredPeers = await sortPeersByLatency(
+      this.peerStore,
+      filteredPeers
+    );
+
+    if (sortedFilteredPeers.length === 0) {
+      this.log.warn(
+        "No peers found. Ensure you have a connection to the network."
+      );
+    }
+
+    return sortedFilteredPeers;
   }
 
   private initializePubsubTopic(

packages/core/src/lib/filter/index.ts

@@ -278,7 +278,7 @@ class Filter extends BaseProtocol implements IReceiver {
   }
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
-    super(FilterCodecs.SUBSCRIBE, libp2p.components, options);
+    super(FilterCodecs.SUBSCRIBE, libp2p.components, log, options);
 
     libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => {
       log.error("Failed to register ", FilterCodecs.PUSH, e);

packages/core/src/lib/filterPeers.spec.ts

@@ -4,9 +4,9 @@ import { createSecp256k1PeerId } from "@libp2p/peer-id-factory";
 import { Tags } from "@waku/interfaces";
 import { expect } from "chai";
 
-import { filterPeers } from "./filterPeers.js";
+import { filterPeersByDiscovery } from "./filterPeers.js";
 
-describe("filterPeers function", function () {
+describe("filterPeersByDiscovery function", function () {
   it("should return all peers when numPeers is 0", async function () {
     const peer1 = await createSecp256k1PeerId();
     const peer2 = await createSecp256k1PeerId();
@@ -27,7 +27,7 @@ describe("filterPeers function", function () {
       }
     ] as unknown as Peer[];
 
-    const result = filterPeers(mockPeers, 0, 10);
+    const result = filterPeersByDiscovery(mockPeers, 0, 10);
     expect(result.length).to.deep.equal(mockPeers.length);
   });
 
@@ -56,7 +56,7 @@ describe("filterPeers function", function () {
       }
     ] as unknown as Peer[];
 
-    const result = filterPeers(mockPeers, 0, 0);
+    const result = filterPeersByDiscovery(mockPeers, 0, 0);
 
     // result should have no bootstrap peers, and a total of 2 peers
     expect(result.length).to.equal(2);
@@ -95,7 +95,7 @@ describe("filterPeers function", function () {
       }
     ] as unknown as Peer[];
 
-    const result = filterPeers(mockPeers, 0, 1);
+    const result = filterPeersByDiscovery(mockPeers, 0, 1);
 
     // result should have 1 bootstrap peers, and a total of 4 peers
     expect(result.length).to.equal(4);
@@ -134,7 +134,7 @@ describe("filterPeers function", function () {
       }
     ] as unknown as Peer[];
 
-    const result = filterPeers(mockPeers, 5, 2);
+    const result = filterPeersByDiscovery(mockPeers, 5, 2);
 
     // check that result has at least 2 bootstrap peers and no more than 5 peers
     expect(result.length).to.be.at.least(2);

packages/core/src/lib/filterPeers.ts

@@ -2,23 +2,31 @@ import { Peer } from "@libp2p/interface/peer-store";
 import { Tags } from "@waku/interfaces";
 
 /**
- * Retrieves a list of peers based on the specified criteria.
+ * Retrieves a list of peers based on the specified criteria:
+ * 1. If numPeers is 0, return all peers
+ * 2. Bootstrap peers are prioritized
+ * 3. Non-bootstrap peers are randomly selected to fill up to numPeers
  *
  * @param peers - The list of peers to filter from.
- * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned.
+ * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned, irrespective of `maxBootstrapPeers`.
  * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
- * @returns A Promise that resolves to an array of peers based on the specified criteria.
+ * @returns An array of peers based on the specified criteria.
  */
-export function filterPeers(
+export function filterPeersByDiscovery(
   peers: Peer[],
   numPeers: number,
   maxBootstrapPeers: number
 ): Peer[] {
   // Collect the bootstrap peers up to the specified maximum
-  const bootstrapPeers = peers
+  let bootstrapPeers = peers
     .filter((peer) => peer.tags.has(Tags.BOOTSTRAP))
     .slice(0, maxBootstrapPeers);
 
+  // If numPeers is less than the number of bootstrap peers, adjust the bootstrapPeers array
+  if (numPeers > 0 && numPeers < bootstrapPeers.length) {
+    bootstrapPeers = bootstrapPeers.slice(0, numPeers);
+  }
+
   // Collect non-bootstrap peers
   const nonBootstrapPeers = peers.filter(
     (peer) => !peer.tags.has(Tags.BOOTSTRAP)

packages/core/src/lib/light_push/index.ts

@@ -46,7 +46,7 @@ class LightPush extends BaseProtocol implements ILightPush {
   private readonly NUM_PEERS_PROTOCOL = 1;
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
-    super(LightPushCodec, libp2p.components, options);
+    super(LightPushCodec, libp2p.components, log, options);
   }
 
   private async preparePushMessage(

packages/core/src/lib/metadata/index.ts

@@ -32,7 +32,7 @@ class Metadata extends BaseProtocol implements IMetadata {
     public shardInfo: ShardingParams,
     libp2p: Libp2pComponents
   ) {
-    super(MetadataCodec, libp2p.components, shardInfo && { shardInfo });
+    super(MetadataCodec, libp2p.components, log, shardInfo && { shardInfo });
     this.libp2pComponents = libp2p;
     void libp2p.registrar.handle(MetadataCodec, (streamData) => {
       void this.onRequest(streamData);
@@ -75,7 +75,10 @@ class Metadata extends BaseProtocol implements IMetadata {
   async query(peerId: PeerId): Promise<ShardInfo> {
     const request = proto_metadata.WakuMetadataRequest.encode(this.shardInfo);
 
-    const peer = await this.getPeer(peerId);
+    const peer = await this.peerStore.get(peerId);
+    if (!peer) {
+      throw new Error(`Peer ${peerId.toString()} not found`);
+    }
 
     const stream = await this.getStream(peer);
 

packages/core/src/lib/store/index.ts

@@ -76,7 +76,7 @@ class Store extends BaseProtocol implements IStore {
   private readonly NUM_PEERS_PROTOCOL = 1;
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
-    super(StoreCodec, libp2p.components, options);
+    super(StoreCodec, libp2p.components, log, options);
   }
 
   /**

packages/dns-discovery/CHANGELOG.md

@@ -46,6 +46,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   * devDependencies
     * @waku/interfaces bumped from 0.0.13 to 0.0.14
 
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @waku/enr bumped from 0.0.19 to 0.0.20
+    * @waku/utils bumped from 0.0.13 to 0.0.14
+  * devDependencies
+    * @waku/interfaces bumped from 0.0.20 to 0.0.21
+
 ## [0.0.19](https://github.com/waku-org/js-waku/compare/dns-discovery-v0.0.18...dns-discovery-v0.0.19) (2023-11-01)
 
 

packages/dns-discovery/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@waku/dns-discovery",
-  "version": "0.0.19",
+  "version": "0.0.20",
   "description": "DNS Peer Discovery (EIP-1459)",
   "types": "./dist/index.d.ts",
   "module": "./dist/index.js",
@@ -51,8 +51,8 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@waku/enr": "0.0.19",
-    "@waku/utils": "0.0.13",
+    "@waku/enr": "0.0.20",
+    "@waku/utils": "0.0.14",
     "debug": "^4.3.4",
     "dns-query": "^0.11.2",
     "hi-base32": "^0.5.1",
@@ -67,7 +67,7 @@
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@types/chai": "^4.3.11",
     "@waku/build-utils": "*",
-    "@waku/interfaces": "0.0.20",
+    "@waku/interfaces": "0.0.21",
     "chai": "^4.3.10",
     "cspell": "^7.3.2",
     "mocha": "^10.2.0",