From ea9fdc166a0350a52a15b984406c0cdeba2e9218 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 15:03:53 +1200 Subject: [PATCH 01/39] Fix handshake types. --- packages/libp2p-daemon-client/src/index.ts | 2 +- packages/libp2p-daemon-protocol/src/index.ts | 158 +++++++----------- .../src/stream-handler.ts | 4 +- 3 files changed, 67 insertions(+), 97 deletions(-) diff --git a/packages/libp2p-daemon-client/src/index.ts b/packages/libp2p-daemon-client/src/index.ts index 2719cef9..90121037 100644 --- a/packages/libp2p-daemon-client/src/index.ts +++ b/packages/libp2p-daemon-client/src/index.ts @@ -175,7 +175,7 @@ class Client implements DaemonClient { throw errcode(new Error(response.error?.msg ?? 'Open stream failed'), 'ERR_OPEN_STREAM_FAILED') } - return sh.rest() + return sh.rest() as Duplex } /** diff --git a/packages/libp2p-daemon-protocol/src/index.ts b/packages/libp2p-daemon-protocol/src/index.ts index e2f06279..32080ca2 100644 --- a/packages/libp2p-daemon-protocol/src/index.ts +++ b/packages/libp2p-daemon-protocol/src/index.ts @@ -110,7 +110,9 @@ export namespace Request { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + type: Type.IDENTIFY + } const end = length == null ? reader.len : reader.pos + length @@ -258,7 +260,10 @@ export namespace Response { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + type: Type.OK, + peers: [] + } const end = length == null ? reader.len : reader.pos + length @@ -282,7 +287,6 @@ export namespace Response { obj.dht = DHTResponse.codec().decode(reader, reader.uint32()) break case 6: - obj.peers = obj.peers ?? [] obj.peers.push(PeerInfo.codec().decode(reader, reader.uint32())) break case 7: @@ -297,16 +301,10 @@ export namespace Response { } } - obj.peers = obj.peers ?? [] - if (obj.type == null) { throw new Error('Protocol error: value for required field "type" was not found in protobuf') } - if (obj.peers == null) { - throw new Error('Protocol error: value for required field "peers" was not found in protobuf') - } - return obj }) } @@ -358,7 +356,10 @@ export namespace IdentifyResponse { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + id: new Uint8Array(0), + addrs: [] + } const end = length == null ? reader.len : reader.pos + length @@ -370,7 +371,6 @@ export namespace IdentifyResponse { obj.id = reader.bytes() break case 2: - obj.addrs = obj.addrs ?? [] obj.addrs.push(reader.bytes()) break default: @@ -379,16 +379,10 @@ export namespace IdentifyResponse { } } - obj.addrs = obj.addrs ?? [] - if (obj.id == null) { throw new Error('Protocol error: value for required field "id" was not found in protobuf') } - if (obj.addrs == null) { - throw new Error('Protocol error: value for required field "addrs" was not found in protobuf') - } - return obj }) } @@ -446,7 +440,10 @@ export namespace ConnectRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + peer: new Uint8Array(0), + addrs: [] + } const end = length == null ? reader.len : reader.pos + length @@ -458,7 +455,6 @@ export namespace ConnectRequest { obj.peer = reader.bytes() break case 2: - obj.addrs = obj.addrs ?? [] obj.addrs.push(reader.bytes()) break case 3: @@ -470,16 +466,10 @@ export namespace ConnectRequest { } } - obj.addrs = obj.addrs ?? [] - if (obj.peer == null) { throw new Error('Protocol error: value for required field "peer" was not found in protobuf') } - if (obj.addrs == null) { - throw new Error('Protocol error: value for required field "addrs" was not found in protobuf') - } - return obj }) } @@ -537,7 +527,10 @@ export namespace StreamOpenRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + peer: new Uint8Array(0), + proto: [] + } const end = length == null ? reader.len : reader.pos + length @@ -549,7 +542,6 @@ export namespace StreamOpenRequest { obj.peer = reader.bytes() break case 2: - obj.proto = obj.proto ?? [] obj.proto.push(reader.string()) break case 3: @@ -561,16 +553,10 @@ export namespace StreamOpenRequest { } } - obj.proto = obj.proto ?? [] - if (obj.peer == null) { throw new Error('Protocol error: value for required field "peer" was not found in protobuf') } - if (obj.proto == null) { - throw new Error('Protocol error: value for required field "proto" was not found in protobuf') - } - return obj }) } @@ -622,7 +608,10 @@ export namespace StreamHandlerRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + addr: new Uint8Array(0), + proto: [] + } const end = length == null ? reader.len : reader.pos + length @@ -634,7 +623,6 @@ export namespace StreamHandlerRequest { obj.addr = reader.bytes() break case 2: - obj.proto = obj.proto ?? [] obj.proto.push(reader.string()) break default: @@ -643,16 +631,10 @@ export namespace StreamHandlerRequest { } } - obj.proto = obj.proto ?? [] - if (obj.addr == null) { throw new Error('Protocol error: value for required field "addr" was not found in protobuf') } - if (obj.proto == null) { - throw new Error('Protocol error: value for required field "proto" was not found in protobuf') - } - return obj }) } @@ -694,7 +676,9 @@ export namespace ErrorResponse { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + msg: '' + } const end = length == null ? reader.len : reader.pos + length @@ -772,7 +756,11 @@ export namespace StreamInfo { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + peer: new Uint8Array(0), + addr: new Uint8Array(0), + proto: '' + } const end = length == null ? reader.len : reader.pos + length @@ -914,7 +902,9 @@ export namespace DHTRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + type: Type.FIND_PEER + } const end = length == null ? reader.len : reader.pos + length @@ -1024,7 +1014,9 @@ export namespace DHTResponse { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + type: Type.BEGIN + } const end = length == null ? reader.len : reader.pos + length @@ -1102,7 +1094,10 @@ export namespace PeerInfo { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + id: new Uint8Array(0), + addrs: [] + } const end = length == null ? reader.len : reader.pos + length @@ -1114,7 +1109,6 @@ export namespace PeerInfo { obj.id = reader.bytes() break case 2: - obj.addrs = obj.addrs ?? [] obj.addrs.push(reader.bytes()) break default: @@ -1123,16 +1117,10 @@ export namespace PeerInfo { } } - obj.addrs = obj.addrs ?? [] - if (obj.id == null) { throw new Error('Protocol error: value for required field "id" was not found in protobuf') } - if (obj.addrs == null) { - throw new Error('Protocol error: value for required field "addrs" was not found in protobuf') - } - return obj }) } @@ -1210,7 +1198,9 @@ export namespace ConnManagerRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + type: Type.TAG_PEER + } const end = length == null ? reader.len : reader.pos + length @@ -1281,7 +1271,9 @@ export namespace DisconnectRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + peer: new Uint8Array(0) + } const end = length == null ? reader.len : reader.pos + length @@ -1375,7 +1367,9 @@ export namespace PSRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + type: Type.GET_TOPICS + } const end = length == null ? reader.len : reader.pos + length @@ -1475,7 +1469,9 @@ export namespace PSMessage { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + topicIDs: [] + } const end = length == null ? reader.len : reader.pos + length @@ -1493,7 +1489,6 @@ export namespace PSMessage { obj.seqno = reader.bytes() break case 4: - obj.topicIDs = obj.topicIDs ?? [] obj.topicIDs.push(reader.string()) break case 5: @@ -1508,12 +1503,6 @@ export namespace PSMessage { } } - obj.topicIDs = obj.topicIDs ?? [] - - if (obj.topicIDs == null) { - throw new Error('Protocol error: value for required field "topicIDs" was not found in protobuf') - } - return obj }) } @@ -1567,7 +1556,10 @@ export namespace PSResponse { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + topics: [], + peerIDs: [] + } const end = length == null ? reader.len : reader.pos + length @@ -1576,11 +1568,9 @@ export namespace PSResponse { switch (tag >>> 3) { case 1: - obj.topics = obj.topics ?? [] obj.topics.push(reader.string()) break case 2: - obj.peerIDs = obj.peerIDs ?? [] obj.peerIDs.push(reader.bytes()) break default: @@ -1589,17 +1579,6 @@ export namespace PSResponse { } } - obj.topics = obj.topics ?? [] - obj.peerIDs = obj.peerIDs ?? [] - - if (obj.topics == null) { - throw new Error('Protocol error: value for required field "topics" was not found in protobuf') - } - - if (obj.peerIDs == null) { - throw new Error('Protocol error: value for required field "peerIDs" was not found in protobuf') - } - return obj }) } @@ -1673,7 +1652,10 @@ export namespace PeerstoreRequest { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + type: Type.GET_PROTOCOLS, + protos: [] + } const end = length == null ? reader.len : reader.pos + length @@ -1688,7 +1670,6 @@ export namespace PeerstoreRequest { obj.id = reader.bytes() break case 3: - obj.protos = obj.protos ?? [] obj.protos.push(reader.string()) break default: @@ -1697,16 +1678,10 @@ export namespace PeerstoreRequest { } } - obj.protos = obj.protos ?? [] - if (obj.type == null) { throw new Error('Protocol error: value for required field "type" was not found in protobuf') } - if (obj.protos == null) { - throw new Error('Protocol error: value for required field "protos" was not found in protobuf') - } - return obj }) } @@ -1756,7 +1731,9 @@ export namespace PeerstoreResponse { writer.ldelim() } }, (reader, length) => { - const obj: any = {} + const obj: any = { + protos: [] + } const end = length == null ? reader.len : reader.pos + length @@ -1768,7 +1745,6 @@ export namespace PeerstoreResponse { obj.peer = PeerInfo.codec().decode(reader, reader.uint32()) break case 2: - obj.protos = obj.protos ?? [] obj.protos.push(reader.string()) break default: @@ -1777,12 +1753,6 @@ export namespace PeerstoreResponse { } } - obj.protos = obj.protos ?? [] - - if (obj.protos == null) { - throw new Error('Protocol error: value for required field "protos" was not found in protobuf') - } - return obj }) } diff --git a/packages/libp2p-daemon-protocol/src/stream-handler.ts b/packages/libp2p-daemon-protocol/src/stream-handler.ts index 46d73c2d..8ee5b6bf 100644 --- a/packages/libp2p-daemon-protocol/src/stream-handler.ts +++ b/packages/libp2p-daemon-protocol/src/stream-handler.ts @@ -14,7 +14,7 @@ export interface StreamHandlerOptions { export class StreamHandler { private readonly stream: Duplex - private readonly shake: Handshake + private readonly shake: Handshake public decoder: Source /** * Create a stream handler for connection @@ -54,7 +54,7 @@ export class StreamHandler { */ rest () { this.shake.rest() - return this.shake.stream + return this.shake.stream as Duplex } /** From bd12e5fd7ffcfe41af07997b3e2c06081a421e3d Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 15:45:12 +1200 Subject: [PATCH 02/39] Fix stream client test. --- packages/libp2p-daemon-client/test/stream.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/libp2p-daemon-client/test/stream.spec.ts b/packages/libp2p-daemon-client/test/stream.spec.ts index 0622f5b6..b7a067ac 100644 --- a/packages/libp2p-daemon-client/test/stream.spec.ts +++ b/packages/libp2p-daemon-client/test/stream.spec.ts @@ -85,7 +85,6 @@ describe('daemon stream client', function () { async (source) => await all(source) ) - expect(data).to.have.lengthOf(1) - expect(uint8ArrayToString(data[0])).to.equal('hello world') + expect(uint8ArrayToString(data[0].subarray())).to.equal('hello world') }) }) From 74bbfb4907e8ce831eac0b007dcaffceaa031172 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:20:20 +1200 Subject: [PATCH 03/39] Fix yargs issues. --- packages/libp2p-daemon/src/index.ts | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 8cafda71..ec13d1be 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -3,18 +3,16 @@ import { Multiaddr } from '@multiformats/multiaddr' import yargs from 'yargs' +import { hideBin } from 'yargs/helpers' // @ts-expect-error no types import YargsPromise from 'yargs-promise' import type { Libp2pServer } from '@libp2p/daemon-server' import esMain from 'es-main' -const args = process.argv.slice(2) -const parser = new YargsPromise(yargs) - const log = console.log export default async function main (processArgs: string[]) { - parser.yargs + const argv: { [key: string]: any } = yargs(hideBin(process.argv)) .option('listen', { desc: 'daemon control listen multiaddr', type: 'string', @@ -95,22 +93,14 @@ export default async function main (processArgs: string[]) { throw err // preserve stack } - if (args.length > 0) { + if (hideBin(process.argv).length > 0) { // eslint-disable-next-line log(msg) } yargs.showHelp() }) - - const { data, argv } = await parser.parse(processArgs) - - if (data != null) { - // Log help and exit - // eslint-disable-next-line - log(data) - process.exit(0) - } + .parse() const daemon = await createLibp2pServer(new Multiaddr(argv.listen), argv) await daemon.start() @@ -124,8 +114,6 @@ export default async function main (processArgs: string[]) { export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { // const libp2p = await createLibp2p(argv) // const daemon = await createServer(new Multiaddr(argv.listen), libp2p) - - throw new Error('Not implemented yet') } if (esMain(import.meta)) { From 6f2bbf1bc1903c8f711a34eeb43ea044686dbd09 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:23:30 +1200 Subject: [PATCH 04/39] Get libp2p to run. --- packages/libp2p-daemon/package.json | 5 +++++ packages/libp2p-daemon/src/index.ts | 28 ++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index 4d082dc3..4ab1ce65 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -136,9 +136,14 @@ "release": "aegir release" }, "dependencies": { + "@chainsafe/libp2p-noise": "^8.0.1", + "@libp2p/bootstrap": "^2.0.0", "@libp2p/daemon-server": "^2.0.0", + "@libp2p/mplex": "^5.2.1", + "@libp2p/tcp": "^3.0.6", "@multiformats/multiaddr": "^10.1.8", "es-main": "^1.0.2", + "libp2p": "^0.39.0", "yargs": "^17.3.1", "yargs-promise": "^1.1.0" }, diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index ec13d1be..46d0160e 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -6,8 +6,12 @@ import yargs from 'yargs' import { hideBin } from 'yargs/helpers' // @ts-expect-error no types import YargsPromise from 'yargs-promise' -import type { Libp2pServer } from '@libp2p/daemon-server' import esMain from 'es-main' +import { Libp2p, createLibp2p } from 'libp2p' +import { Noise } from "@chainsafe/libp2p-noise"; +import { Mplex } from "@libp2p/mplex"; +import { TCP } from "@libp2p/tcp"; +import { Bootstrap } from "@libp2p/bootstrap"; const log = console.log @@ -111,9 +115,25 @@ export default async function main (processArgs: string[]) { } } -export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { - // const libp2p = await createLibp2p(argv) - // const daemon = await createServer(new Multiaddr(argv.listen), libp2p) +export function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { + return createLibp2p({ + addresses: { + listen: argv.hostAddrs.split(",") + }, + + transports: [ + new TCP() + ], + + connectionEncryption: [new Noise()], + streamMuxers: [new Mplex()], + peerDiscovery: [ + new Bootstrap({ + interval: 60e3, + list: argv.bootstrapPeers.split(",") + }) + ] + }) } if (esMain(import.meta)) { From dae43259f74c8ee656f1b707583a712047b5bc8a Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:26:32 +1200 Subject: [PATCH 05/39] Add ipfs boostrap peers as default. --- packages/libp2p-daemon/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 46d0160e..bdb0d421 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -52,7 +52,7 @@ export default async function main (processArgs: string[]) { .option('bootstrapPeers', { desc: 'Comma separated list of bootstrap peers; defaults to the IPFS DHT peers', type: 'string', - default: '' + default: '/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ,/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dnsaddr/bootstrap.libp2p.io/p2p/QmZa1sAxajnQjVM8WjWXoMbmPd7NsWhfKsPkErzpm9wGkp,/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt' }) .option('dht', { desc: 'Enables the DHT in full node mode', From f73d832eafc1d765c778890fc5029e9c7a0c3193 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:32:22 +1200 Subject: [PATCH 06/39] Add websockets to transports. --- packages/libp2p-daemon/package.json | 1 + packages/libp2p-daemon/src/index.ts | 44 ++++++++++++++++------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index 4ab1ce65..764a95cd 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -141,6 +141,7 @@ "@libp2p/daemon-server": "^2.0.0", "@libp2p/mplex": "^5.2.1", "@libp2p/tcp": "^3.0.6", + "@libp2p/websockets": "^3.0.3", "@multiformats/multiaddr": "^10.1.8", "es-main": "^1.0.2", "libp2p": "^0.39.0", diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index bdb0d421..10c2a309 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -8,10 +8,11 @@ import { hideBin } from 'yargs/helpers' import YargsPromise from 'yargs-promise' import esMain from 'es-main' import { Libp2p, createLibp2p } from 'libp2p' -import { Noise } from "@chainsafe/libp2p-noise"; -import { Mplex } from "@libp2p/mplex"; -import { TCP } from "@libp2p/tcp"; -import { Bootstrap } from "@libp2p/bootstrap"; +import { Noise } from '@chainsafe/libp2p-noise' +import { Mplex } from '@libp2p/mplex' +import { TCP } from '@libp2p/tcp' +import { WebSockets } from '@libp2p/websockets' +import { Bootstrap } from '@libp2p/bootstrap' const log = console.log @@ -116,24 +117,27 @@ export default async function main (processArgs: string[]) { } export function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { - return createLibp2p({ - addresses: { - listen: argv.hostAddrs.split(",") - }, + const options = { + addresses: { + listen: argv.hostAddrs.split(",") + }, - transports: [ - new TCP() - ], + transports: [ + new TCP(), + new WebSockets() + ], - connectionEncryption: [new Noise()], - streamMuxers: [new Mplex()], - peerDiscovery: [ - new Bootstrap({ - interval: 60e3, - list: argv.bootstrapPeers.split(",") - }) - ] - }) + connectionEncryption: [new Noise()], + streamMuxers: [new Mplex()], + peerDiscovery: [ + new Bootstrap({ + interval: 60e3, + list: argv.bootstrapPeers.split(",") + }) + ] + } + + return createLibp2p(options) } if (esMain(import.meta)) { From 11ecd7e09684773e2fcd79526e587cb388054711 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:42:06 +1200 Subject: [PATCH 07/39] Add pubsub support. --- packages/libp2p-daemon/package.json | 2 ++ packages/libp2p-daemon/src/index.ts | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index 764a95cd..1030631e 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -136,9 +136,11 @@ "release": "aegir release" }, "dependencies": { + "@chainsafe/libp2p-gossipsub": "^4.1.1", "@chainsafe/libp2p-noise": "^8.0.1", "@libp2p/bootstrap": "^2.0.0", "@libp2p/daemon-server": "^2.0.0", + "@libp2p/floodsub": "^3.0.7", "@libp2p/mplex": "^5.2.1", "@libp2p/tcp": "^3.0.6", "@libp2p/websockets": "^3.0.3", diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 10c2a309..8bc2bd4b 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -7,12 +7,14 @@ import { hideBin } from 'yargs/helpers' // @ts-expect-error no types import YargsPromise from 'yargs-promise' import esMain from 'es-main' -import { Libp2p, createLibp2p } from 'libp2p' +import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' import { Noise } from '@chainsafe/libp2p-noise' import { Mplex } from '@libp2p/mplex' import { TCP } from '@libp2p/tcp' import { WebSockets } from '@libp2p/websockets' import { Bootstrap } from '@libp2p/bootstrap' +import { GossipSub } from '@chainsafe/libp2p-gossipsub' +import { FloodSub } from '@libp2p/floodsub' const log = console.log @@ -117,7 +119,7 @@ export default async function main (processArgs: string[]) { } export function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { - const options = { + const options: Libp2pOptions = { addresses: { listen: argv.hostAddrs.split(",") }, @@ -137,6 +139,19 @@ export function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise Date: Fri, 9 Sep 2022 16:43:27 +1200 Subject: [PATCH 08/39] Remove yargs-promise. --- packages/libp2p-daemon/package.json | 3 +-- packages/libp2p-daemon/src/index.ts | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index 1030631e..831200e8 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -147,8 +147,7 @@ "@multiformats/multiaddr": "^10.1.8", "es-main": "^1.0.2", "libp2p": "^0.39.0", - "yargs": "^17.3.1", - "yargs-promise": "^1.1.0" + "yargs": "^17.3.1" }, "devDependencies": { "aegir": "^37.2.0", diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 8bc2bd4b..0cf32f73 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -4,8 +4,6 @@ import { Multiaddr } from '@multiformats/multiaddr' import yargs from 'yargs' import { hideBin } from 'yargs/helpers' -// @ts-expect-error no types -import YargsPromise from 'yargs-promise' import esMain from 'es-main' import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' import { Noise } from '@chainsafe/libp2p-noise' From a99dd27ab5618da562634fd37758ae3c56a61159 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:45:29 +1200 Subject: [PATCH 09/39] Add default option for hostAddrs. --- packages/libp2p-daemon/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 0cf32f73..f634eeec 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -37,7 +37,7 @@ export default async function main (processArgs: string[]) { .option('hostAddrs', { desc: 'Comma separated list of multiaddrs the host should listen on', type: 'string', - default: '' + default: '/ip4/0.0.0.0/tcp/0' }) .option('announceAddrs', { desc: 'Comma separated list of multiaddrs the host should announce to the network', From 0828d1ebca949a6e36a895920317029d1f20ff97 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:45:37 +1200 Subject: [PATCH 10/39] Enable cli tests. --- packages/libp2p-daemon/test/cli.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/test/cli.spec.ts b/packages/libp2p-daemon/test/cli.spec.ts index 2712cb8d..9d535e53 100644 --- a/packages/libp2p-daemon/test/cli.spec.ts +++ b/packages/libp2p-daemon/test/cli.spec.ts @@ -4,7 +4,7 @@ import { expect } from 'aegir/chai' import sinon from 'sinon' import cli from '../src/index.js' -describe.skip('cli', () => { +describe('cli', () => { const daemon = { createDaemon: (options: any) => {} } afterEach(() => { From 544deb4fb4bd3f7eb71f616896dbe55e858d3bfc Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 16:53:39 +1200 Subject: [PATCH 11/39] Get the daemon-server to start. --- packages/libp2p-daemon/src/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index f634eeec..098a23f8 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -5,6 +5,7 @@ import { Multiaddr } from '@multiformats/multiaddr' import yargs from 'yargs' import { hideBin } from 'yargs/helpers' import esMain from 'es-main' +import { createServer, Libp2pServer } from '@libp2p/daemon-server' import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' import { Noise } from '@chainsafe/libp2p-noise' import { Mplex } from '@libp2p/mplex' @@ -116,7 +117,7 @@ export default async function main (processArgs: string[]) { } } -export function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { +export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { const options: Libp2pOptions = { addresses: { listen: argv.hostAddrs.split(",") @@ -150,7 +151,10 @@ export function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise Date: Fri, 9 Sep 2022 17:05:21 +1200 Subject: [PATCH 12/39] Fix listen address and argument issues. --- packages/libp2p-daemon/src/index.ts | 12 +++++++----- packages/libp2p-daemon/test/cli.spec.ts | 8 ++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 098a23f8..97bf6015 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -18,11 +18,13 @@ import { FloodSub } from '@libp2p/floodsub' const log = console.log export default async function main (processArgs: string[]) { - const argv: { [key: string]: any } = yargs(hideBin(process.argv)) + const argv: { [key: string]: any } = yargs(hideBin(processArgs)) .option('listen', { desc: 'daemon control listen multiaddr', type: 'string', - default: '/unix/tmp/p2pd.sock' + //default: '/unix/tmp/p2pd.sock' + // UNIX sockets are not supported by @libp2p/tcp yet... + default: '/ip4/127.0.0.1/tcp/0' }) .option('quiet', { alias: 'q', @@ -99,7 +101,7 @@ export default async function main (processArgs: string[]) { throw err // preserve stack } - if (hideBin(process.argv).length > 0) { + if (hideBin(processArgs).length > 0) { // eslint-disable-next-line log(msg) } @@ -151,8 +153,8 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } } - const libp2p: Libp2p = await createLibp2p(options); - const daemon: Libp2pServer = await createServer(new Multiaddr(argv.listen), libp2p) + const libp2p: Libp2p = await createLibp2p(options); + const daemon: Libp2pServer = createServer(listenAddr, libp2p) return daemon; } diff --git a/packages/libp2p-daemon/test/cli.spec.ts b/packages/libp2p-daemon/test/cli.spec.ts index 9d535e53..0030efa1 100644 --- a/packages/libp2p-daemon/test/cli.spec.ts +++ b/packages/libp2p-daemon/test/cli.spec.ts @@ -59,7 +59,9 @@ describe('cli', () => { id: '/path/to/key', q: true, quiet: true, - listen: '/unix/tmp/d.sock' + //listen: '/unix/tmp/d.sock' + // UNIX sockets are not supported by @libp2p/tcp yet... + listen: '/ip4/127.0.0.1/tcp/0' }) return { start: () => {}, @@ -79,7 +81,9 @@ describe('cli', () => { '--dhtClient=true', '--quiet=true', '--id=/path/to/key', - '--listen=/unix/tmp/d.sock' + //'--listen=/unix/tmp/d.sock' + // UNIX sockets are not supported by @libp2p/tcp yet... + '--listen=/ip4/127.0.0.1/tcp/0' ]) }) }) From 3f0fcc58a6f527fa97fba100aad813d2186e9dd4 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 17:22:45 +1200 Subject: [PATCH 13/39] Handle announceAddrs argument. --- packages/libp2p-daemon/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 97bf6015..c354d2b7 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -122,7 +122,8 @@ export default async function main (processArgs: string[]) { export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { const options: Libp2pOptions = { addresses: { - listen: argv.hostAddrs.split(",") + listen: argv.hostAddrs.split(","), + announce: argv.announceAddrs.split(",") }, transports: [ From a79b2200260a925daf454070d31deb554f6f5ed7 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 17:36:05 +1200 Subject: [PATCH 14/39] Only bootstrap peers if enabled. --- packages/libp2p-daemon/src/index.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index c354d2b7..3fc4ed4d 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -126,20 +126,19 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro announce: argv.announceAddrs.split(",") }, - transports: [ - new TCP(), - new WebSockets() - ], + transports: [ new TCP(), new WebSockets() ], + connectionEncryption: [ new Noise() ], + streamMuxers: [ new Mplex() ] + } - connectionEncryption: [new Noise()], - streamMuxers: [new Mplex()], - peerDiscovery: [ + if (argv.bootstrap) { + options.peerDiscovery = [ new Bootstrap({ interval: 60e3, list: argv.bootstrapPeers.split(",") }) ] - } + } if (argv.pubsub) { switch (argv.pubsubRouter) { From d7cf21404c5d7aa7bf7c3559e9176abb58e46b18 Mon Sep 17 00:00:00 2001 From: saul Date: Fri, 9 Sep 2022 18:32:59 +1200 Subject: [PATCH 15/39] Handle the id argument. --- packages/libp2p-daemon/package.json | 2 ++ packages/libp2p-daemon/src/index.ts | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index 831200e8..c9842a4e 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -139,9 +139,11 @@ "@chainsafe/libp2p-gossipsub": "^4.1.1", "@chainsafe/libp2p-noise": "^8.0.1", "@libp2p/bootstrap": "^2.0.0", + "@libp2p/crypto": "^1.0.4", "@libp2p/daemon-server": "^2.0.0", "@libp2p/floodsub": "^3.0.7", "@libp2p/mplex": "^5.2.1", + "@libp2p/peer-id-factory": "^1.0.18", "@libp2p/tcp": "^3.0.6", "@libp2p/websockets": "^3.0.3", "@multiformats/multiaddr": "^10.1.8", diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 3fc4ed4d..458daf13 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -1,6 +1,7 @@ #! /usr/bin/env node /* eslint no-console: ["error", { allow: ["log", "warn", "error"] }] */ +import { promises as fs } from 'fs' import { Multiaddr } from '@multiformats/multiaddr' import yargs from 'yargs' import { hideBin } from 'yargs/helpers' @@ -14,6 +15,8 @@ import { WebSockets } from '@libp2p/websockets' import { Bootstrap } from '@libp2p/bootstrap' import { GossipSub } from '@chainsafe/libp2p-gossipsub' import { FloodSub } from '@libp2p/floodsub' +import { unmarshalPrivateKey } from '@libp2p/crypto/keys' +import { createFromPrivKey } from '@libp2p/peer-id-factory' const log = console.log @@ -131,14 +134,22 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro streamMuxers: [ new Mplex() ] } - if (argv.bootstrap) { - options.peerDiscovery = [ + if (argv.id) { + const marshaledKey: Buffer = await fs.readFile(argv.id); + const unmarshaledKey = await unmarshalPrivateKey(marshaledKey) + const peerId = await createFromPrivKey(unmarshaledKey) + + options.peerId = peerId; + } + + if (argv.bootstrap) { + options.peerDiscovery = [ new Bootstrap({ interval: 60e3, list: argv.bootstrapPeers.split(",") }) ] - } + } if (argv.pubsub) { switch (argv.pubsubRouter) { From f71bf2cb21c07216b1d678b0f0ce09342b94d7a8 Mon Sep 17 00:00:00 2001 From: saul Date: Sat, 10 Sep 2022 14:34:30 +1200 Subject: [PATCH 16/39] Add DHT support. --- packages/libp2p-daemon/package.json | 1 + packages/libp2p-daemon/src/index.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index c9842a4e..c4b944b8 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -142,6 +142,7 @@ "@libp2p/crypto": "^1.0.4", "@libp2p/daemon-server": "^2.0.0", "@libp2p/floodsub": "^3.0.7", + "@libp2p/kad-dht": "^3.0.4", "@libp2p/mplex": "^5.2.1", "@libp2p/peer-id-factory": "^1.0.18", "@libp2p/tcp": "^3.0.6", diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 458daf13..4f9e9d34 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -17,6 +17,7 @@ import { GossipSub } from '@chainsafe/libp2p-gossipsub' import { FloodSub } from '@libp2p/floodsub' import { unmarshalPrivateKey } from '@libp2p/crypto/keys' import { createFromPrivKey } from '@libp2p/peer-id-factory' +import { KadDHT } from '@libp2p/kad-dht' const log = console.log @@ -164,6 +165,10 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } } + if (argv.dht) { + options.dht = new KadDHT() + } + const libp2p: Libp2p = await createLibp2p(options); const daemon: Libp2pServer = createServer(listenAddr, libp2p) From 8de058eea904afbe621c189e6f633dd91d662e1e Mon Sep 17 00:00:00 2001 From: saul Date: Sat, 10 Sep 2022 14:36:54 +1200 Subject: [PATCH 17/39] Documentation. --- packages/libp2p-daemon/src/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 4f9e9d34..6bf3faee 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -73,7 +73,7 @@ export default async function main (processArgs: string[]) { default: false }) .option('nat', { - desc: 'Enables UPnP NAT hole punching', + desc: '(Not yet supported) Enables UPnP NAT hole punching', type: 'boolean', default: false }) @@ -124,6 +124,7 @@ export default async function main (processArgs: string[]) { } export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { + // Minimum libp2p setup. const options: Libp2pOptions = { addresses: { listen: argv.hostAddrs.split(","), @@ -135,6 +136,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro streamMuxers: [ new Mplex() ] } + // Load key file as peer ID. if (argv.id) { const marshaledKey: Buffer = await fs.readFile(argv.id); const unmarshaledKey = await unmarshalPrivateKey(marshaledKey) @@ -143,6 +145,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro options.peerId = peerId; } + // Enable bootstrap peers. if (argv.bootstrap) { options.peerDiscovery = [ new Bootstrap({ @@ -152,6 +155,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro ] } + // Configure PubSub if (argv.pubsub) { switch (argv.pubsubRouter) { case "gossipsub": @@ -165,6 +169,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } } + // Enable DHT if (argv.dht) { options.dht = new KadDHT() } From 452cb8e7d579e2a8049424eda973b027b7cdb3cb Mon Sep 17 00:00:00 2001 From: saul Date: Sat, 10 Sep 2022 14:50:44 +1200 Subject: [PATCH 18/39] Fix linter errors. --- packages/libp2p-daemon/src/index.ts | 38 ++++++++++++------------- packages/libp2p-daemon/test/cli.spec.ts | 4 +-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 6bf3faee..b0f79954 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -26,7 +26,7 @@ export default async function main (processArgs: string[]) { .option('listen', { desc: 'daemon control listen multiaddr', type: 'string', - //default: '/unix/tmp/p2pd.sock' + // default: '/unix/tmp/p2pd.sock' // UNIX sockets are not supported by @libp2p/tcp yet... default: '/ip4/127.0.0.1/tcp/0' }) @@ -112,7 +112,7 @@ export default async function main (processArgs: string[]) { yargs.showHelp() }) - .parse() + .parse() const daemon = await createLibp2pServer(new Multiaddr(argv.listen), argv) await daemon.start() @@ -127,57 +127,57 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro // Minimum libp2p setup. const options: Libp2pOptions = { addresses: { - listen: argv.hostAddrs.split(","), - announce: argv.announceAddrs.split(",") + listen: argv.hostAddrs.split(','), + announce: argv.announceAddrs.split(',') }, - transports: [ new TCP(), new WebSockets() ], - connectionEncryption: [ new Noise() ], - streamMuxers: [ new Mplex() ] + transports: [new TCP(), new WebSockets()], + connectionEncryption: [new Noise()], + streamMuxers: [new Mplex()] } // Load key file as peer ID. - if (argv.id) { - const marshaledKey: Buffer = await fs.readFile(argv.id); + if (argv.id == null) { + const marshaledKey: Buffer = await fs.readFile(argv.id) const unmarshaledKey = await unmarshalPrivateKey(marshaledKey) const peerId = await createFromPrivKey(unmarshaledKey) - options.peerId = peerId; + options.peerId = peerId } // Enable bootstrap peers. - if (argv.bootstrap) { + if (argv.bootstrap == null) { options.peerDiscovery = [ new Bootstrap({ interval: 60e3, - list: argv.bootstrapPeers.split(",") + list: argv.bootstrapPeers.split(',') }) ] } // Configure PubSub - if (argv.pubsub) { + if (argv.pubsub == null) { switch (argv.pubsubRouter) { - case "gossipsub": + case 'gossipsub': options.pubsub = new GossipSub({ allowPublishToZeroPeers: true }) break - case "floodsub": + case 'floodsub': options.pubsub = new FloodSub() break default: - throw new Error("invalid pubsubRouter type") + throw new Error('invalid pubsubRouter type') } } // Enable DHT - if (argv.dht) { + if (argv.dht == null) { options.dht = new KadDHT() } - const libp2p: Libp2p = await createLibp2p(options); + const libp2p: Libp2p = await createLibp2p(options) const daemon: Libp2pServer = createServer(listenAddr, libp2p) - return daemon; + return daemon } if (esMain(import.meta)) { diff --git a/packages/libp2p-daemon/test/cli.spec.ts b/packages/libp2p-daemon/test/cli.spec.ts index 0030efa1..0dc0ecc3 100644 --- a/packages/libp2p-daemon/test/cli.spec.ts +++ b/packages/libp2p-daemon/test/cli.spec.ts @@ -59,7 +59,7 @@ describe('cli', () => { id: '/path/to/key', q: true, quiet: true, - //listen: '/unix/tmp/d.sock' + // listen: '/unix/tmp/d.sock' // UNIX sockets are not supported by @libp2p/tcp yet... listen: '/ip4/127.0.0.1/tcp/0' }) @@ -81,7 +81,7 @@ describe('cli', () => { '--dhtClient=true', '--quiet=true', '--id=/path/to/key', - //'--listen=/unix/tmp/d.sock' + // '--listen=/unix/tmp/d.sock' // UNIX sockets are not supported by @libp2p/tcp yet... '--listen=/ip4/127.0.0.1/tcp/0' ]) From cf89521458b911dcfd149575b46cabda9892d4be Mon Sep 17 00:00:00 2001 From: saul Date: Sat, 10 Sep 2022 14:59:08 +1200 Subject: [PATCH 19/39] Specify the internal port. --- packages/libp2p-daemon/src/index.ts | 2 +- packages/libp2p-daemon/test/cli.spec.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index b0f79954..ce7f9d30 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -28,7 +28,7 @@ export default async function main (processArgs: string[]) { type: 'string', // default: '/unix/tmp/p2pd.sock' // UNIX sockets are not supported by @libp2p/tcp yet... - default: '/ip4/127.0.0.1/tcp/0' + default: '/ip4/127.0.0.1/tcp/1234' }) .option('quiet', { alias: 'q', diff --git a/packages/libp2p-daemon/test/cli.spec.ts b/packages/libp2p-daemon/test/cli.spec.ts index 0dc0ecc3..64ece212 100644 --- a/packages/libp2p-daemon/test/cli.spec.ts +++ b/packages/libp2p-daemon/test/cli.spec.ts @@ -61,7 +61,7 @@ describe('cli', () => { quiet: true, // listen: '/unix/tmp/d.sock' // UNIX sockets are not supported by @libp2p/tcp yet... - listen: '/ip4/127.0.0.1/tcp/0' + listen: '/ip4/127.0.0.1/tcp/1234' }) return { start: () => {}, @@ -83,7 +83,7 @@ describe('cli', () => { '--id=/path/to/key', // '--listen=/unix/tmp/d.sock' // UNIX sockets are not supported by @libp2p/tcp yet... - '--listen=/ip4/127.0.0.1/tcp/0' + '--listen=/ip4/127.0.0.1/tcp/1234' ]) }) }) From 52c9577cf8eb69ee3cc519a87a35e2f2a29f5f36 Mon Sep 17 00:00:00 2001 From: saul Date: Sat, 10 Sep 2022 15:01:26 +1200 Subject: [PATCH 20/39] Add global flag to the install command. --- packages/libp2p-daemon/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/README.md b/packages/libp2p-daemon/README.md index d262a0b1..b8c789ea 100644 --- a/packages/libp2p-daemon/README.md +++ b/packages/libp2p-daemon/README.md @@ -20,7 +20,7 @@ ## Install ```console -$ npm i @libp2p/daemon +$ npm i --location=global @libp2p/daemon ``` ## Specs From 260ff6d6e17f8df20261eb883c0dbcc57fe9c292 Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 12 Sep 2022 10:46:14 +1200 Subject: [PATCH 21/39] Fix argument condition statements. --- packages/libp2p-daemon/src/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index ce7f9d30..3b3b0cec 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -137,7 +137,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Load key file as peer ID. - if (argv.id == null) { + if (argv.id != null) { const marshaledKey: Buffer = await fs.readFile(argv.id) const unmarshaledKey = await unmarshalPrivateKey(marshaledKey) const peerId = await createFromPrivKey(unmarshaledKey) @@ -146,7 +146,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Enable bootstrap peers. - if (argv.bootstrap == null) { + if (argv.bootstrap != null) { options.peerDiscovery = [ new Bootstrap({ interval: 60e3, @@ -156,7 +156,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Configure PubSub - if (argv.pubsub == null) { + if (argv.pubsub != null) { switch (argv.pubsubRouter) { case 'gossipsub': options.pubsub = new GossipSub({ allowPublishToZeroPeers: true }) @@ -170,7 +170,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Enable DHT - if (argv.dht == null) { + if (argv.dht != null) { options.dht = new KadDHT() } From 8a51bddbdaf189c203d1112fe580f299b6df8a13 Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 12 Sep 2022 11:00:03 +1200 Subject: [PATCH 22/39] Add option for PSK. --- packages/libp2p-daemon/src/index.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 3b3b0cec..d9b2bd1c 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -8,6 +8,7 @@ import { hideBin } from 'yargs/helpers' import esMain from 'es-main' import { createServer, Libp2pServer } from '@libp2p/daemon-server' import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' +import { PreSharedKeyConnectionProtector } from "libp2p/pnet"; import { Noise } from '@chainsafe/libp2p-noise' import { Mplex } from '@libp2p/mplex' import { TCP } from '@libp2p/tcp' @@ -100,6 +101,10 @@ export default async function main (processArgs: string[]) { type: 'string', default: 'gossipsub' }) + .option('psk', { + desc: 'Pre-shared key file', + type: 'string' + }) .fail((msg: string, err: Error | undefined, yargs?: any) => { if (err != null) { throw err // preserve stack @@ -174,6 +179,15 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro options.dht = new KadDHT() } + // Configure PSK + if (argv.psk != null) { + const swarmKey: Buffer = await fs.readFile(argv.psk) + + options.connectionProtector = new PreSharedKeyConnectionProtector({ + psk: swarmKey + }) + } + const libp2p: Libp2p = await createLibp2p(options) const daemon: Libp2pServer = createServer(listenAddr, libp2p) From b5f7ae7491f49369ca130ddf46743b6c395d421c Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 12 Sep 2022 11:07:34 +1200 Subject: [PATCH 23/39] Fix boolean argument conditions. --- packages/libp2p-daemon/src/index.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index d9b2bd1c..a1feb9a4 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -8,7 +8,7 @@ import { hideBin } from 'yargs/helpers' import esMain from 'es-main' import { createServer, Libp2pServer } from '@libp2p/daemon-server' import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' -import { PreSharedKeyConnectionProtector } from "libp2p/pnet"; +import { PreSharedKeyConnectionProtector } from 'libp2p/pnet' import { Noise } from '@chainsafe/libp2p-noise' import { Mplex } from '@libp2p/mplex' import { TCP } from '@libp2p/tcp' @@ -39,8 +39,7 @@ export default async function main (processArgs: string[]) { }) .option('id', { desc: 'peer identity; private key file', - type: 'string', - default: '' + type: 'string' }) .option('hostAddrs', { desc: 'Comma separated list of multiaddrs the host should listen on', @@ -151,7 +150,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Enable bootstrap peers. - if (argv.bootstrap != null) { + if (argv.bootstrap === true) { options.peerDiscovery = [ new Bootstrap({ interval: 60e3, @@ -161,7 +160,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Configure PubSub - if (argv.pubsub != null) { + if (argv.pubsub === true) { switch (argv.pubsubRouter) { case 'gossipsub': options.pubsub = new GossipSub({ allowPublishToZeroPeers: true }) @@ -175,7 +174,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Enable DHT - if (argv.dht != null) { + if (argv.dht === true) { options.dht = new KadDHT() } From c552285639a6ddf0cb7815acc530849f8f119d52 Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 12 Sep 2022 12:50:36 +1200 Subject: [PATCH 24/39] Add option for pubsub peer discovery. --- packages/libp2p-daemon/package.json | 1 + packages/libp2p-daemon/src/index.ts | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index b68f44b6..e7bb9884 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -145,6 +145,7 @@ "@libp2p/kad-dht": "^3.0.4", "@libp2p/mplex": "^5.2.1", "@libp2p/peer-id-factory": "^1.0.18", + "@libp2p/pubsub-peer-discovery": "^6.0.2", "@libp2p/tcp": "^3.0.6", "@libp2p/websockets": "^3.0.3", "@multiformats/multiaddr": "^10.1.8", diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index a1feb9a4..d2a49259 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -16,6 +16,7 @@ import { WebSockets } from '@libp2p/websockets' import { Bootstrap } from '@libp2p/bootstrap' import { GossipSub } from '@chainsafe/libp2p-gossipsub' import { FloodSub } from '@libp2p/floodsub' +import { PubSubPeerDiscovery } from '@libp2p/pubsub-peer-discovery' import { unmarshalPrivateKey } from '@libp2p/crypto/keys' import { createFromPrivKey } from '@libp2p/peer-id-factory' import { KadDHT } from '@libp2p/kad-dht' @@ -100,6 +101,11 @@ export default async function main (processArgs: string[]) { type: 'string', default: 'gossipsub' }) + .option('pubsubDiscovery', { + desc: 'Enables pubsub peer discovery', + type: 'boolean', + default: false + }) .option('psk', { desc: 'Pre-shared key file', type: 'string' @@ -137,7 +143,8 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro transports: [new TCP(), new WebSockets()], connectionEncryption: [new Noise()], - streamMuxers: [new Mplex()] + streamMuxers: [new Mplex()], + peerDiscovery: [] } // Load key file as peer ID. @@ -150,17 +157,18 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro } // Enable bootstrap peers. - if (argv.bootstrap === true) { - options.peerDiscovery = [ + if (argv.bootstrap === true && options.peerDiscovery != null) { + options.peerDiscovery.push( new Bootstrap({ interval: 60e3, list: argv.bootstrapPeers.split(',') }) - ] + ) } // Configure PubSub if (argv.pubsub === true) { + // Router implementation. switch (argv.pubsubRouter) { case 'gossipsub': options.pubsub = new GossipSub({ allowPublishToZeroPeers: true }) @@ -171,6 +179,13 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro default: throw new Error('invalid pubsubRouter type') } + + // Peer discovery + if (argv.pubsubDiscovery === true && options.peerDiscovery != null) { + options.peerDiscovery.push( + new PubSubPeerDiscovery({ interval: 60e3 }) + ) + } } // Enable DHT From d087784b0ef38988c320054f806ad28479746aeb Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 12 Sep 2022 12:55:38 +1200 Subject: [PATCH 25/39] Add option for specifying the peer discovery interval. --- packages/libp2p-daemon/src/index.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index d2a49259..2c40d3ef 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -110,6 +110,11 @@ export default async function main (processArgs: string[]) { desc: 'Pre-shared key file', type: 'string' }) + .option('discoveryInterval', { + desc: 'The interval (ms) to perform peer discovery', + type: 'number', + default: 60e3 + }) .fail((msg: string, err: Error | undefined, yargs?: any) => { if (err != null) { throw err // preserve stack @@ -160,7 +165,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro if (argv.bootstrap === true && options.peerDiscovery != null) { options.peerDiscovery.push( new Bootstrap({ - interval: 60e3, + interval: argv.discoveryInterval, list: argv.bootstrapPeers.split(',') }) ) @@ -183,7 +188,7 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro // Peer discovery if (argv.pubsubDiscovery === true && options.peerDiscovery != null) { options.peerDiscovery.push( - new PubSubPeerDiscovery({ interval: 60e3 }) + new PubSubPeerDiscovery({ interval: argv.discoveryInterval }) ) } } From 00cde14cc61114509e90f84a282d11c4173ee04d Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 12 Sep 2022 13:36:43 +1200 Subject: [PATCH 26/39] Add options for relay configuration. --- packages/libp2p-daemon/src/index.ts | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 2c40d3ef..5491edcf 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -115,6 +115,31 @@ export default async function main (processArgs: string[]) { type: 'number', default: 60e3 }) + .option('relay', { + desc: 'Enables relay', + type: 'boolean', + default: false + }) + .option('relayHop', { + desc: 'Enables relay HOP', + type: 'boolean', + default: false + }) + .option('relayAdvertise', { + desc: 'Enables realy HOP advertisement', + type: 'boolean', + default: false + }) + .option('relayAuto', { + desc: 'Enables Auto Relay', + type: 'boolean', + default: false + }) + .option('relayAutoListeners', { + desc: 'Maximum number of simultaneous HOP connections for Auto Relay to open', + type: 'number', + default: 2 + }) .fail((msg: string, err: Error | undefined, yargs?: any) => { if (err != null) { throw err // preserve stack @@ -207,6 +232,32 @@ export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Pro }) } + // Configure relay + if (argv.relay === true) { + options.relay = { + enabled: true + } + + if (argv.relayAuto === true) { + options.relay.autoRelay = { + enabled: true, + maxListeners: argv.relayAutoListeners + } + } + + if (argv.relayHop === true) { + options.relay.hop = { + enabled: true + } + } + + if (argv.relayAdvertise === true) { + options.relay.advertise = { + enabled: true + } + } + } + const libp2p: Libp2p = await createLibp2p(options) const daemon: Libp2pServer = createServer(listenAddr, libp2p) From 881206e821bc745a5134d3becdcedfa4c3735d10 Mon Sep 17 00:00:00 2001 From: saul Date: Tue, 13 Sep 2022 10:37:18 +1200 Subject: [PATCH 27/39] Fix type errors. --- packages/libp2p-daemon-client/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/libp2p-daemon-client/src/index.ts b/packages/libp2p-daemon-client/src/index.ts index a19c1b4f..92d652aa 100644 --- a/packages/libp2p-daemon-client/src/index.ts +++ b/packages/libp2p-daemon-client/src/index.ts @@ -178,7 +178,7 @@ class Client implements DaemonClient { throw errcode(new Error(response.error?.msg ?? 'Open stream failed'), 'ERR_OPEN_STREAM_FAILED') } - return sh.rest() as Duplex + return sh.rest() as Duplex } /** @@ -211,7 +211,7 @@ class Client implements DaemonClient { throw errcode(new Error('Incorrect protocol'), 'ERR_OPEN_STREAM_FAILED') } - await handler(sh.rest()) + await handler(sh.rest() as Duplex) }) .finally(() => { connection.close() From ffd2e649d0f7755ff6c0e6c000f63d6392f2b9d5 Mon Sep 17 00:00:00 2001 From: saul Date: Tue, 13 Sep 2022 12:27:51 +1200 Subject: [PATCH 28/39] Refactor createLibp2pServer into separate file. --- packages/libp2p-daemon/src/index.ts | 119 +-------------------------- packages/libp2p-daemon/src/server.ts | 119 +++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 117 deletions(-) create mode 100644 packages/libp2p-daemon/src/server.ts diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 5491edcf..4c1fcc91 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -1,25 +1,11 @@ #! /usr/bin/env node /* eslint no-console: ["error", { allow: ["log", "warn", "error"] }] */ -import { promises as fs } from 'fs' import { Multiaddr } from '@multiformats/multiaddr' import yargs from 'yargs' import { hideBin } from 'yargs/helpers' import esMain from 'es-main' -import { createServer, Libp2pServer } from '@libp2p/daemon-server' -import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' -import { PreSharedKeyConnectionProtector } from 'libp2p/pnet' -import { Noise } from '@chainsafe/libp2p-noise' -import { Mplex } from '@libp2p/mplex' -import { TCP } from '@libp2p/tcp' -import { WebSockets } from '@libp2p/websockets' -import { Bootstrap } from '@libp2p/bootstrap' -import { GossipSub } from '@chainsafe/libp2p-gossipsub' -import { FloodSub } from '@libp2p/floodsub' -import { PubSubPeerDiscovery } from '@libp2p/pubsub-peer-discovery' -import { unmarshalPrivateKey } from '@libp2p/crypto/keys' -import { createFromPrivKey } from '@libp2p/peer-id-factory' -import { KadDHT } from '@libp2p/kad-dht' +import server from './server.js' const log = console.log @@ -154,7 +140,7 @@ export default async function main (processArgs: string[]) { }) .parse() - const daemon = await createLibp2pServer(new Multiaddr(argv.listen), argv) + const daemon = await server.createLibp2pServer(new Multiaddr(argv.listen), argv) await daemon.start() if (argv.quiet !== true) { @@ -163,107 +149,6 @@ export default async function main (processArgs: string[]) { } } -export async function createLibp2pServer (listenAddr: Multiaddr, argv: any): Promise { - // Minimum libp2p setup. - const options: Libp2pOptions = { - addresses: { - listen: argv.hostAddrs.split(','), - announce: argv.announceAddrs.split(',') - }, - - transports: [new TCP(), new WebSockets()], - connectionEncryption: [new Noise()], - streamMuxers: [new Mplex()], - peerDiscovery: [] - } - - // Load key file as peer ID. - if (argv.id != null) { - const marshaledKey: Buffer = await fs.readFile(argv.id) - const unmarshaledKey = await unmarshalPrivateKey(marshaledKey) - const peerId = await createFromPrivKey(unmarshaledKey) - - options.peerId = peerId - } - - // Enable bootstrap peers. - if (argv.bootstrap === true && options.peerDiscovery != null) { - options.peerDiscovery.push( - new Bootstrap({ - interval: argv.discoveryInterval, - list: argv.bootstrapPeers.split(',') - }) - ) - } - - // Configure PubSub - if (argv.pubsub === true) { - // Router implementation. - switch (argv.pubsubRouter) { - case 'gossipsub': - options.pubsub = new GossipSub({ allowPublishToZeroPeers: true }) - break - case 'floodsub': - options.pubsub = new FloodSub() - break - default: - throw new Error('invalid pubsubRouter type') - } - - // Peer discovery - if (argv.pubsubDiscovery === true && options.peerDiscovery != null) { - options.peerDiscovery.push( - new PubSubPeerDiscovery({ interval: argv.discoveryInterval }) - ) - } - } - - // Enable DHT - if (argv.dht === true) { - options.dht = new KadDHT() - } - - // Configure PSK - if (argv.psk != null) { - const swarmKey: Buffer = await fs.readFile(argv.psk) - - options.connectionProtector = new PreSharedKeyConnectionProtector({ - psk: swarmKey - }) - } - - // Configure relay - if (argv.relay === true) { - options.relay = { - enabled: true - } - - if (argv.relayAuto === true) { - options.relay.autoRelay = { - enabled: true, - maxListeners: argv.relayAutoListeners - } - } - - if (argv.relayHop === true) { - options.relay.hop = { - enabled: true - } - } - - if (argv.relayAdvertise === true) { - options.relay.advertise = { - enabled: true - } - } - } - - const libp2p: Libp2p = await createLibp2p(options) - const daemon: Libp2pServer = createServer(listenAddr, libp2p) - - return daemon -} - if (esMain(import.meta)) { main(process.argv) .catch((err) => { diff --git a/packages/libp2p-daemon/src/server.ts b/packages/libp2p-daemon/src/server.ts new file mode 100644 index 00000000..7dbe2958 --- /dev/null +++ b/packages/libp2p-daemon/src/server.ts @@ -0,0 +1,119 @@ +import { promises as fs } from 'fs' +import type { Multiaddr } from '@multiformats/multiaddr' +import { createServer, Libp2pServer } from '@libp2p/daemon-server' +import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' +import { PreSharedKeyConnectionProtector } from 'libp2p/pnet' +import { Noise } from '@chainsafe/libp2p-noise' +import { Mplex } from '@libp2p/mplex' +import { TCP } from '@libp2p/tcp' +import { WebSockets } from '@libp2p/websockets' +import { Bootstrap } from '@libp2p/bootstrap' +import { GossipSub } from '@chainsafe/libp2p-gossipsub' +import { FloodSub } from '@libp2p/floodsub' +import { PubSubPeerDiscovery } from '@libp2p/pubsub-peer-discovery' +import { unmarshalPrivateKey } from '@libp2p/crypto/keys' +import { createFromPrivKey } from '@libp2p/peer-id-factory' +import { KadDHT } from '@libp2p/kad-dht' + +export default { + createLibp2pServer: async function (listenAddr: Multiaddr, argv: any): Promise { + // Minimum libp2p setup. + const options: Libp2pOptions = { + addresses: { + listen: argv.hostAddrs.split(','), + announce: argv.announceAddrs.split(',') + }, + + transports: [new TCP(), new WebSockets()], + connectionEncryption: [new Noise()], + streamMuxers: [new Mplex()], + peerDiscovery: [] + } + + // Load key file as peer ID. + if (argv.id != null) { + const marshaledKey: Buffer = await fs.readFile(argv.id) + const unmarshaledKey = await unmarshalPrivateKey(marshaledKey) + const peerId = await createFromPrivKey(unmarshaledKey) + + options.peerId = peerId + } + + // Enable bootstrap peers. + if (argv.bootstrap === true && options.peerDiscovery != null) { + options.peerDiscovery.push( + new Bootstrap({ + interval: argv.discoveryInterval, + list: argv.bootstrapPeers.split(',') + }) + ) + } + + // Configure PubSub + if (argv.pubsub === true) { + // Router implementation. + switch (argv.pubsubRouter) { + case 'gossipsub': + options.pubsub = new GossipSub({ allowPublishToZeroPeers: true }) + break + case 'floodsub': + options.pubsub = new FloodSub() + break + default: + throw new Error('invalid pubsubRouter type') + } + + // Peer discovery + if (argv.pubsubDiscovery === true && options.peerDiscovery != null) { + options.peerDiscovery.push( + new PubSubPeerDiscovery({ interval: argv.discoveryInterval }) + ) + } + } + + // Enable DHT + if (argv.dht === true) { + options.dht = new KadDHT() + } + + // Configure PSK + if (argv.psk != null) { + const swarmKey: Buffer = await fs.readFile(argv.psk) + + options.connectionProtector = new PreSharedKeyConnectionProtector({ + psk: swarmKey + }) + } + + // Configure relay + if (argv.relay === true) { + options.relay = { + enabled: true + } + + if (argv.relayAuto === true) { + options.relay.autoRelay = { + enabled: true, + maxListeners: argv.relayAutoListeners + } + } + + if (argv.relayHop === true) { + options.relay.hop = { + enabled: true + } + } + + if (argv.relayAdvertise === true) { + options.relay.advertise = { + enabled: true + } + } + } + + const libp2p: Libp2p = await createLibp2p(options) + const daemon: Libp2pServer = createServer(listenAddr, libp2p) + + return daemon + } +} From ab644294af9b4b1a635bd8f99a1db8f1ffb2a09c Mon Sep 17 00:00:00 2001 From: saul Date: Tue, 13 Sep 2022 12:27:58 +1200 Subject: [PATCH 29/39] Fix cli tests. --- packages/libp2p-daemon/test/cli.spec.ts | 31 ++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/libp2p-daemon/test/cli.spec.ts b/packages/libp2p-daemon/test/cli.spec.ts index 64ece212..f9f6612c 100644 --- a/packages/libp2p-daemon/test/cli.spec.ts +++ b/packages/libp2p-daemon/test/cli.spec.ts @@ -2,37 +2,40 @@ import { expect } from 'aegir/chai' import sinon from 'sinon' +import { Multiaddr } from '@multiformats/multiaddr' import cli from '../src/index.js' +import server from '../src/server.js' describe('cli', () => { - const daemon = { createDaemon: (options: any) => {} } - afterEach(() => { sinon.restore() }) it('should create a daemon with default options', async () => { - sinon.stub(daemon, 'createDaemon').callsFake((options) => { + sinon.stub(server, 'createLibp2pServer').callsFake(async (ma, options) => { + const bootstrapPeers: string = '/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ,/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dnsaddr/bootstrap.libp2p.io/p2p/QmZa1sAxajnQjVM8WjWXoMbmPd7NsWhfKsPkErzpm9wGkp,/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt' + expect(options).to.include({ b: false, bootstrap: false, - 'bootstrap-peers': '', - bootstrapPeers: '', - hostAddrs: '', + 'bootstrap-peers': bootstrapPeers, + bootstrapPeers, + hostAddrs: '/ip4/0.0.0.0/tcp/0', announceAddrs: '', 'conn-mgr': false, connMgr: false, dht: false, 'dht-client': false, dhtClient: false, - id: '', q: false, quiet: false, - listen: '/unix/tmp/p2pd.sock' + listen: '/ip4/127.0.0.1/tcp/1234' }) + return { - start: () => {}, - stop: () => {} + start: async () => {}, + stop: async () => {}, + getMultiaddr: () => new Multiaddr() } }) @@ -43,7 +46,7 @@ describe('cli', () => { }) it('should be able to specify options', async () => { - sinon.stub(daemon, 'createDaemon').callsFake((options) => { + sinon.stub(server, 'createLibp2pServer').callsFake(async (ma, options) => { expect(options).to.include({ b: true, bootstrap: true, @@ -63,9 +66,11 @@ describe('cli', () => { // UNIX sockets are not supported by @libp2p/tcp yet... listen: '/ip4/127.0.0.1/tcp/1234' }) + return { - start: () => {}, - stop: () => {} + start: async () => {}, + stop: async () => {}, + getMultiaddr: () => new Multiaddr() } }) From 086e4dd8e2d09eba7363a0b47a1c0e2144d5c4a4 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 15 Sep 2022 10:45:30 +1200 Subject: [PATCH 30/39] Bump @libp2p/tcp. --- packages/libp2p-daemon-client/package.json | 2 +- packages/libp2p-daemon-server/package.json | 2 +- packages/libp2p-daemon/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/libp2p-daemon-client/package.json b/packages/libp2p-daemon-client/package.json index 04d72cb5..1ad5840a 100644 --- a/packages/libp2p-daemon-client/package.json +++ b/packages/libp2p-daemon-client/package.json @@ -139,7 +139,7 @@ "@libp2p/interface-peer-info": "^1.0.1", "@libp2p/logger": "^2.0.0", "@libp2p/peer-id": "^1.1.10", - "@libp2p/tcp": "^3.0.0", + "@libp2p/tcp": "^3.1.0", "@multiformats/multiaddr": "^10.1.8", "err-code": "^3.0.1", "it-stream-types": "^1.0.4", diff --git a/packages/libp2p-daemon-server/package.json b/packages/libp2p-daemon-server/package.json index 69c59c84..6eced46c 100644 --- a/packages/libp2p-daemon-server/package.json +++ b/packages/libp2p-daemon-server/package.json @@ -147,7 +147,7 @@ "@libp2p/interfaces": "^3.0.2", "@libp2p/logger": "^2.0.0", "@libp2p/peer-id": "^1.1.10", - "@libp2p/tcp": "^3.0.0", + "@libp2p/tcp": "^3.1.0", "@multiformats/multiaddr": "^10.1.8", "it-drain": "^1.0.5", "it-length-prefixed": "^8.0.2", diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index e7bb9884..7262f7a8 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -146,7 +146,7 @@ "@libp2p/mplex": "^5.2.1", "@libp2p/peer-id-factory": "^1.0.18", "@libp2p/pubsub-peer-discovery": "^6.0.2", - "@libp2p/tcp": "^3.0.6", + "@libp2p/tcp": "^3.1.0", "@libp2p/websockets": "^3.0.3", "@multiformats/multiaddr": "^10.1.8", "es-main": "^1.0.2", From e3e12beb330887613ce6253d0200cb82ac8d75a8 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 15 Sep 2022 10:58:21 +1200 Subject: [PATCH 31/39] Add missing deps. --- packages/libp2p-daemon/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index 7262f7a8..632fe749 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -139,12 +139,15 @@ "@chainsafe/libp2p-gossipsub": "^4.1.1", "@chainsafe/libp2p-noise": "^8.0.1", "@libp2p/bootstrap": "^2.0.0", + "@libp2p/connection": "^4.0.1", "@libp2p/crypto": "^1.0.4", "@libp2p/daemon-server": "^2.0.0", "@libp2p/floodsub": "^3.0.7", "@libp2p/kad-dht": "^3.0.4", "@libp2p/mplex": "^5.2.1", "@libp2p/peer-id-factory": "^1.0.18", + "@libp2p/peer-record": "^4.0.2", + "@libp2p/peer-store": "^3.1.3", "@libp2p/pubsub-peer-discovery": "^6.0.2", "@libp2p/tcp": "^3.1.0", "@libp2p/websockets": "^3.0.3", From 111784dbed07f9f6d822f65015357ea6366280d3 Mon Sep 17 00:00:00 2001 From: saul Date: Thu, 15 Sep 2022 11:08:11 +1200 Subject: [PATCH 32/39] Set the default listen address to a unix domain socket. --- packages/libp2p-daemon/src/index.ts | 4 +--- packages/libp2p-daemon/test/cli.spec.ts | 10 +++------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 4c1fcc91..a3b34cfd 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -14,9 +14,7 @@ export default async function main (processArgs: string[]) { .option('listen', { desc: 'daemon control listen multiaddr', type: 'string', - // default: '/unix/tmp/p2pd.sock' - // UNIX sockets are not supported by @libp2p/tcp yet... - default: '/ip4/127.0.0.1/tcp/1234' + default: '/unix/tmp/p2pd.sock' }) .option('quiet', { alias: 'q', diff --git a/packages/libp2p-daemon/test/cli.spec.ts b/packages/libp2p-daemon/test/cli.spec.ts index f9f6612c..e873d257 100644 --- a/packages/libp2p-daemon/test/cli.spec.ts +++ b/packages/libp2p-daemon/test/cli.spec.ts @@ -29,7 +29,7 @@ describe('cli', () => { dhtClient: false, q: false, quiet: false, - listen: '/ip4/127.0.0.1/tcp/1234' + listen: '/unix/tmp/p2pd.sock' }) return { @@ -62,9 +62,7 @@ describe('cli', () => { id: '/path/to/key', q: true, quiet: true, - // listen: '/unix/tmp/d.sock' - // UNIX sockets are not supported by @libp2p/tcp yet... - listen: '/ip4/127.0.0.1/tcp/1234' + listen: '/unix/tmp/d.sock' }) return { @@ -86,9 +84,7 @@ describe('cli', () => { '--dhtClient=true', '--quiet=true', '--id=/path/to/key', - // '--listen=/unix/tmp/d.sock' - // UNIX sockets are not supported by @libp2p/tcp yet... - '--listen=/ip4/127.0.0.1/tcp/1234' + '--listen=/unix/tmp/d.sock' ]) }) }) From 47cb01058b81bf7d17034b8a42889436a534be58 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 2 Nov 2022 11:49:12 +1300 Subject: [PATCH 33/39] Fix type issue. --- packages/libp2p-daemon-protocol/src/stream-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/libp2p-daemon-protocol/src/stream-handler.ts b/packages/libp2p-daemon-protocol/src/stream-handler.ts index 6bbb1f9c..484283ee 100644 --- a/packages/libp2p-daemon-protocol/src/stream-handler.ts +++ b/packages/libp2p-daemon-protocol/src/stream-handler.ts @@ -54,7 +54,7 @@ export class StreamHandler { */ rest (): Duplex { this.shake.rest() - return this.shake.stream as Duplex + return this.shake.stream } /** From 41a1d6156e48fb98d9919dc98c15a798972353e9 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 2 Nov 2022 12:58:58 +1300 Subject: [PATCH 34/39] Bump packages. --- packages/libp2p-daemon/package.json | 35 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/libp2p-daemon/package.json b/packages/libp2p-daemon/package.json index 05836123..b1f2ca9d 100644 --- a/packages/libp2p-daemon/package.json +++ b/packages/libp2p-daemon/package.json @@ -136,25 +136,24 @@ "release": "aegir release" }, "dependencies": { - "@chainsafe/libp2p-gossipsub": "^4.1.1", - "@chainsafe/libp2p-noise": "^8.0.1", - "@libp2p/bootstrap": "^2.0.0", - "@libp2p/connection": "^4.0.1", - "@libp2p/crypto": "^1.0.4", - "@libp2p/daemon-server": "^2.0.0", - "@libp2p/floodsub": "^3.0.7", - "@libp2p/kad-dht": "^3.0.4", - "@libp2p/mplex": "^5.2.1", - "@libp2p/peer-id-factory": "^1.0.18", - "@libp2p/peer-record": "^4.0.2", - "@libp2p/peer-store": "^3.1.3", - "@libp2p/pubsub-peer-discovery": "^6.0.2", - "@libp2p/tcp": "^3.1.0", - "@libp2p/websockets": "^3.0.3", - "@multiformats/multiaddr": "^11.0.0", - "@libp2p/daemon-server": "^3.0.0", + "@chainsafe/libp2p-gossipsub": "^5.0.0", + "@chainsafe/libp2p-noise": "^10.0.0", + "@libp2p/bootstrap": "^5.0.0", + "@libp2p/connection": "^4.0.2", + "@libp2p/crypto": "^1.0.7", + "@libp2p/daemon-server": "^3.0.5", + "@libp2p/floodsub": "^5.0.0", + "@libp2p/kad-dht": "^5.0.1", + "@libp2p/mplex": "^7.0.0", + "@libp2p/peer-id-factory": "^1.0.19", + "@libp2p/peer-record": "^4.0.4", + "@libp2p/peer-store": "^5.0.0", + "@libp2p/pubsub-peer-discovery": "^7.0.0", + "@libp2p/tcp": "^5.0.1", + "@libp2p/websockets": "^5.0.0", + "@multiformats/multiaddr": "^11.0.6", "es-main": "^1.0.2", - "libp2p": "^0.39.0", + "libp2p": "^0.40.0", "yargs": "^17.3.1" }, "devDependencies": { From 3c2da35dde4a84894abcc02cda251d97e39aefd1 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 2 Nov 2022 12:59:13 +1300 Subject: [PATCH 35/39] Fix multiaddr in tests. --- packages/libp2p-daemon/test/cli.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/libp2p-daemon/test/cli.spec.ts b/packages/libp2p-daemon/test/cli.spec.ts index e873d257..71a2d013 100644 --- a/packages/libp2p-daemon/test/cli.spec.ts +++ b/packages/libp2p-daemon/test/cli.spec.ts @@ -2,7 +2,7 @@ import { expect } from 'aegir/chai' import sinon from 'sinon' -import { Multiaddr } from '@multiformats/multiaddr' +import { multiaddr } from '@multiformats/multiaddr' import cli from '../src/index.js' import server from '../src/server.js' @@ -35,7 +35,7 @@ describe('cli', () => { return { start: async () => {}, stop: async () => {}, - getMultiaddr: () => new Multiaddr() + getMultiaddr: () => multiaddr() } }) @@ -68,7 +68,7 @@ describe('cli', () => { return { start: async () => {}, stop: async () => {}, - getMultiaddr: () => new Multiaddr() + getMultiaddr: () => multiaddr() } }) From a16d9d7b5d0888448d415a33723476934adad3e3 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 2 Nov 2022 13:04:07 +1300 Subject: [PATCH 36/39] Migrate server to libp2p 0.40. --- packages/libp2p-daemon/src/server.ts | 46 +++++++++++++++------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/packages/libp2p-daemon/src/server.ts b/packages/libp2p-daemon/src/server.ts index 7dbe2958..390bfa55 100644 --- a/packages/libp2p-daemon/src/server.ts +++ b/packages/libp2p-daemon/src/server.ts @@ -2,18 +2,18 @@ import { promises as fs } from 'fs' import type { Multiaddr } from '@multiformats/multiaddr' import { createServer, Libp2pServer } from '@libp2p/daemon-server' import { Libp2p, createLibp2p, Libp2pOptions } from 'libp2p' -import { PreSharedKeyConnectionProtector } from 'libp2p/pnet' -import { Noise } from '@chainsafe/libp2p-noise' -import { Mplex } from '@libp2p/mplex' -import { TCP } from '@libp2p/tcp' -import { WebSockets } from '@libp2p/websockets' -import { Bootstrap } from '@libp2p/bootstrap' -import { GossipSub } from '@chainsafe/libp2p-gossipsub' -import { FloodSub } from '@libp2p/floodsub' -import { PubSubPeerDiscovery } from '@libp2p/pubsub-peer-discovery' +import { preSharedKey } from 'libp2p/pnet' +import { noise } from '@chainsafe/libp2p-noise' +import { mplex } from '@libp2p/mplex' +import { tcp } from '@libp2p/tcp' +import { webSockets } from '@libp2p/websockets' +import { bootstrap } from '@libp2p/bootstrap' +import { gossipsub } from '@chainsafe/libp2p-gossipsub' +import { floodsub } from '@libp2p/floodsub' +import { pubsubPeerDiscovery } from '@libp2p/pubsub-peer-discovery' import { unmarshalPrivateKey } from '@libp2p/crypto/keys' import { createFromPrivKey } from '@libp2p/peer-id-factory' -import { KadDHT } from '@libp2p/kad-dht' +import { kadDHT } from '@libp2p/kad-dht' export default { createLibp2pServer: async function (listenAddr: Multiaddr, argv: any): Promise { @@ -24,9 +24,9 @@ export default { announce: argv.announceAddrs.split(',') }, - transports: [new TCP(), new WebSockets()], - connectionEncryption: [new Noise()], - streamMuxers: [new Mplex()], + transports: [tcp(), webSockets()], + connectionEncryption: [noise()], + streamMuxers: [mplex()], peerDiscovery: [] } @@ -42,8 +42,8 @@ export default { // Enable bootstrap peers. if (argv.bootstrap === true && options.peerDiscovery != null) { options.peerDiscovery.push( - new Bootstrap({ - interval: argv.discoveryInterval, + bootstrap({ + timeout: argv.discoveryInterval, list: argv.bootstrapPeers.split(',') }) ) @@ -54,10 +54,10 @@ export default { // Router implementation. switch (argv.pubsubRouter) { case 'gossipsub': - options.pubsub = new GossipSub({ allowPublishToZeroPeers: true }) + options.pubsub = gossipsub({ allowPublishToZeroPeers: true }) break case 'floodsub': - options.pubsub = new FloodSub() + options.pubsub = floodsub() break default: throw new Error('invalid pubsubRouter type') @@ -65,22 +65,24 @@ export default { // Peer discovery if (argv.pubsubDiscovery === true && options.peerDiscovery != null) { - options.peerDiscovery.push( - new PubSubPeerDiscovery({ interval: argv.discoveryInterval }) - ) + const discovery = pubsubPeerDiscovery({ interval: argv.discoveryInterval }) + + // @libp2p/pubsub-peer-discovery at version 7.0.0 seems to have type compatibility problems. + // @ts-expect-error + options.peerDiscovery.push(discovery) } } // Enable DHT if (argv.dht === true) { - options.dht = new KadDHT() + options.dht = kadDHT() } // Configure PSK if (argv.psk != null) { const swarmKey: Buffer = await fs.readFile(argv.psk) - options.connectionProtector = new PreSharedKeyConnectionProtector({ + options.connectionProtector = preSharedKey({ psk: swarmKey }) } From 2b75a0e222a483d8e26357d1693bf232dd91c07d Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 2 Nov 2022 13:06:00 +1300 Subject: [PATCH 37/39] Prevent importsNotUsedAsValues error in libp2p-gossipsub. --- packages/libp2p-daemon/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/tsconfig.json b/packages/libp2p-daemon/tsconfig.json index 6b7a2ea4..da56af63 100644 --- a/packages/libp2p-daemon/tsconfig.json +++ b/packages/libp2p-daemon/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "aegir/src/config/tsconfig.aegir.json", "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "importsNotUsedAsValues": "remove" }, "include": [ "src", From c8f7438610c9af29ef667ec6fc13980a7615cade Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 2 Nov 2022 13:07:43 +1300 Subject: [PATCH 38/39] Fix linting in libp2p-daemon-client. --- packages/libp2p-daemon-client/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/libp2p-daemon-client/src/index.ts b/packages/libp2p-daemon-client/src/index.ts index 58629773..7f67c033 100644 --- a/packages/libp2p-daemon-client/src/index.ts +++ b/packages/libp2p-daemon-client/src/index.ts @@ -195,7 +195,7 @@ class Client implements DaemonClient { throw errcode(new Error(response.error?.msg ?? 'Open stream failed'), 'ERR_OPEN_STREAM_FAILED') } - return sh.rest() as Duplex + return sh.rest() } /** @@ -228,7 +228,7 @@ class Client implements DaemonClient { throw errcode(new Error('Incorrect protocol'), 'ERR_OPEN_STREAM_FAILED') } - await handler(sh.rest() as Duplex) + await handler(sh.rest()) }) .finally(() => { connection.close() From 8e58de144d97430508d4beba14b30f2d7f8b5604 Mon Sep 17 00:00:00 2001 From: saul Date: Wed, 2 Nov 2022 13:10:32 +1300 Subject: [PATCH 39/39] Change hostAddrs default to localhost. --- packages/libp2p-daemon/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/libp2p-daemon/src/index.ts b/packages/libp2p-daemon/src/index.ts index 4407ff09..85aef49c 100755 --- a/packages/libp2p-daemon/src/index.ts +++ b/packages/libp2p-daemon/src/index.ts @@ -29,7 +29,7 @@ export default async function main (processArgs: string[]) { .option('hostAddrs', { desc: 'Comma separated list of multiaddrs the host should listen on', type: 'string', - default: '/ip4/0.0.0.0/tcp/0' + default: '/ip4/127.0.0.1/tcp/0' }) .option('announceAddrs', { desc: 'Comma separated list of multiaddrs the host should announce to the network',