feat: update dht (#1009)

Changes dht creation to use factory function and updates docs

BREAKING CHANGE: libp2p-kad-dht has a new event-based API which is exposed as `_dht`

Author: achingbrain

.github/workflows/main.yml

@@ -17,7 +17,6 @@ jobs:
         node-version: 14
     - run: npm install
     - run: npx aegir lint
-    - uses: gozala/typescript-error-reporter-action@v1.0.8
     - run: npx aegir build
     - run: npx aegir dep-check
     - uses: ipfs/aegir/actions/bundle-size@v32.1.0
@@ -67,4 +66,4 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: npm install
-      - run: cd node_modules/interop-libp2p && yarn && LIBP2P_JS=${GITHUB_WORKSPACE}/src/index.js npx aegir test -t node --bail -- --exit
+      - run: npm run test:interop -- --bail -- --exit

doc/CONFIGURATION.md

@@ -1,37 +1,37 @@
-# 
-
-- [Configuration](#configuration)
-  - [Overview](#overview)
-  - [Modules](#modules)
-    - [Transport](#transport)
-    - [Stream Multiplexing](#stream-multiplexing)
-    - [Connection Encryption](#connection-encryption)
-    - [Peer Discovery](#peer-discovery)
-    - [Content Routing](#content-routing)
-    - [Peer Routing](#peer-routing)
-    - [DHT](#dht)
-    - [Pubsub](#pubsub)
-  - [Customizing libp2p](#customizing-libp2p)
-    - [Examples](#examples)
-      - [Basic setup](#basic-setup)
-      - [Customizing Peer Discovery](#customizing-peer-discovery)
-      - [Setup webrtc transport and discovery](#setup-webrtc-transport-and-discovery)
-      - [Customizing Pubsub](#customizing-pubsub)
-      - [Customizing DHT](#customizing-dht)
-      - [Setup with Content and Peer Routing](#setup-with-content-and-peer-routing)
-      - [Setup with Relay](#setup-with-relay)
-      - [Setup with Auto Relay](#setup-with-auto-relay)
-      - [Setup with Keychain](#setup-with-keychain)
-      - [Configuring Dialing](#configuring-dialing)
-      - [Configuring Connection Manager](#configuring-connection-manager)
-      - [Configuring Transport Manager](#configuring-transport-manager)
-      - [Configuring Metrics](#configuring-metrics)
-      - [Configuring PeerStore](#configuring-peerstore)
-      - [Customizing Transports](#customizing-transports)
-      - [Configuring the NAT Manager](#configuring-the-nat-manager)
-        - [Browser support](#browser-support)
-        - [UPnP and NAT-PMP](#upnp-and-nat-pmp)
-  - [Configuration examples](#configuration-examples)
+#
+
+- [Overview](#overview)
+- [Modules](#modules)
+  - [Transport](#transport)
+  - [Stream Multiplexing](#stream-multiplexing)
+  - [Connection Encryption](#connection-encryption)
+  - [Peer Discovery](#peer-discovery)
+  - [Content Routing](#content-routing)
+  - [Peer Routing](#peer-routing)
+  - [DHT](#dht)
+  - [Pubsub](#pubsub)
+- [Customizing libp2p](#customizing-libp2p)
+  - [Examples](#examples)
+    - [Basic setup](#basic-setup)
+    - [Customizing Peer Discovery](#customizing-peer-discovery)
+    - [Setup webrtc transport and discovery](#setup-webrtc-transport-and-discovery)
+    - [Customizing Pubsub](#customizing-pubsub)
+    - [Customizing DHT](#customizing-dht)
+    - [Setup with Content and Peer Routing](#setup-with-content-and-peer-routing)
+    - [Setup with Relay](#setup-with-relay)
+    - [Setup with Auto Relay](#setup-with-auto-relay)
+    - [Setup with Keychain](#setup-with-keychain)
+    - [Configuring Dialing](#configuring-dialing)
+    - [Configuring Connection Manager](#configuring-connection-manager)
+    - [Configuring Transport Manager](#configuring-transport-manager)
+    - [Configuring Metrics](#configuring-metrics)
+    - [Configuring PeerStore](#configuring-peerstore)
+    - [Customizing Transports](#customizing-transports)
+    - [Configuring the NAT Manager](#configuring-the-nat-manager)
+      - [Browser support](#browser-support)
+      - [UPnP and NAT-PMP](#upnp-and-nat-pmp)
+    - [Configuring protocol name](#configuring-protocol-name)
+- [Configuration examples](#configuration-examples)
 
 ## Overview
 
@@ -374,11 +374,7 @@ const node = await Libp2p.create({
     dht: {                        // The DHT options (and defaults) can be found in its documentation
       kBucketSize: 20,
       enabled: true,              // This flag is required for DHT to run (disabled by default)
-      randomWalk: {
-        enabled: true,            // Allows to disable discovery (enabled by default)
-        interval: 300e3,
-        timeout: 10e3
-      }
+      clientMode: false           // Whether to run the WAN DHT in client or server mode (default: client mode)
     }
   }
 })
@@ -788,7 +784,7 @@ By default under nodejs libp2p will attempt to use [UPnP](https://en.wikipedia.o
 
 #### Configuring protocol name
 
-Changing the protocol name prefix can isolate default public network (IPFS) for custom purposes. 
+Changing the protocol name prefix can isolate default public network (IPFS) for custom purposes.
 
 ```js
 const node = await Libp2p.create({

examples/delegated-routing/package.json

@@ -7,7 +7,7 @@
     "libp2p": "github:libp2p/js-libp2p#master",
     "libp2p-delegated-content-routing": "~0.2.2",
     "libp2p-delegated-peer-routing": "~0.2.2",
-    "libp2p-kad-dht": "~0.14.12",
+    "libp2p-kad-dht": "^0.26.5",
     "libp2p-mplex": "~0.8.5",
     "libp2p-secio": "~0.11.1",
     "libp2p-webrtc-star": "~0.15.8",

examples/transports/README.md

@@ -91,7 +91,7 @@ const concat = require('it-concat')
 const MPLEX = require('libp2p-mplex')
 ```
 
-We are going to reuse the `createNode` function from step 1, but this time add a stream multiplexer from `libp2p-mplex`. 
+We are going to reuse the `createNode` function from step 1, but this time add a stream multiplexer from `libp2p-mplex`.
 ```js
 const createNode = async () => {
   const node = await Libp2p.create({

package.json

@@ -41,6 +41,7 @@
     "test:node": "aegir test -t node -f \"./test/**/*.{node,spec}.js\"",
     "test:browser": "aegir test -t browser",
     "test:examples": "cd examples && npm run test:all",
+    "test:interop": "LIBP2P_JS=$PWD npx aegir test -t node -f ./node_modules/libp2p-interop/test/*",
     "prepare": "aegir build --no-bundle",
     "release": "aegir release -t node -t browser",
     "release-minor": "aegir release --type minor -t node -t browser",
@@ -142,7 +143,6 @@
     "buffer": "^6.0.3",
     "datastore-core": "^6.0.7",
     "delay": "^5.0.0",
-    "interop-libp2p": "^0.4.0",
     "into-stream": "^7.0.0",
     "ipfs-http-client": "^52.0.2",
     "it-concat": "^2.0.0",
@@ -155,7 +155,8 @@
     "libp2p-floodsub": "^0.27.0",
     "libp2p-gossipsub": "^0.11.0",
     "libp2p-interfaces-compliance-tests": "^1.0.0",
-    "libp2p-kad-dht": "^0.24.2",
+    "libp2p-interop": "^0.5.0",
+    "libp2p-kad-dht": "^0.26.5",
     "libp2p-mdns": "^0.17.0",
     "libp2p-mplex": "^0.10.1",
     "libp2p-tcp": "^0.17.0",

src/content-routing/index.js

@@ -8,9 +8,10 @@ const {
   requirePeers,
   maybeLimitSource
 } = require('./utils')
-
+const drain = require('it-drain')
 const merge = require('it-merge')
 const { pipe } = require('it-pipe')
+const { DHTContentRouting } = require('../dht/dht-content-routing')
 
 /**
  * @typedef {import('peer-id')} PeerId
@@ -38,7 +39,7 @@ class ContentRouting {
 
     // If we have the dht, add it to the available content routers
     if (this.dht && libp2p._config.dht.enabled) {
-      this.routers.push(this.dht)
+      this.routers.push(new DHTContentRouting(this.dht))
     }
   }
 
@@ -91,12 +92,12 @@ class ContentRouting {
    * @param {number} [options.minPeers] - minimum number of peers required to successfully put
    * @returns {Promise<void>}
    */
-  put (key, value, options) {
+  async put (key, value, options) {
     if (!this.libp2p.isStarted() || !this.dht.isStarted) {
       throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED)
     }
 
-    return this.dht.put(key, value, options)
+    await drain(this.dht.put(key, value, options))
   }
 
   /**
@@ -108,12 +109,18 @@ class ContentRouting {
    * @param {number} [options.timeout] - optional timeout (default: 60000)
    * @returns {Promise<GetData>}
    */
-  get (key, options) {
+  async get (key, options) {
     if (!this.libp2p.isStarted() || !this.dht.isStarted) {
       throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED)
     }
 
-    return this.dht.get(key, options)
+    for await (const event of this.dht.get(key, options)) {
+      if (event.name === 'VALUE') {
+        return { from: event.peerId, val: event.value }
+      }
+    }
+
+    throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND)
   }
 
   /**
@@ -123,14 +130,33 @@ class ContentRouting {
    * @param {number} nVals
    * @param {Object} [options] - get options
    * @param {number} [options.timeout] - optional timeout (default: 60000)
-   * @returns {Promise<GetData[]>}
    */
-  async getMany (key, nVals, options) { // eslint-disable-line require-await
+  async * getMany (key, nVals, options) { // eslint-disable-line require-await
     if (!this.libp2p.isStarted() || !this.dht.isStarted) {
       throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED)
     }
 
-    return this.dht.getMany(key, nVals, options)
+    if (!nVals) {
+      return
+    }
+
+    let gotValues = 0
+
+    for await (const event of this.dht.get(key, options)) {
+      if (event.name === 'VALUE') {
+        yield { from: event.peerId, val: event.value }
+
+        gotValues++
+
+        if (gotValues === nVals) {
+          break
+        }
+      }
+    }
+
+    if (gotValues === 0) {
+      throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND)
+    }
   }
 }
 

src/dht/dht-content-routing.js

@@ -0,0 +1,44 @@
+'use strict'
+
+const drain = require('it-drain')
+
+/**
+ * @typedef {import('peer-id')} PeerId
+ * @typedef {import('libp2p-interfaces/src/content-routing/types').ContentRouting} ContentRoutingModule
+ * @typedef {import('multiformats/cid').CID} CID
+ */
+
+/**
+ * Wrapper class to convert events into returned values
+ *
+ * @implements {ContentRoutingModule}
+ */
+class DHTContentRouting {
+  /**
+   * @param {import('libp2p-kad-dht').DHT} dht
+   */
+  constructor (dht) {
+    this._dht = dht
+  }
+
+  /**
+   * @param {CID} cid
+   */
+  async provide (cid) {
+    await drain(this._dht.provide(cid))
+  }
+
+  /**
+   * @param {CID} cid
+   * @param {*} options
+   */
+  async * findProviders (cid, options) {
+    for await (const event of this._dht.findProviders(cid, options)) {
+      if (event.name === 'PROVIDER') {
+        yield * event.providers
+      }
+    }
+  }
+}
+
+module.exports = { DHTContentRouting }

src/dht/dht-peer-routing.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const errCode = require('err-code')
+const { messages, codes } = require('../errors')
+
+/**
+ * @typedef {import('peer-id')} PeerId
+ * @typedef {import('libp2p-interfaces/src/peer-routing/types').PeerRouting} PeerRoutingModule
+ */
+
+/**
+ * Wrapper class to convert events into returned values
+ *
+ * @implements {PeerRoutingModule}
+ */
+class DHTPeerRouting {
+  /**
+   * @param {import('libp2p-kad-dht').DHT} dht
+   */
+  constructor (dht) {
+    this._dht = dht
+  }
+
+  /**
+   * @param {PeerId} peerId
+   * @param {any} options
+   */
+  async findPeer (peerId, options = {}) {
+    for await (const event of this._dht.findPeer(peerId, options)) {
+      if (event.name === 'FINAL_PEER') {
+        return event.peer
+      }
+    }
+
+    throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND)
+  }
+
+  /**
+   * @param {Uint8Array} key
+   * @param {any} options
+   */
+  async * getClosestPeers (key, options = {}) {
+    for await (const event of this._dht.getClosestPeers(key, options)) {
+      if (event.name === 'PEER_RESPONSE') {
+        yield * event.closer
+      }
+    }
+  }
+}
+
+module.exports = { DHTPeerRouting }

src/errors.js

@@ -3,7 +3,8 @@
 exports.messages = {
   NOT_STARTED_YET: 'The libp2p node is not started yet',
   DHT_DISABLED: 'DHT is not available',
-  CONN_ENCRYPTION_REQUIRED: 'At least one connection encryption module is required'
+  CONN_ENCRYPTION_REQUIRED: 'At least one connection encryption module is required',
+  NOT_FOUND: 'Not found'
 }
 
 exports.codes = {
@@ -29,6 +30,7 @@ exports.codes = {
   ERR_INVALID_PARAMETERS: 'ERR_INVALID_PARAMETERS',
   ERR_INVALID_PEER: 'ERR_INVALID_PEER',
   ERR_MUXER_UNAVAILABLE: 'ERR_MUXER_UNAVAILABLE',
+  ERR_NOT_FOUND: 'ERR_NOT_FOUND',
   ERR_TIMEOUT: 'ERR_TIMEOUT',
   ERR_TRANSPORT_UNAVAILABLE: 'ERR_TRANSPORT_UNAVAILABLE',
   ERR_TRANSPORT_DIAL_FAILED: 'ERR_TRANSPORT_DIAL_FAILED',

src/index.js

@@ -301,14 +301,9 @@ class Libp2p extends EventEmitter {
     // dht provided components (peerRouting, contentRouting, dht)
     if (this._modules.dht) {
       const DHT = this._modules.dht
-      // @ts-ignore Object is not constructable
-      this._dht = new DHT({
+      // @ts-ignore TODO: types need fixing - DHT is an `object` which has no `create` method
+      this._dht = DHT.create({
         libp2p: this,
-        dialer: this.dialer,
-        peerId: this.peerId,
-        peerStore: this.peerStore,
-        registrar: this.registrar,
-        datastore: this.datastore,
         ...this._config.dht
       })
     }
@@ -624,7 +619,7 @@ class Libp2p extends EventEmitter {
 
     // DHT subsystem
     if (this._config.dht.enabled) {
-      this._dht && this._dht.start()
+      this._dht && await this._dht.start()
 
       // TODO: this should be modified once random-walk is used as
       // the other discovery modules

src/nat-manager.js

@@ -114,7 +114,7 @@ class NatManager {
       const client = this._getClient()
       const publicIp = this._externalIp || await client.externalIp()
 
-      // @ts-ignore isPrivate has no call signatures
+      // @ts-expect-error types are wrong
       if (isPrivateIp(publicIp)) {
         throw new Error(`${publicIp} is private - please set config.nat.externalIp to an externally routable IP or ensure you are not behind a double NAT`)
       }

src/peer-routing.js

@@ -21,6 +21,7 @@ const {
   clearDelayedInterval
 // @ts-ignore module with no types
 } = require('set-delayed-interval')
+const { DHTPeerRouting } = require('./dht/dht-peer-routing')
 
 /**
  * @typedef {import('peer-id')} PeerId
@@ -51,7 +52,7 @@ class PeerRouting {
 
     // If we have the dht, add it to the available peer routers
     if (libp2p._dht && libp2p._config.dht.enabled) {
-      this._routers.push(libp2p._dht)
+      this._routers.push(new DHTPeerRouting(libp2p._dht))
     }
 
     this._refreshManagerOptions = libp2p._options.peerRouting.refreshManager

test/ts-use/package.json

@@ -10,7 +10,7 @@
     "libp2p-delegated-peer-routing": "^0.10.0",
     "libp2p-gossipsub": "^0.9.0",
     "libp2p-interfaces": "^1.0.1",
-    "libp2p-kad-dht": "^0.23.1",
+    "libp2p-kad-dht": "^0.26.5",
     "libp2p-mplex": "^0.10.4",
     "@chainsafe/libp2p-noise": "^4.1.0",
     "libp2p-record": "^0.10.4",