.aegir.js

@@ -6,7 +6,9 @@ const server = createServer()
 
 module.exports = {
   hooks: {
-    pre: server.start.bind(server),
-    post: server.stop.bind(server)
+    browser: {
+      pre: () => server.start(),
+      post: () => server.stop()
+    }
   }
 }

.gitignore

@@ -61,4 +61,5 @@ typings/
 .env
 
 yarn.lock
-package-lock.json
+package-lock.json
+dist/

CHANGELOG.md

@@ -1,3 +1,63 @@
+<a name="0.4.3"></a>
+## [0.4.3](https://github.com/libp2p/js-libp2p-delegated-content-routing/compare/v0.4.2...v0.4.3) (2020-02-04)
+
+
+
+<a name="0.4.2"></a>
+## [0.4.2](https://github.com/libp2p/js-libp2p-delegated-content-routing/compare/v0.4.1...v0.4.2) (2019-12-20)
+
+
+
+<a name="0.4.1"></a>
+## [0.4.1](https://github.com/libp2p/js-libp2p-delegated-content-routing/compare/v0.4.0...v0.4.1) (2019-12-01)
+
+
+### Bug Fixes
+
+* async it refs ([#19](https://github.com/libp2p/js-libp2p-delegated-content-routing/issues/19)) ([b2690c7](https://github.com/libp2p/js-libp2p-delegated-content-routing/commit/b2690c7))
+
+
+
+<a name="0.4.0"></a>
+# [0.4.0](https://github.com/libp2p/js-libp2p-delegated-content-routing/compare/v0.3.1...v0.4.0) (2019-11-29)
+
+
+### Chores
+
+* rename timeout option ([#18](https://github.com/libp2p/js-libp2p-delegated-content-routing/issues/18)) ([22ff40c](https://github.com/libp2p/js-libp2p-delegated-content-routing/commit/22ff40c))
+
+
+### BREAKING CHANGES
+
+* maxTimeout option renamed to timeout
+
+
+
+<a name="0.3.1"></a>
+## [0.3.1](https://github.com/libp2p/js-libp2p-delegated-content-routing/compare/v0.3.0...v0.3.1) (2019-07-24)
+
+
+### Bug Fixes
+
+* limit concurrent HTTP requests ([#16](https://github.com/libp2p/js-libp2p-delegated-content-routing/issues/16)) ([55d6108](https://github.com/libp2p/js-libp2p-delegated-content-routing/commit/55d6108)), closes [#12](https://github.com/libp2p/js-libp2p-delegated-content-routing/issues/12) [/github.com/libp2p/js-libp2p-delegated-content-routing/issues/12#issuecomment-514591525](https://github.com//github.com/libp2p/js-libp2p-delegated-content-routing/issues/12/issues/issuecomment-514591525)
+
+
+
+<a name="0.3.0"></a>
+# [0.3.0](https://github.com/libp2p/js-libp2p-delegated-content-routing/compare/v0.2.3...v0.3.0) (2019-07-12)
+
+
+### Features
+
+* refactor to use async/await ([#7](https://github.com/libp2p/js-libp2p-delegated-content-routing/issues/7)) ([0e2d91f](https://github.com/libp2p/js-libp2p-delegated-content-routing/commit/0e2d91f))
+
+
+### BREAKING CHANGES
+
+* API refactored to use async/await
+
+
+
 <a name="0.2.3"></a>
 ## [0.2.3](https://github.com/libp2p/js-libp2p-delegated-content-routing/compare/v0.2.2...v0.2.3) (2019-07-11)
 

README.md

@@ -1,7 +1,17 @@
 # js-libp2p-delegated-content-routing
 
+[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://protocol.ai)
+[![](https://img.shields.io/badge/project-libp2p-yellow.svg?style=flat-square)](http://libp2p.io/)
+[![](https://img.shields.io/badge/freenode-%23libp2p-yellow.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23libp2p)
+[![Discourse posts](https://img.shields.io/discourse/https/discuss.libp2p.io/posts.svg)](https://discuss.libp2p.io)
+[![](https://img.shields.io/codecov/c/github/libp2p/js-libp2p-delegated-content-routing.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-delegated-content-routing)
+[![](https://img.shields.io/travis/libp2p/js-libp2p-delegated-content-routing.svg?style=flat-square)](https://travis-ci.com/libp2p/js-libp2p-delegated-content-routing)
+[![Dependency Status](https://david-dm.org/libp2p/js-libp2p-delegated-content-routing.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-delegated-content-routing)
+
 Leverage other peers in the network to perform Content Routing calls.
 
+Requires access to `/api/v0/dht/findprovs` and `/api/v0/refs` HTTP API endpoints of the delegate node.
+
 ## Lead Maintainer
 
 [Jacob Heun](https://github.com/jacobheun)
@@ -18,22 +28,14 @@ const routing = new DelegatedContentRouing(peerId, {
   port: 443,
   host: 'ipfs.io'
 })
+const cid = new CID('QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv')
 
-routing.findProviders(key, (err, peerInfos) => {
-  if (err) {
-    return console.error(err)
-  }
-
-  console.log('found peers', peerInfos)
-})
-
-routing.provide(key, (err) => {
-  if (err) {
-    return console.error(err)
-  }
+for await (const peerInfo of routing.findProviders(cid)) {
+  console.log('found peer', peerInfo)
+}
 
-  console.log('providing %s', key)
-})
+await routing.provide(cid)
+console.log('providing %s', cid.toBaseEncodedString())
 ```
 
 ## License

package.json

@@ -1,6 +1,6 @@
 {
   "name": "libp2p-delegated-content-routing",
-  "version": "0.2.3",
+  "version": "0.4.3",
   "description": "Leverage other peers in the libp2p network to perform Content Routing calls.",
   "leadMaintainer": "Jacob Heun <jacobheun@gmail.com>",
   "main": "src/index.js",
@@ -18,23 +18,29 @@
     "coverage": "aegir coverage"
   },
   "devDependencies": {
-    "aegir": "^19.0.5",
+    "aegir": "^20.4.1",
     "chai": "^4.2.0",
     "cids": "^0.7.1",
-    "go-ipfs-dep": "^0.4.21",
-    "ipfsd-ctl": "^0.43.0"
+    "go-ipfs-dep": "^0.4.23",
+    "ipfsd-ctl": "~0.47.4",
+    "peer-id": "^0.13.7"
   },
   "dependencies": {
-    "async": "^2.6.2",
-    "ipfs-http-client": "^33.0.2",
-    "multiaddr": "^6.1.0",
-    "peer-id": "^0.12.2",
-    "peer-info": "^0.15.1"
+    "debug": "^4.1.1",
+    "ipfs-http-client": "^42.0.0",
+    "it-all": "^1.0.0",
+    "multiaddr": "^7.2.1",
+    "p-defer": "^3.0.0",
+    "p-queue": "^6.2.1",
+    "peer-info": "^0.17.1"
   },
   "contributors": [
     "Alan Shaw <alan.shaw@protocol.ai>",
+    "Alex Potsides <alex@achingbrain.net>",
     "David Dias <daviddias.p@gmail.com>",
     "Friedel Ziegelmayer <dignifiedquire@gmail.com>",
-    "Jacob Heun <jacobheun@gmail.com>"
+    "Jacob Heun <jacobheun@gmail.com>",
+    "Marcin Rataj <lidel@lidel.org>",
+    "Vasco Santos <vasco.santos@moxy.studio>"
   ]
 }

src/index.js

@@ -1,31 +1,26 @@
 'use strict'
 
-const dht = require('ipfs-http-client/src/dht')
-const swarm = require('ipfs-http-client/src/swarm')
-const refs = require('ipfs-http-client/src/files-regular/refs')
-const defaultConfig = require('ipfs-http-client/src/utils/default-config')
-const series = require('async/series')
-const parallel = require('async/parallel')
-const reflect = require('async/reflect')
-const multiaddr = require('multiaddr')
-
-const DEFAULT_MAX_TIMEOUT = 30e3 // 30 second default
+const debug = require('debug')
+const PeerId = require('peer-id')
+const PeerInfo = require('peer-info')
+const createFindProvs = require('ipfs-http-client/src/dht/find-provs')
+const createRefs = require('ipfs-http-client/src/refs')
+
+const { default: PQueue } = require('p-queue')
+const all = require('it-all')
+const defer = require('p-defer')
+
+const log = debug('libp2p-delegated-content-routing')
+log.error = debug('libp2p-delegated-content-routing:error')
+
+const DEFAULT_TIMEOUT = 30e3 // 30 second default
 const DEFAULT_IPFS_API = {
   protocol: 'https',
   port: 443,
-  host: 'ipfs.io'
+  host: 'node0.delegate.ipfs.io'
 }
 
-const DEFAULT_BOOSTRAP_NODES = [
-  '/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
-  '/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
-  '/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM',
-  '/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu',
-  '/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
-  '/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
-  '/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-  '/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
-]
+const CONCURRENT_HTTP_REQUESTS = 4
 
 /**
  * An implementation of content routing, using a delegated peer.
@@ -36,20 +31,27 @@ class DelegatedContentRouting {
    *
    * @param {PeerID} peerId - the id of the node that is using this routing.
    * @param {object} [api] - (Optional) the api endpoint of the delegated node to use.
-   * @param {Array<Multiaddr>} [bootstrappers] - (Optional) list of bootstrapper nodes we are connected to.
    */
-  constructor (peerId, api, bootstrappers) {
+  constructor (peerId, api) {
     if (peerId == null) {
       throw new Error('missing self peerId')
     }
 
-    this.api = Object.assign({}, defaultConfig(), DEFAULT_IPFS_API, api)
-    this.dht = dht(this.api)
-    this.swarm = swarm(this.api)
-    this.refs = refs(this.api)
-
+    this.api = Object.assign({}, DEFAULT_IPFS_API, api)
+    this.dht = { findProvs: createFindProvs(this.api) }
+    this.refs = createRefs(this.api)
     this.peerId = peerId
-    this.bootstrappers = bootstrappers || DEFAULT_BOOSTRAP_NODES.map((addr) => multiaddr(addr))
+
+    // limit concurrency to avoid request flood in web browser
+    // https://github.com/libp2p/js-libp2p-delegated-content-routing/issues/12
+    const concurrency = { concurrency: CONCURRENT_HTTP_REQUESTS }
+    this._httpQueue = new PQueue(concurrency)
+    // sometimes refs requests take long time, they need separate queue
+    // to not suffocate regular bussiness
+    this._httpQueueRefs = new PQueue(Object.assign({}, concurrency, {
+      concurrency: 2
+    }))
+    log(`enabled DelegatedContentRouting via ${this.api.protocol}://${this.api.host}:${this.api.port}`)
   }
 
   /**
@@ -59,64 +61,59 @@ class DelegatedContentRouting {
    *
    * @param {CID} key
    * @param {object} options
-   * @param {number} options.maxTimeout How long the query can take. Defaults to 30 seconds
-   * @param {function(Error, Array<PeerInfo>)} callback
-   * @returns {void}
+   * @param {number} options.timeout How long the query can take. Defaults to 30 seconds
+   * @returns {AsyncIterable<PeerInfo>}
    */
-  findProviders (key, options, callback) {
-    if (typeof options === 'function') {
-      callback = options
-      options = {}
-    } else if (typeof options === 'number') { // This will be deprecated in a next release
-      options = {
-        maxTimeout: options
-      }
-    } else {
-      options = options || {}
-    }
+  async * findProviders (key, options = {}) {
+    const keyString = `${key}`
+    log('findProviders starts:', keyString)
+    options.timeout = options.timeout || DEFAULT_TIMEOUT
+
+    const onStart = defer()
+    const onFinish = defer()
 
-    options.maxTimeout = options.maxTimeout || DEFAULT_MAX_TIMEOUT
+    this._httpQueue.add(() => {
+      onStart.resolve()
+      return onFinish.promise
+    })
+
+    try {
+      await onStart.promise
 
-    this.dht.findProvs(key.toString(), {
-      timeout: `${options.maxTimeout}ms` // The api requires specification of the time unit (s/ms)
-    }, callback)
+      const providers = this.dht.findProvs(key, {
+        numProviders: options.numProviders,
+        timeout: `${options.timeout}ms` // The api requires specification of the time unit (s/ms)
+      })
+
+      for await (const { id, addrs } of providers) {
+        const peerInfo = new PeerInfo(PeerId.createFromCID(id))
+        addrs.forEach(addr => peerInfo.multiaddrs.add(addr))
+        yield peerInfo
+      }
+    } finally {
+      onFinish.resolve()
+      log('findProviders finished:', keyString)
+    }
   }
 
   /**
    * Announce to the network that the delegated node can provide the given key.
    *
    * Currently this uses the following hack
-   * - call swarm.connect on the delegated node to us, to ensure we are connected
-   * - call refs --recursive on the delegated node, so it fetches the content
+   * - delegate is one of bootstrap nodes, so we are always connected to it
+   * - call refs on the delegated node, so it fetches the content
    *
    * @param {CID} key
    * @param {function(Error)} callback
-   * @returns {void}
+   * @returns {Promise<void>}
    */
-  provide (key, callback) {
-    const addrs = this.bootstrappers.map((addr) => {
-      return addr.encapsulate(`/p2p-circuit/ipfs/${this.peerId.toB58String()}`)
-    })
-
-    series([
-      (cb) => parallel(addrs.map((addr) => {
-        return reflect((cb) => this.swarm.connect(addr.toString(), cb))
-      }), (err, results) => {
-        if (err) {
-          return cb(err)
-        }
-
-        // only some need to succeed
-        const success = results.filter((res) => res.error == null)
-        if (success.length === 0) {
-          return cb(new Error('unable to swarm.connect using p2p-circuit'))
-        }
-        cb()
-      }),
-      (cb) => {
-        this.refs(key.toString(), { recursive: true }, cb)
-      }
-    ], (err) => callback(err))
+  async provide (key) {
+    const keyString = `${key}`
+    log('provide starts:', keyString)
+    const results = await this._httpQueueRefs.add(() =>
+      all(this.refs(keyString, { recursive: false }))
+    )
+    log('provide finished:', keyString, results)
   }
 }