This repository has been archived by the owner on Feb 12, 2024. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: pubsub over gRPC Browsers can only have six concurrently open connections to a host name. Pubsub works over HTTP by holding a connection open per subscription, which means you can only subscribe six times before things start to hang. gRPC runs over websockets so doesn't have this limitation. This PR adds pubsub support to the gRPC server and `ipfs-client` module so you can subscribe to lots and lots of channels concurrently, working around the browser connection limitation. Refs: #3741
- Loading branch information
1 parent
5ab3ced
commit e7d5509
Showing
18 changed files
with
440 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
packages/ipfs-grpc-client/src/core-api/pubsub/subscribe.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| 'use strict' | ||
|
|
||
| const serverStreamToIterator = require('../../utils/server-stream-to-iterator') | ||
| const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') | ||
| const subscriptions = require('./subscriptions') | ||
| const defer = require('p-defer') | ||
|
|
||
| /** | ||
| * @param {import('@improbable-eng/grpc-web').grpc} grpc | ||
| * @param {*} service | ||
| * @param {import('../../types').Options} opts | ||
| */ | ||
| module.exports = function grpcPubsubSubscribe (grpc, service, opts) { | ||
| /** | ||
| * @type {import('ipfs-core-types/src/pubsub').API["subscribe"]} | ||
| */ | ||
| async function pubsubSubscribe (topic, handler, options = {}) { | ||
| const request = { | ||
| topic | ||
| } | ||
|
|
||
| const deferred = defer() | ||
|
|
||
| Promise.resolve().then(async () => { | ||
| try { | ||
| for await (const result of serverStreamToIterator(grpc, service, request, { | ||
| host: opts.url, | ||
| debug: Boolean(process.env.DEBUG), | ||
| metadata: options, | ||
| agent: opts.agent | ||
| })) { | ||
| if (result.handler) { | ||
| const subs = subscriptions.get(topic) || new Map() | ||
| subs.set(result.handler, handler) | ||
| subscriptions.set(topic, subs) | ||
|
|
||
| deferred.resolve() | ||
| } else { | ||
| handler({ | ||
| from: result.from, | ||
| seqno: result.seqno, | ||
| data: result.data, | ||
| topicIDs: result.topicIDs | ||
| }) | ||
| } | ||
| } | ||
| } catch (err) { | ||
| if (options && options.onError) { | ||
| options.onError(err) | ||
| } | ||
| } | ||
| }) | ||
|
|
||
| await deferred.promise | ||
| } | ||
|
|
||
| return withTimeoutOption(pubsubSubscribe) | ||
| } |
10 changes: 10 additions & 0 deletions
10
packages/ipfs-grpc-client/src/core-api/pubsub/subscriptions.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| 'use strict' | ||
|
|
||
| /** | ||
| * @typedef {import('ipfs-core-types/src/pubsub').MessageHandlerFn} Subscription | ||
| */ | ||
|
|
||
| /** @type {Map<string, Map<string, Subscription>>} */ | ||
| const subs = new Map() | ||
|
|
||
| module.exports = subs |
56 changes: 56 additions & 0 deletions
56
packages/ipfs-grpc-client/src/core-api/pubsub/unsubscribe.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| 'use strict' | ||
|
|
||
| const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') | ||
| const toHeaders = require('../../utils/to-headers') | ||
| const unaryToPromise = require('../../utils/unary-to-promise') | ||
| const subscriptions = require('./subscriptions') | ||
|
|
||
| /** | ||
| * @param {import('@improbable-eng/grpc-web').grpc} grpc | ||
| * @param {*} service | ||
| * @param {import('../../types').Options} opts | ||
| */ | ||
| module.exports = function grpcPubsubUnsubscribe (grpc, service, opts) { | ||
| /** | ||
| * @type {import('ipfs-core-types/src/pubsub').API["unsubscribe"]} | ||
| */ | ||
| async function pubsubUnsubscribe (topic, handler, options = {}) { | ||
| const handlers = [] | ||
| const subs = subscriptions.get(topic) | ||
|
|
||
| if (!subs) { | ||
| return | ||
| } | ||
|
|
||
| if (handler) { | ||
| for (const [key, value] of subs.entries()) { | ||
| if (value === handler) { | ||
| handlers.push(key) | ||
| } | ||
| } | ||
| } else { | ||
|
|
||
| } | ||
|
|
||
| const request = { | ||
| topic, | ||
| handlers | ||
| } | ||
|
|
||
| await unaryToPromise(grpc, service, request, { | ||
| host: opts.url, | ||
| metadata: toHeaders(options), | ||
| agent: opts.agent | ||
| }) | ||
|
|
||
| for (const handlerId of handlers) { | ||
| subs.delete(handlerId) | ||
| } | ||
|
|
||
| if (!subs.size) { | ||
| subscriptions.delete(topic) | ||
| } | ||
| } | ||
|
|
||
| return withTimeoutOption(pubsubUnsubscribe) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| syntax = "proto3"; | ||
|
|
||
| import "common.proto"; | ||
|
|
||
| package ipfs; | ||
|
|
||
| service PubSub { | ||
| rpc subscribe (SubscribeRequest) returns (stream SubscribeResponse) {} | ||
| rpc unsubscribe (UnSubscribeRequest) returns (UnSubscribeResponse) {} | ||
| } | ||
|
|
||
| message SubscribeRequest { | ||
| string topic = 1; | ||
| } | ||
|
|
||
| message SubscribeResponse { | ||
| string handler = 1; | ||
| string from = 2; | ||
| bytes seqno = 3; | ||
| bytes data = 4; | ||
| repeated string topicIDs = 5; | ||
| } | ||
|
|
||
| message UnSubscribeRequest { | ||
| string topic = 1; | ||
| repeated string handlers = 2; | ||
| } | ||
|
|
||
| message UnSubscribeResponse { | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.