Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 3250ff4

Browse files
achingbrainvasco-santossimonovic86
authoredOct 27, 2020
feat: enable custom formats for dag put and get (#3347)
Ensures we can use custom/extra IPLD formats with `ipfs.dag.get` and `ipfs.dag.put` over HTTP as well as directly into core. Adds an example to show how to do it. The basic idea is you configure your node with the extra formats and pass that node into the http server, so instead of having the IPLD format logic in the http method endpoint and in core it's just in core. The http client works the same as it did before. Co-authored-by: Vasco Santos <vasco.santos@moxy.studio> Co-authored-by: Janko Simonovic <simonovic86@gmail.com>
·
test-ipfs-example@3.0.0interface-ipfs-core@0.141.0
1 parent 4eb196c commit 3250ff4

File tree

16 files changed

+369
-109
lines changed

16 files changed

+369
-109
lines changed
 

‎examples/custom-ipld-formats/README.md

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Custom IPLD formats
2+
3+
This example shows you how to configure an IPFS daemon with the ability to load extra IPLD formats so you can use them in your applications.
4+
5+
## Before you start
6+
7+
First clone this repo, install dependencies in the project root and build the project.
8+
9+
```console
10+
$ git clone https://github.com/ipfs/js-ipfs.git
11+
$ cd js-ipfs
12+
$ npm install
13+
$ npm run build
14+
```
15+
16+
## Running the example
17+
18+
Running this example should result in metrics being logged out to the console every few seconds.
19+
20+
```
21+
> npm start
22+
```
23+
24+
## Play with the configuration!
25+
26+
By default, IPFS is only configured to support a few common IPLD formats. Your application may require extra or more esoteric formats, in which case you can configure your node to support them using `options.ipld.formats` passed to the client or an in-process node or even a daemon if you start it with a wrapper.
27+
28+
See the following files for different configuration:
29+
30+
* [./in-process-node.js](./in-process-node.js) for running an in-process node as part of your confiugration
31+
* [./daemon-node.js](./daemon-node.js) for running a node as a separate daemon process

‎examples/custom-ipld-formats/daemon-node.js

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// ordinarily we'd open a PR against the multicodec module to get our
2+
// codec number added but since we're just testing we shim our new
3+
// codec into the base-table.json file - this has to be done
4+
// before requiring other modules as the int table will become read-only
5+
const codecName = 'dag-test'
6+
const codecNumber = 392091
7+
8+
const baseTable = require('multicodec/src/base-table.json')
9+
baseTable[codecName] = codecNumber
10+
11+
// now require modules as usual
12+
const IPFSDaemon = require('ipfs-cli/src/daemon')
13+
const multihashing = require('multihashing-async')
14+
const multihash = multihashing.multihash
15+
const multicodec = require('multicodec')
16+
const CID = require('cids')
17+
const ipfsHttpClient = require('ipfs-http-client')
18+
const uint8ArrayToString = require('uint8arrays/to-string')
19+
20+
async function main () {
21+
// see https://github.com/ipld/interface-ipld-format for the interface definition
22+
const format = {
23+
codec: codecNumber,
24+
defaultHashAlg: multicodec.SHA2_256,
25+
util: {
26+
serialize (data) {
27+
return Buffer.from(JSON.stringify(data))
28+
},
29+
deserialize (buf) {
30+
return JSON.parse(uint8ArrayToString(buf))
31+
},
32+
async cid (buf) {
33+
const multihash = await multihashing(buf, format.defaultHashAlg)
34+
35+
return new CID(1, format.codec, multihash)
36+
}
37+
},
38+
resolver: {
39+
resolve: (buf, path) => {
40+
return {
41+
value: format.util.deserialize(buf),
42+
remainderPath: path
43+
}
44+
}
45+
}
46+
}
47+
48+
// start an IPFS Daemon
49+
const daemon = new IPFSDaemon({
50+
ipld: {
51+
formats: [
52+
format
53+
]
54+
}
55+
})
56+
await daemon.start()
57+
58+
// in another process:
59+
const client = ipfsHttpClient({
60+
url: `http://localhost:${daemon._httpApi._apiServers[0].info.port}`,
61+
ipld: {
62+
formats: [
63+
format
64+
]
65+
}
66+
})
67+
68+
const data = {
69+
hello: 'world'
70+
}
71+
72+
const cid = await client.dag.put(data, {
73+
format: codecName,
74+
hashAlg: multihash.codes[format.defaultHashAlg]
75+
})
76+
77+
console.info(`Put ${JSON.stringify(data)} = CID(${cid})`)
78+
79+
const {
80+
value
81+
} = await client.dag.get(cid)
82+
83+
console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`)
84+
85+
await daemon.stop()
86+
}
87+
88+
main()
89+
.catch(err => {
90+
console.error(err)
91+
process.exit(1)
92+
})
93+
.then(() => {
94+
// https://github.com/libp2p/js-libp2p/issues/779
95+
process.exit(0)
96+
})

‎examples/custom-ipld-formats/in-process-node.js

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// ordinarily we'd open a PR against the multicodec module to get our
2+
// codec number added but since we're just testing we shim our new
3+
// codec into the base-table.json file - this has to be done
4+
// before requiring other modules as the int table will become read-only
5+
const codecName = 'dag-test'
6+
const codecNumber = 392091
7+
8+
const baseTable = require('multicodec/src/base-table.json')
9+
baseTable[codecName] = codecNumber
10+
11+
// now require modules as usual
12+
const IPFS = require('ipfs-core')
13+
const multihashing = require('multihashing-async')
14+
const multicodec = require('multicodec')
15+
const CID = require('cids')
16+
17+
async function main () {
18+
// see https://github.com/ipld/interface-ipld-format for the interface definition
19+
const format = {
20+
codec: codecNumber,
21+
defaultHashAlg: multicodec.SHA2_256,
22+
util: {
23+
serialize (data) {
24+
return Buffer.from(JSON.stringify(data))
25+
},
26+
deserialize (buf) {
27+
return JSON.parse(buf.toString('utf8'))
28+
},
29+
async cid (buf) {
30+
const multihash = await multihashing(buf, format.defaultHashAlg)
31+
32+
return new CID(1, format.codec, multihash)
33+
}
34+
}
35+
}
36+
37+
const node = await IPFS.create({
38+
ipld: {
39+
formats: [
40+
format
41+
]
42+
}
43+
})
44+
45+
const data = {
46+
hello: 'world'
47+
}
48+
49+
const cid = await node.dag.put(data, {
50+
format: codecName,
51+
hashAlg: format.defaultHashAlg
52+
})
53+
54+
console.info(`Put ${JSON.stringify(data)} = CID(${cid})`)
55+
56+
const {
57+
value
58+
} = await node.dag.get(cid)
59+
60+
console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`)
61+
62+
await node.stop()
63+
}
64+
65+
main()
66+
.catch(err => {
67+
console.error(err)
68+
process.exit(1)
69+
})
70+
.then(() => {
71+
// https://github.com/libp2p/js-libp2p/issues/779
72+
process.exit(0)
73+
})

‎examples/custom-ipld-formats/package.json

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"name": "example-custom-ipld-formats",
3+
"version": "1.0.0",
4+
"private": true,
5+
"scripts": {
6+
"test": "test-ipfs-example"
7+
},
8+
"license": "MIT",
9+
"devDependencies": {
10+
"execa": "^4.0.3",
11+
"test-ipfs-example": "^2.0.3"
12+
},
13+
"dependencies": {
14+
"cids": "^1.0.0",
15+
"ipfs-cli": "0.0.1",
16+
"ipfs-core": "0.0.1",
17+
"ipfs-http-client": "^47.0.0",
18+
"multicodec": "^2.0.1",
19+
"multihashing-async": "^2.0.1",
20+
"uint8arrays": "^1.1.0"
21+
}
22+
}

‎examples/custom-ipld-formats/test.js

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
'use strict'
2+
3+
const path = require('path')
4+
const {
5+
waitForOutput
6+
} = require('test-ipfs-example/utils')
7+
8+
const testInProcessNode = async () => {
9+
await waitForOutput(
10+
'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' +
11+
'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'in-process-node.js'))
12+
}
13+
14+
const testDaemonNode = async () => {
15+
await waitForOutput(
16+
'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' +
17+
'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'daemon-node.js'))
18+
}
19+
20+
async function test () {
21+
console.info('Testing in-process node')
22+
await testInProcessNode()
23+
24+
console.info('Testing daemon node')
25+
await testDaemonNode()
26+
}
27+
28+
module.exports = test

‎packages/ipfs-cli/src/daemon.js

+5
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ class Daemon {
2828
}
2929
}
3030

31+
/**
32+
* Starts the IPFS HTTP server
33+
*
34+
* @returns {Promise<Daemon>}
35+
*/
3136
async start () {
3237
log('starting')
3338

‎packages/ipfs-core/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@
7777
"ipfs-unixfs-exporter": "^3.0.4",
7878
"ipfs-unixfs-importer": "^3.0.4",
7979
"ipfs-utils": "^4.0.0",
80-
"ipld": "^0.27.1",
80+
"ipld": "^0.27.2",
8181
"ipld-bitcoin": "^0.4.0",
8282
"ipld-block": "^0.10.1",
8383
"ipld-dag-cbor": "^0.17.0",

‎packages/ipfs-core/src/components/start.js

+1
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ function createApi ({
294294
id: Components.id({ peerId, libp2p }),
295295
init: async () => { throw new AlreadyInitializedError() }, // eslint-disable-line require-await
296296
isOnline,
297+
ipld,
297298
key: {
298299
export: Components.key.export({ keychain }),
299300
gen: Components.key.gen({ keychain }),

‎packages/ipfs-http-client/src/dag/get.js

+10-21
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,29 @@
11
'use strict'
22

3-
const dagPB = require('ipld-dag-pb')
4-
const dagCBOR = require('ipld-dag-cbor')
5-
const raw = require('ipld-raw')
63
const configure = require('../lib/configure')
4+
const multicodec = require('multicodec')
5+
const loadFormat = require('../lib/ipld-formats')
76

8-
const resolvers = {
9-
'dag-cbor': dagCBOR.resolver,
10-
'dag-pb': dagPB.resolver,
11-
raw: raw.resolver
12-
}
13-
14-
module.exports = configure((api, options) => {
15-
const getBlock = require('../block/get')(options)
16-
const dagResolve = require('./resolve')(options)
7+
module.exports = configure((api, opts) => {
8+
const getBlock = require('../block/get')(opts)
9+
const dagResolve = require('./resolve')(opts)
10+
const load = loadFormat(opts.ipld)
1711

1812
/**
1913
* @type {import('..').Implements<import('ipfs-core/src/components/dag/get')>}
2014
*/
2115
const get = async (cid, options = {}) => {
2216
const resolved = await dagResolve(cid, options)
2317
const block = await getBlock(resolved.cid, options)
24-
const dagResolver = resolvers[resolved.cid.codec]
2518

26-
if (!dagResolver) {
27-
throw Object.assign(
28-
new Error(`Missing IPLD format "${resolved.cid.codec}"`),
29-
{ missingMulticodec: resolved.cid.codec }
30-
)
31-
}
19+
const codecName = multicodec.getName(resolved.cid.code)
20+
const format = await load(codecName)
3221

33-
if (resolved.cid.codec === 'raw' && !resolved.remainderPath) {
22+
if (resolved.cid.code === multicodec.RAW && !resolved.remainderPath) {
3423
resolved.remainderPath = '/'
3524
}
3625

37-
return dagResolver.resolve(block.data, resolved.remainderPath)
26+
return format.resolver.resolve(block.data, resolved.remainderPath)
3827
}
3928

4029
return get

0 commit comments

Comments
 (0)