Skip to content
This repository was archived by the owner on Aug 12, 2020. It is now read-only.

Commit 211e4e3

Browse files
pgtedaviddias
pgte
authored and
daviddias
committedNov 12, 2017
feat: exporter maxDepth (#197)
* big exporter overhaul: centralized dag reslving inside internal resolver stream * exporter: maxDepth instead of recursive flag * exporter: exposing name and depth * exporter: exposing dir size * tests: increased timeout for importing big file * tests: test exporter maxDepth * fixed linting error * exporter: exporting link size insteaf of node size to mimc go-ipfs * test: moving timeout def to the test top * tests: fixed this.timeout because arrow functions
·
v0.35.4v0.24.0
1 parent e0b9da3 commit 211e4e3

12 files changed

+219
-114
lines changed
 

‎package.json

-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@
7070
"pull-batch": "^1.0.0",
7171
"pull-block": "1.2.0",
7272
"pull-cat": "^1.1.11",
73-
"pull-defer": "~0.2.2",
7473
"pull-pair": "^1.1.0",
7574
"pull-paramap": "^1.2.2",
7675
"pull-pause": "0.0.1",

‎src/exporter/dir-flat.js

+15-17
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,38 @@
11
'use strict'
22

33
const pull = require('pull-stream')
4-
const paramap = require('pull-paramap')
5-
const CID = require('cids')
64
const cat = require('pull-cat')
75

86
// Logic to export a unixfs directory.
97
module.exports = dirExporter
108

11-
function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
9+
function dirExporter (node, name, path, pathRest, resolve, size, dag, parent, depth) {
1210
const accepts = pathRest[0]
1311

1412
const dir = {
15-
path: name,
16-
hash: node.multihash
13+
name: name,
14+
depth: depth,
15+
path: path,
16+
hash: node.multihash,
17+
size: node.size,
18+
type: 'dir'
1719
}
1820

1921
const streams = [
2022
pull(
2123
pull.values(node.links),
2224
pull.map((link) => ({
25+
depth: depth + 1,
26+
size: link.size,
27+
name: link.name,
28+
path: path + '/' + link.name,
29+
multihash: link.multihash,
2330
linkName: link.name,
24-
path: name + '/' + link.name,
25-
hash: link.multihash
31+
pathRest: pathRest.slice(1),
32+
type: 'dir'
2633
})),
2734
pull.filter((item) => accepts === undefined || item.linkName === accepts),
28-
paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
29-
if (err) {
30-
return cb(err)
31-
}
32-
33-
cb(null, resolve(n.value, accepts || item.path, pathRest, ipldResolver, name, parent))
34-
})),
35-
pull.flatten()
35+
resolve
3636
)
3737
]
3838

@@ -41,7 +41,5 @@ function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
4141
streams.unshift(pull.values([dir]))
4242
}
4343

44-
pathRest.shift()
45-
4644
return cat(streams)
4745
}

‎src/exporter/dir-hamt-sharded.js

+17-30
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
'use strict'
22

33
const pull = require('pull-stream')
4-
const paramap = require('pull-paramap')
5-
const CID = require('cids')
64
const cat = require('pull-cat')
75
const cleanHash = require('./clean-multihash')
86

97
// Logic to export a unixfs directory.
108
module.exports = shardedDirExporter
119

12-
function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
10+
function shardedDirExporter (node, name, path, pathRest, resolve, size, dag, parent, depth) {
1311
let dir
14-
if (!parent || parent.path !== name) {
15-
dir = [{
16-
path: name,
17-
hash: cleanHash(node.multihash)
18-
}]
12+
if (!parent || (parent.path !== path)) {
13+
dir = {
14+
name: name,
15+
depth: depth,
16+
path: path,
17+
hash: cleanHash(node.multihash),
18+
size: node.size,
19+
type: 'dir'
20+
}
1921
}
2022

2123
const streams = [
@@ -24,47 +26,32 @@ function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent
2426
pull.map((link) => {
2527
// remove the link prefix (2 chars for the bucket index)
2628
const p = link.name.substring(2)
27-
const pp = p ? name + '/' + p : name
29+
const pp = p ? path + '/' + p : path
2830
let accept = true
29-
let fromPathRest = false
3031

3132
if (p && pathRest.length) {
32-
fromPathRest = true
3333
accept = (p === pathRest[0])
3434
}
3535
if (accept) {
3636
return {
37-
fromPathRest: fromPathRest,
37+
depth: depth + 1,
3838
name: p,
3939
path: pp,
40-
hash: link.multihash,
41-
pathRest: p ? pathRest.slice(1) : pathRest
40+
multihash: link.multihash,
41+
pathRest: p ? pathRest.slice(1) : pathRest,
42+
parent: dir || parent
4243
}
4344
} else {
4445
return ''
4546
}
4647
}),
4748
pull.filter(Boolean),
48-
paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
49-
if (err) {
50-
return cb(err)
51-
}
52-
53-
cb(
54-
null,
55-
resolve(
56-
n.value,
57-
item.fromPathRest ? item.name : item.path,
58-
item.pathRest,
59-
ipldResolver,
60-
(dir && dir[0]) || parent))
61-
})),
62-
pull.flatten()
49+
resolve
6350
)
6451
]
6552

6653
if (!pathRest.length) {
67-
streams.unshift(pull.values(dir))
54+
streams.unshift(pull.values([dir]))
6855
}
6956

7057
return cat(streams)

‎src/exporter/file.js

+9-6
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ const pull = require('pull-stream')
77
const paramap = require('pull-paramap')
88

99
// Logic to export a single (possibly chunked) unixfs file.
10-
module.exports = (node, name, pathRest, ipldResolver) => {
10+
module.exports = (node, name, path, pathRest, resolve, size, dag, parent, depth) => {
1111
function getData (node) {
1212
try {
1313
const file = UnixFS.unmarshal(node.data)
@@ -20,14 +20,14 @@ module.exports = (node, name, pathRest, ipldResolver) => {
2020
function visitor (node) {
2121
return pull(
2222
pull.values(node.links),
23-
paramap((link, cb) => ipldResolver.get(new CID(link.multihash), cb)),
23+
paramap((link, cb) => dag.get(new CID(link.multihash), cb)),
2424
pull.map((result) => result.value)
2525
)
2626
}
2727

28-
const accepts = pathRest.shift()
28+
const accepts = pathRest[0]
2929

30-
if (accepts !== undefined && accepts !== name) {
30+
if (accepts !== undefined && accepts !== path) {
3131
return pull.empty()
3232
}
3333

@@ -38,9 +38,12 @@ module.exports = (node, name, pathRest, ipldResolver) => {
3838

3939
const file = UnixFS.unmarshal(node.data)
4040
return pull.values([{
41+
depth: depth,
4142
content: content,
42-
path: name,
43+
name: name,
44+
path: path,
4345
hash: node.multihash,
44-
size: file.fileSize()
46+
size: size || file.fileSize(),
47+
type: 'file'
4548
}])
4649
}

‎src/exporter/index.js

+55-17
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@
22

33
const pull = require('pull-stream')
44
const CID = require('cids')
5-
const pullDefer = require('pull-defer')
65

7-
const resolve = require('./resolve').resolve
6+
const createResolver = require('./resolve').createResolver
87

98
function pathBaseAndRest (path) {
109
// Buffer -> raw multihash or CID in buffer
@@ -36,28 +35,67 @@ function pathBaseAndRest (path) {
3635
}
3736
}
3837

39-
module.exports = (path, dag) => {
38+
const defaultOptions = {
39+
maxDepth: Infinity
40+
}
41+
42+
module.exports = (path, dag, _options) => {
43+
const options = Object.assign({}, defaultOptions, _options)
44+
45+
let dPath
4046
try {
41-
path = pathBaseAndRest(path)
47+
dPath = pathBaseAndRest(path)
4248
} catch (err) {
4349
return pull.error(err)
4450
}
4551

46-
const d = pullDefer.source()
52+
const pathLengthToCut = join(
53+
[dPath.base].concat(dPath.rest.slice(0, dPath.rest.length - 1))).length
4754

48-
const cid = new CID(path.base)
55+
return pull(
56+
pull.values([{
57+
multihash: new CID(dPath.base),
58+
name: dPath.base,
59+
path: dPath.base,
60+
pathRest: dPath.rest,
61+
depth: 0
62+
}]),
63+
createResolver(dag, options),
64+
pull.filter(Boolean),
65+
pull.map((node) => {
66+
return {
67+
depth: node.depth,
68+
name: node.name,
69+
path: finalPathFor(node),
70+
size: node.size,
71+
hash: node.hash || node.multihash,
72+
content: node.content,
73+
type: node.type
74+
}
75+
})
76+
)
4977

50-
dag.get(cid, (err, node) => {
51-
if (err) {
52-
return pull.error(err)
78+
function finalPathFor (node) {
79+
if (!dPath.rest.length) {
80+
return node.path
5381
}
54-
d.resolve(pull.values([node]))
55-
})
5682

57-
return pull(
58-
d,
59-
pull.map((result) => result.value),
60-
pull.map((node) => resolve(node, path.base, path.rest, dag)),
61-
pull.flatten()
62-
)
83+
let retPath = node.path.substring(pathLengthToCut)
84+
if (retPath.charAt(0) === '/') {
85+
retPath = retPath.substring(1)
86+
}
87+
if (!retPath) {
88+
retPath = dPath.rest[dPath.rest.length - 1] || dPath.base
89+
}
90+
return retPath
91+
}
92+
}
93+
94+
function join (paths) {
95+
return paths.reduce((acc, path) => {
96+
if (acc.length) {
97+
acc += '/'
98+
}
99+
return acc + path
100+
}, '')
63101
}

‎src/exporter/object.js

+16-21
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,29 @@
22

33
const CID = require('cids')
44
const pull = require('pull-stream')
5-
const pullDefer = require('pull-defer')
65

7-
module.exports = (node, name, pathRest, ipldResolver, resolve) => {
6+
module.exports = (node, name, path, pathRest, resolve, size, dag, parent, depth) => {
87
let newNode
98
if (pathRest.length) {
10-
const pathElem = pathRest.shift()
9+
const pathElem = pathRest[0]
1110
newNode = node[pathElem]
12-
const newName = name + '/' + pathElem
13-
if (CID.isCID(newNode)) {
14-
const d = pullDefer.source()
15-
ipldResolver.get(sanitizeCID(newNode), (err, newNode) => {
16-
if (err) {
17-
d.resolve(pull.error(err))
18-
} else {
19-
d.resolve(resolve(newNode.value, newName, pathRest, ipldResolver, node))
20-
}
21-
})
22-
return d
23-
} else if (newNode !== undefined) {
24-
return resolve(newNode, newName, pathRest, ipldResolver, node)
25-
} else {
11+
const newName = path + '/' + pathElem
12+
if (!newNode) {
2613
return pull.error('not found')
2714
}
15+
const isCID = CID.isCID(newNode)
16+
return pull(
17+
pull.values([{
18+
depth: depth,
19+
name: pathElem,
20+
path: newName,
21+
pathRest: pathRest.slice(1),
22+
multihash: isCID && newNode,
23+
object: !isCID && newNode,
24+
parent: parent
25+
}]),
26+
resolve)
2827
} else {
2928
return pull.error(new Error('invalid node type'))
3029
}
3130
}
32-
33-
function sanitizeCID (cid) {
34-
return new CID(cid.version, cid.codec, cid.multihash)
35-
}

‎src/exporter/resolve.js

+49-7
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
const UnixFS = require('ipfs-unixfs')
44
const pull = require('pull-stream')
5+
const paramap = require('pull-paramap')
6+
const CID = require('cids')
57

68
const resolvers = {
79
directory: require('./dir-flat'),
@@ -11,17 +13,53 @@ const resolvers = {
1113
}
1214

1315
module.exports = Object.assign({
14-
resolve: resolve,
16+
createResolver: createResolver,
1517
typeOf: typeOf
1618
}, resolvers)
1719

18-
function resolve (node, hash, pathRest, ipldResolver, parentNode) {
19-
const type = typeOf(node)
20-
const resolver = resolvers[type]
21-
if (!resolver) {
22-
return pull.error(new Error('Unkown node type ' + type))
20+
function createResolver (dag, options, depth, parent) {
21+
if (!depth) {
22+
depth = 0
23+
}
24+
25+
if (depth > options.maxDepth) {
26+
return pull.map(identity)
27+
}
28+
29+
return pull(
30+
paramap((item, cb) => {
31+
if ((typeof item.depth) !== 'number') {
32+
return pull.error(new Error('no depth'))
33+
}
34+
if (item.object) {
35+
return cb(null, resolveItem(item.object, item))
36+
}
37+
dag.get(new CID(item.multihash), (err, node) => {
38+
if (err) {
39+
return cb(err)
40+
}
41+
// const name = item.fromPathRest ? item.name : item.path
42+
cb(null, resolveItem(node.value, item))
43+
})
44+
}),
45+
pull.flatten(),
46+
pull.filter(Boolean),
47+
pull.filter((node) => node.depth <= options.maxDepth)
48+
)
49+
50+
function resolveItem (node, item) {
51+
return resolve(node, item.name, item.path, item.pathRest, item.size, dag, item.parent || parent, item.depth)
52+
}
53+
54+
function resolve (node, name, path, pathRest, size, dag, parentNode, depth) {
55+
const type = typeOf(node)
56+
const nodeResolver = resolvers[type]
57+
if (!nodeResolver) {
58+
return pull.error(new Error('Unkown node type ' + type))
59+
}
60+
const resolveDeep = createResolver(dag, options, depth, node)
61+
return nodeResolver(node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth)
2362
}
24-
return resolver(node, hash, pathRest, ipldResolver, resolve, parentNode)
2563
}
2664

2765
function typeOf (node) {
@@ -31,3 +69,7 @@ function typeOf (node) {
3169
return 'object'
3270
}
3371
}
72+
73+
function identity (o) {
74+
return o
75+
}

‎src/importer/tree-builder.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ function createTreeBuilder (ipldResolver, _options) {
9191
// ---- Add to tree
9292

9393
function addToTree (elem, callback) {
94-
const pathElems = elem.path.split('/').filter(notEmpty)
94+
const pathElems = (elem.path || '').split('/').filter(notEmpty)
9595
let parent = tree
9696
const lastIndex = pathElems.length - 1
9797

‎test/builder-dir-sharding.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ module.exports = (repo) => {
9191
expect(nodes[0].path).to.be.eql(expectedHash)
9292
expect(mh.toB58String(nodes[0].hash)).to.be.eql(expectedHash)
9393
expect(nodes[1].path).to.be.eql(expectedHash + '/b')
94-
expect(nodes[1].size).to.be.eql(21)
94+
expect(nodes[1].size).to.be.eql(29)
9595
pull(
9696
nodes[1].content,
9797
pull.collect(collected)

‎test/exporter-subtree.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ const exporter = unixFSEngine.exporter
1616
const smallFile = loadFixture(__dirname, 'fixtures/200Bytes.txt')
1717

1818
module.exports = (repo) => {
19-
describe('exporter', function () {
20-
this.timeout(10 * 1000)
19+
describe('exporter subtree', () => {
20+
// this.timeout(10 * 1000)
2121

2222
let ipldResolver
2323

‎test/exporter.js

+50-9
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ module.exports = (repo) => {
7373
)
7474
})
7575

76-
it('export a small file with links', (done) => {
76+
it('export a small file with links', function (done) {
77+
this.timeout(30 * 1000)
7778
const hash = 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q'
7879
pull(
7980
exporter(hash, ipldResolver),
@@ -83,9 +84,10 @@ module.exports = (repo) => {
8384
fileEql(files[0], bigFile, done)
8485
})
8586
)
86-
}).timeout(30 * 1000)
87+
})
8788

88-
it('export a small file with links using CID instead of multihash', (done) => {
89+
it('export a small file with links using CID instead of multihash', function (done) {
90+
this.timeout(30 * 1000)
8991
const cid = new CID('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q')
9092

9193
pull(
@@ -96,9 +98,10 @@ module.exports = (repo) => {
9698
fileEql(files[0], bigFile, done)
9799
})
98100
)
99-
}).timeout(30 * 1000)
101+
})
100102

101-
it('export a large file > 5mb', (done) => {
103+
it('export a large file > 5mb', function (done) {
104+
this.timeout(30 * 1000)
102105
const hash = 'QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE'
103106
pull(
104107
exporter(hash, ipldResolver),
@@ -109,16 +112,17 @@ module.exports = (repo) => {
109112
fileEql(files[0], null, done)
110113
})
111114
)
112-
}).timeout(30 * 1000)
115+
})
113116

114-
it('export a directory', (done) => {
117+
it('export a directory', function (done) {
118+
this.timeout(30 * 1000)
115119
const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
116120

117121
pull(
118122
exporter(hash, ipldResolver),
119123
pull.collect((err, files) => {
120-
files.forEach(file => expect(file).to.have.property('hash'))
121124
expect(err).to.not.exist()
125+
files.forEach(file => expect(file).to.have.property('hash'))
122126

123127
expect(
124128
files.map((file) => file.path)
@@ -149,7 +153,44 @@ module.exports = (repo) => {
149153
)
150154
})
151155
)
152-
}).timeout(30 * 1000)
156+
})
157+
158+
it('export a directory one deep', function (done) {
159+
this.timeout(30 * 1000)
160+
const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
161+
162+
pull(
163+
exporter(hash, ipldResolver, { maxDepth: 1 }),
164+
pull.collect((err, files) => {
165+
expect(err).to.not.exist()
166+
files.forEach(file => expect(file).to.have.property('hash'))
167+
168+
expect(
169+
files.map((file) => file.path)
170+
).to.be.eql([
171+
'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN',
172+
'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/200Bytes.txt',
173+
'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/dir-another',
174+
'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1'
175+
])
176+
177+
pull(
178+
pull.values(files),
179+
pull.map((file) => Boolean(file.content)),
180+
pull.collect((err, contents) => {
181+
expect(err).to.not.exist()
182+
expect(contents).to.be.eql([
183+
false,
184+
true,
185+
false,
186+
false
187+
])
188+
done()
189+
})
190+
)
191+
})
192+
)
193+
})
153194

154195
it('returns an empty stream for dir', (done) => {
155196
const hash = 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'

‎test/importer.js

+4-2
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,8 @@ module.exports = (repo) => {
298298
}
299299
})
300300

301-
it('file bigger than a single chunk', (done) => {
301+
it('file bigger than a single chunk', function (done) {
302+
this.timeout(60 * 1000)
302303
pull(
303304
pull.values([{
304305
path: '1.2MiB.txt',
@@ -313,7 +314,8 @@ module.exports = (repo) => {
313314
)
314315
})
315316

316-
it('file bigger than a single chunk inside a dir', (done) => {
317+
it('file bigger than a single chunk inside a dir', function (done) {
318+
this.timeout(60 * 1000)
317319
pull(
318320
pull.values([{
319321
path: 'foo-big/1.2MiB.txt',

0 commit comments

Comments
 (0)
This repository has been archived.