Skip to content

Commit 6aad264

Browse files
phatedphated
authoredJan 3, 2023
feat!: Replace glob with anymatch & custom directory walk (#118)
feat!: Combine GlobStream & GlobReadable into unified API
·
v8.0.2v8.0.0
1 parent 872a957 commit 6aad264

File tree

5 files changed

+477
-378
lines changed

5 files changed

+477
-378
lines changed
 

‎index.js

+261-47
Original file line numberDiff line numberDiff line change
@@ -1,79 +1,293 @@
11
'use strict';
22

3-
var Combine = require('ordered-read-streams');
4-
var unique = require('unique-stream');
5-
var pumpify = require('pumpify');
3+
var fs = require('fs');
4+
var path = require('path');
5+
var EventEmitter = require('events');
6+
7+
var fastq = require('fastq');
8+
var anymatch = require('anymatch');
9+
var Readable = require('readable-stream').Readable;
10+
var isGlob = require('is-glob');
11+
var globParent = require('glob-parent');
12+
var normalizePath = require('normalize-path');
613
var isNegatedGlob = require('is-negated-glob');
14+
var toAbsoluteGlob = require('@gulpjs/to-absolute-glob');
715

8-
var GlobStream = require('./readable');
16+
var globErrMessage1 = 'File not found with singular glob: ';
17+
var globErrMessage2 = ' (if this was purposeful, use `allowEmpty` option)';
918

10-
function globStream(globs, opt) {
11-
if (!opt) {
12-
opt = {};
19+
function isFound(glob) {
20+
// All globs are "found", while singular globs are only found when matched successfully
21+
// This is due to the fact that a glob can match any number of files (0..Infinity) but
22+
// a signular glob is always expected to match
23+
return isGlob(glob);
24+
}
25+
26+
function walkdir() {
27+
var readdirOpts = {
28+
withFileTypes: true,
29+
};
30+
31+
var ee = new EventEmitter();
32+
33+
var queue = fastq(readdir, 1);
34+
queue.drain = function () {
35+
ee.emit('end');
36+
};
37+
queue.error(onError);
38+
39+
function onError(err) {
40+
if (err) {
41+
ee.emit('error', err);
42+
}
1343
}
1444

15-
var ourOpt = Object.assign({}, opt);
16-
var ignore = ourOpt.ignore;
45+
ee.pause = function () {
46+
queue.pause();
47+
};
48+
ee.resume = function () {
49+
queue.resume();
50+
};
51+
ee.end = function () {
52+
queue.kill();
53+
};
54+
ee.walk = function (filepath) {
55+
queue.push(filepath);
56+
};
1757

18-
ourOpt.cwd = typeof ourOpt.cwd === 'string' ? ourOpt.cwd : process.cwd();
19-
ourOpt.dot = typeof ourOpt.dot === 'boolean' ? ourOpt.dot : false;
20-
ourOpt.silent = typeof ourOpt.silent === 'boolean' ? ourOpt.silent : true;
21-
ourOpt.cwdbase = typeof ourOpt.cwdbase === 'boolean' ? ourOpt.cwdbase : false;
22-
ourOpt.uniqueBy =
23-
typeof ourOpt.uniqueBy === 'string' || typeof ourOpt.uniqueBy === 'function'
24-
? ourOpt.uniqueBy
25-
: 'path';
58+
function readdir(filepath, cb) {
59+
fs.readdir(filepath, readdirOpts, onReaddir);
2660

27-
if (ourOpt.cwdbase) {
28-
ourOpt.base = ourOpt.cwd;
61+
function onReaddir(err, dirents) {
62+
if (err) {
63+
return cb(err);
64+
}
65+
66+
dirents.forEach(processDirent);
67+
68+
cb();
69+
}
70+
71+
function processDirent(dirent) {
72+
var nextpath = path.join(filepath, dirent.name);
73+
ee.emit('path', nextpath, dirent);
74+
75+
if (dirent.isDirectory()) {
76+
queue.push(nextpath);
77+
}
78+
}
79+
}
80+
81+
return ee;
82+
}
83+
84+
function validateGlobs(globs) {
85+
var hasPositiveGlob = false;
86+
87+
globs.forEach(validateGlobs);
88+
89+
function validateGlobs(globString, index) {
90+
if (typeof globString !== 'string') {
91+
throw new Error('Invalid glob at index ' + index);
92+
}
93+
94+
var result = isNegatedGlob(globString);
95+
if (result.negated === false) {
96+
hasPositiveGlob = true;
97+
}
98+
}
99+
100+
if (hasPositiveGlob === false) {
101+
throw new Error('Missing positive glob');
102+
}
103+
}
104+
105+
function validateOptions(opts) {
106+
if (typeof opts.cwd !== 'string') {
107+
throw new Error('The `cwd` option must be a string');
108+
}
109+
110+
if (typeof opts.dot !== 'boolean') {
111+
throw new Error('The `dot` option must be a boolean');
112+
}
113+
114+
if (typeof opts.cwdbase !== 'boolean') {
115+
throw new Error('The `cwdbase` option must be a boolean');
116+
}
117+
118+
if (
119+
typeof opts.uniqueBy !== 'string' &&
120+
typeof opts.uniqueBy !== 'function'
121+
) {
122+
throw new Error('The `uniqueBy` option must be a string or function');
123+
}
124+
125+
if (typeof opts.allowEmpty !== 'boolean') {
126+
throw new Error('The `allowEmpty` option must be a boolean');
127+
}
128+
129+
if (opts.base && typeof opts.base !== 'string') {
130+
throw new Error('The `base` option must be a string if specified');
29131
}
30-
// Normalize string `ignore` to array
31-
if (typeof ignore === 'string') {
32-
ignore = [ignore];
132+
133+
if (!Array.isArray(opts.ignore)) {
134+
throw new Error('The `ignore` option must be a string or array');
33135
}
34-
// Ensure `ignore` is an array
35-
if (!Array.isArray(ignore)) {
36-
ignore = [];
136+
}
137+
138+
function uniqueBy(comparator) {
139+
var seen = new Set();
140+
141+
if (typeof comparator === 'string') {
142+
return isUniqueByKey;
143+
} else {
144+
return isUniqueByFunc;
37145
}
38146

39-
// Only one glob no need to aggregate
147+
function isUnique(value) {
148+
if (seen.has(value)) {
149+
return false;
150+
} else {
151+
seen.add(value);
152+
return true;
153+
}
154+
}
155+
156+
function isUniqueByKey(obj) {
157+
return isUnique(obj[comparator]);
158+
}
159+
160+
function isUniqueByFunc(obj) {
161+
return isUnique(comparator(obj));
162+
}
163+
}
164+
165+
function globStream(globs, opt) {
40166
if (!Array.isArray(globs)) {
41167
globs = [globs];
42168
}
43169

44-
var positives = [];
45-
var negatives = [];
170+
validateGlobs(globs);
46171

47-
globs.forEach(sortGlobs);
172+
var ourOpt = Object.assign(
173+
{},
174+
{
175+
highWaterMark: 16,
176+
cwd: process.cwd(),
177+
dot: false,
178+
cwdbase: false,
179+
uniqueBy: 'path',
180+
allowEmpty: false,
181+
ignore: [],
182+
},
183+
opt
184+
);
185+
// Normalize `ignore` to array
186+
ourOpt.ignore =
187+
typeof ourOpt.ignore === 'string' ? [ourOpt.ignore] : ourOpt.ignore;
48188

49-
function sortGlobs(globString, index) {
50-
if (typeof globString !== 'string') {
51-
throw new Error('Invalid glob at index ' + index);
52-
}
189+
validateOptions(ourOpt);
190+
191+
var base = ourOpt.base;
192+
if (ourOpt.cwdbase) {
193+
base = ourOpt.cwd;
194+
}
195+
196+
var walker = walkdir();
53197

54-
var glob = isNegatedGlob(globString);
55-
var globArray = glob.negated ? negatives : positives;
198+
var stream = new Readable({
199+
objectMode: true,
200+
highWaterMark: ourOpt.highWaterMark,
201+
read: read,
202+
destroy: destroy,
203+
});
56204

57-
globArray.push(glob.pattern);
205+
// Remove path relativity to make globs make sense
206+
var ourGlobs = globs.map(resolveGlob);
207+
ourOpt.ignore = ourOpt.ignore.map(resolveGlob);
208+
209+
var found = ourGlobs.map(isFound);
210+
211+
var matcher = anymatch(ourGlobs, null, ourOpt);
212+
213+
var isUnique = uniqueBy(ourOpt.uniqueBy);
214+
215+
walker.on('path', onPath);
216+
walker.once('end', onEnd);
217+
walker.once('error', onError);
218+
walker.walk(ourOpt.cwd);
219+
220+
function read() {
221+
walker.resume();
58222
}
59223

60-
if (positives.length === 0) {
61-
throw new Error('Missing positive glob');
224+
function destroy(err) {
225+
walker.end();
226+
227+
process.nextTick(function () {
228+
if (err) {
229+
stream.emit('error', err);
230+
}
231+
stream.emit('close');
232+
});
233+
}
234+
235+
function resolveGlob(glob) {
236+
return toAbsoluteGlob(glob, ourOpt);
237+
}
238+
239+
function onPath(filepath, dirent) {
240+
var matchIdx = matcher(filepath, true);
241+
// If the matcher doesn't match (but it is a directory),
242+
// we want to add a trailing separator to check the match again
243+
if (matchIdx === -1 && dirent.isDirectory()) {
244+
matchIdx = matcher(filepath + path.sep, true);
245+
}
246+
if (matchIdx !== -1) {
247+
found[matchIdx] = true;
248+
249+
// Extract base path from glob
250+
var basePath = base || globParent(ourGlobs[matchIdx]);
251+
252+
var obj = {
253+
cwd: ourOpt.cwd,
254+
base: basePath,
255+
// We always want to normalize the path to posix-style slashes
256+
path: normalizePath(filepath, true),
257+
};
258+
259+
var unique = isUnique(obj);
260+
if (unique) {
261+
var drained = stream.push(obj);
262+
if (!drained) {
263+
walker.pause();
264+
}
265+
}
266+
}
62267
}
63268

64-
// Create all individual streams
65-
var streams = positives.map(streamFromPositive);
269+
function onEnd() {
270+
var destroyed = false;
66271

67-
// Then just pipe them to a single unique stream and return it
68-
var aggregate = new Combine(streams);
69-
var uniqueStream = unique(ourOpt.uniqueBy);
272+
found.forEach(function (matchFound, idx) {
273+
if (ourOpt.allowEmpty !== true && !matchFound) {
274+
destroyed = true;
275+
var err = new Error(globErrMessage1 + ourGlobs[idx] + globErrMessage2);
70276

71-
return pumpify.obj(aggregate, uniqueStream);
277+
return stream.destroy(err);
278+
}
279+
});
72280

73-
function streamFromPositive(positive) {
74-
var negativeGlobs = negatives.concat(ignore);
75-
return new GlobStream(positive, negativeGlobs, ourOpt);
281+
if (destroyed === false) {
282+
stream.push(null);
283+
}
76284
}
285+
286+
function onError(err) {
287+
stream.destroy(err);
288+
}
289+
290+
return stream;
77291
}
78292

79293
module.exports = globStream;

‎package.json

+6-8
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
"main": "index.js",
1616
"files": [
1717
"index.js",
18-
"readable.js",
1918
"LICENSE"
2019
],
2120
"scripts": {
@@ -24,15 +23,14 @@
2423
"test": "nyc mocha --async-only"
2524
},
2625
"dependencies": {
27-
"glob": "^8.0.3",
26+
"@gulpjs/to-absolute-glob": "^4.0.0",
27+
"anymatch": "^3.1.3",
28+
"fastq": "^1.13.0",
2829
"glob-parent": "^6.0.2",
30+
"is-glob": "^4.0.3",
2931
"is-negated-glob": "^1.0.0",
30-
"ordered-read-streams": "^1.0.1",
31-
"pumpify": "^2.0.1",
32-
"readable-stream": "^3.6.0",
33-
"remove-trailing-separator": "^1.1.0",
34-
"to-absolute-glob": "^3.0.0",
35-
"unique-stream": "^2.3.1"
32+
"normalize-path": "^3.0.0",
33+
"readable-stream": "^3.6.0"
3634
},
3735
"devDependencies": {
3836
"eslint": "^7.0.0",

‎readable.js

-116
This file was deleted.

‎test/index.js

+210-38
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
var expect = require('expect');
44
var miss = require('mississippi');
5+
var sinon = require('sinon');
6+
7+
// Need to wrap this to cause walker to emit an error
8+
var fs = require('fs');
59

610
var globStream = require('../');
711

@@ -119,7 +123,7 @@ describe('glob-stream', function () {
119123

120124
pipe(
121125
[
122-
globStream('./fixtures/has (parens)/*.dmc', { cwd: dir }),
126+
globStream('./fixtures/has \\(parens\\)/*.dmc', { cwd: dir }),
123127
concat(assert),
124128
],
125129
done
@@ -213,7 +217,7 @@ describe('glob-stream', function () {
213217
);
214218
});
215219

216-
it('properly orders objects when given multiple paths and specified base', function (done) {
220+
it('emits all objects (unordered) when given multiple paths and specified base', function (done) {
217221
var base = dir + '/fixtures';
218222

219223
var expected = [
@@ -242,13 +246,15 @@ describe('glob-stream', function () {
242246

243247
function assert(pathObjs) {
244248
expect(pathObjs.length).toEqual(3);
245-
expect(pathObjs).toEqual(expected);
249+
expect(pathObjs).toContainEqual(expected[0]);
250+
expect(pathObjs).toContainEqual(expected[1]);
251+
expect(pathObjs).toContainEqual(expected[2]);
246252
}
247253

248254
pipe([globStream(paths, { cwd: base, base: base }), concat(assert)], done);
249255
});
250256

251-
it('properly orders objects when given multiple paths and cwdbase', function (done) {
257+
it('emits all objects (unordered) when given multiple paths and cwdbase', function (done) {
252258
var base = dir + '/fixtures';
253259

254260
var expected = [
@@ -277,7 +283,9 @@ describe('glob-stream', function () {
277283

278284
function assert(pathObjs) {
279285
expect(pathObjs.length).toEqual(3);
280-
expect(pathObjs).toEqual(expected);
286+
expect(pathObjs).toContainEqual(expected[0]);
287+
expect(pathObjs).toContainEqual(expected[1]);
288+
expect(pathObjs).toContainEqual(expected[2]);
281289
}
282290

283291
pipe(
@@ -286,7 +294,7 @@ describe('glob-stream', function () {
286294
);
287295
});
288296

289-
it('properly orders objects when given multiple globs with globstars', function (done) {
297+
it('emits all objects (unordered) when given multiple globs with globstars', function (done) {
290298
var expected = [
291299
{
292300
cwd: dir,
@@ -324,13 +332,17 @@ describe('glob-stream', function () {
324332

325333
function assert(pathObjs) {
326334
expect(pathObjs.length).toEqual(5);
327-
expect(pathObjs).toEqual(expected);
335+
expect(pathObjs).toContainEqual(expected[0]);
336+
expect(pathObjs).toContainEqual(expected[1]);
337+
expect(pathObjs).toContainEqual(expected[2]);
338+
expect(pathObjs).toContainEqual(expected[3]);
339+
expect(pathObjs).toContainEqual(expected[4]);
328340
}
329341

330342
pipe([globStream(globs, { cwd: dir }), concat(assert)], done);
331343
});
332344

333-
it('properly orders objects when given multiple absolute paths and no cwd', function (done) {
345+
it('emits all objects (unordered) when given multiple absolute paths and no cwd', function (done) {
334346
var expected = [
335347
{
336348
cwd: process.cwd(),
@@ -357,7 +369,9 @@ describe('glob-stream', function () {
357369

358370
function assert(pathObjs) {
359371
expect(pathObjs.length).toEqual(3);
360-
expect(pathObjs).toEqual(expected);
372+
expect(pathObjs).toContainEqual(expected[0]);
373+
expect(pathObjs).toContainEqual(expected[1]);
374+
expect(pathObjs).toContainEqual(expected[2]);
361375
}
362376

363377
pipe([globStream(paths), concat(assert)], done);
@@ -640,6 +654,7 @@ describe('glob-stream', function () {
640654

641655
it('emits an error when a singular path in multiple paths not found', function (done) {
642656
function assert(err) {
657+
expect(err).toEqual(expect.anything());
643658
expect(err.toString()).toMatch(/File not found with singular glob/);
644659
done();
645660
}
@@ -689,8 +704,6 @@ describe('options', function () {
689704
var defaultedOpts = {
690705
cwd: process.cwd(),
691706
dot: false,
692-
silent: true,
693-
nonull: false,
694707
cwdbase: false,
695708
};
696709

@@ -703,33 +716,30 @@ describe('options', function () {
703716
pipe([stream, concat()], done);
704717
});
705718

706-
describe('silent', function () {
707-
it('accepts a boolean', function (done) {
708-
pipe(
709-
[
710-
globStream(dir + '/fixtures/stuff/run.dmc', { silent: false }),
711-
concat(),
712-
],
713-
done
714-
);
715-
});
716-
});
717-
718-
describe('nonull', function () {
719-
it('accepts a boolean', function (done) {
720-
pipe([globStream('notfound{a,b}', { nonull: true }), concat()], done);
721-
});
722-
723-
it('does not have any effect on our results', function (done) {
724-
function assert(pathObjs) {
725-
expect(pathObjs.length).toEqual(0);
726-
}
719+
it('throws on invalid options', function (done) {
720+
expect(function () {
721+
globStream('./fixtures/stuff/*.dmc', { cwd: 1234 });
722+
}).toThrow('must be a string');
723+
expect(function () {
724+
globStream('./fixtures/stuff/*.dmc', { dot: 1234 });
725+
}).toThrow('must be a boolean');
726+
expect(function () {
727+
globStream('./fixtures/stuff/*.dmc', { cwdbase: 1234 });
728+
}).toThrow('must be a boolean');
729+
expect(function () {
730+
globStream('./fixtures/stuff/*.dmc', { uniqueBy: 1234 });
731+
}).toThrow('must be a string or function');
732+
expect(function () {
733+
globStream('./fixtures/stuff/*.dmc', { allowEmpty: 1234 });
734+
}).toThrow('must be a boolean');
735+
expect(function () {
736+
globStream('./fixtures/stuff/*.dmc', { base: 1234 });
737+
}).toThrow('must be a string if specified');
738+
expect(function () {
739+
globStream('./fixtures/stuff/*.dmc', { ignore: 1234 });
740+
}).toThrow('must be a string or array');
727741

728-
pipe(
729-
[globStream('notfound{a,b}', { nonull: true }), concat(assert)],
730-
done
731-
);
732-
});
742+
done();
733743
});
734744

735745
describe('ignore', function () {
@@ -799,7 +809,7 @@ describe('options', function () {
799809
);
800810
});
801811

802-
it('merges ignore option and negative globs', function (done) {
812+
it('can use both ignore option and negative globs', function (done) {
803813
var globs = ['./fixtures/stuff/*.dmc', '!./fixtures/stuff/test.dmc'];
804814

805815
function assert(pathObjs) {
@@ -815,4 +825,166 @@ describe('options', function () {
815825
);
816826
});
817827
});
828+
829+
it('emits an error if there are no matches', function (done) {
830+
function assert(err) {
831+
expect(err.message).toMatch(/^File not found with singular glob/g);
832+
done();
833+
}
834+
835+
pipe([globStream('notfound', { cwd: dir }), concat()], assert);
836+
});
837+
838+
it('throws an error if you try to write to it', function (done) {
839+
var gs = globStream('notfound', { cwd: dir });
840+
gs.on('error', function () {});
841+
842+
try {
843+
gs.write({});
844+
} catch (err) {
845+
expect(err).toEqual(expect.anything());
846+
done();
847+
}
848+
});
849+
850+
it('does not throw an error if you push to it', function (done) {
851+
var stub = {
852+
cwd: dir,
853+
base: dir,
854+
path: dir,
855+
};
856+
857+
var gs = globStream('./fixtures/test.coffee', { cwd: dir });
858+
859+
gs.push(stub);
860+
861+
function assert(pathObjs) {
862+
expect(pathObjs.length).toEqual(2);
863+
expect(pathObjs[0]).toEqual(stub);
864+
}
865+
866+
pipe([gs, concat(assert)], done);
867+
});
868+
869+
it('accepts a file path', function (done) {
870+
var expected = {
871+
cwd: dir,
872+
base: dir + '/fixtures',
873+
path: dir + '/fixtures/test.coffee',
874+
};
875+
876+
function assert(pathObjs) {
877+
expect(pathObjs.length).toBe(1);
878+
expect(pathObjs[0]).toMatchObject(expected);
879+
}
880+
881+
pipe(
882+
[globStream('./fixtures/test.coffee', { cwd: dir }), concat(assert)],
883+
done
884+
);
885+
});
886+
887+
it('accepts a glob', function (done) {
888+
var expected = [
889+
{
890+
cwd: dir,
891+
base: dir + '/fixtures',
892+
path: dir + '/fixtures/has (parens)/test.dmc',
893+
},
894+
{
895+
cwd: dir,
896+
base: dir + '/fixtures',
897+
path: dir + '/fixtures/stuff/run.dmc',
898+
},
899+
{
900+
cwd: dir,
901+
base: dir + '/fixtures',
902+
path: dir + '/fixtures/stuff/test.dmc',
903+
},
904+
];
905+
906+
function assert(pathObjs) {
907+
expect(pathObjs.length).toBe(3);
908+
expect(pathObjs).toContainEqual(expected[0]);
909+
expect(pathObjs).toContainEqual(expected[1]);
910+
expect(pathObjs).toContainEqual(expected[2]);
911+
}
912+
913+
pipe(
914+
[globStream('./fixtures/**/*.dmc', { cwd: dir }), concat(assert)],
915+
done
916+
);
917+
});
918+
919+
it('pauses the globber upon backpressure', function (done) {
920+
var gs = globStream('./fixtures/**/*.dmc', { cwd: dir, highWaterMark: 1 });
921+
922+
function waiter(pathObj, _, cb) {
923+
setTimeout(function () {
924+
cb(null, pathObj);
925+
}, 500);
926+
}
927+
928+
function assert(pathObjs) {
929+
expect(pathObjs.length).toEqual(3);
930+
}
931+
932+
pipe(
933+
[gs, through2.obj({ highWaterMark: 1 }, waiter), concat(assert)],
934+
done
935+
);
936+
});
937+
938+
it('destroys the stream with an error if no match is found', function (done) {
939+
var gs = globStream('notfound', { cwd: dir });
940+
941+
var spy = sinon.spy(gs, 'destroy');
942+
943+
function assert(err) {
944+
sinon.restore();
945+
expect(spy.getCall(0).args[0]).toBe(err);
946+
expect(err.toString()).toMatch(/File not found with singular glob/);
947+
done();
948+
}
949+
950+
pipe([gs, concat()], assert);
951+
});
952+
953+
it('destroys the stream if walker errors', function (done) {
954+
var expectedError = new Error('Stubbed error');
955+
956+
var gs = globStream('./fixtures/**/*.dmc', { cwd: dir });
957+
958+
function stubError(dirpath, opts, cb) {
959+
cb(expectedError);
960+
}
961+
962+
var spy = sinon.spy(gs, 'destroy');
963+
sinon.stub(fs, 'readdir').callsFake(stubError);
964+
965+
function assert(err) {
966+
sinon.restore();
967+
expect(spy.called).toEqual(true);
968+
expect(err).toBe(expectedError);
969+
done();
970+
}
971+
972+
pipe([gs, concat()], assert);
973+
});
974+
975+
it('does not emit an error if stream is destroyed without an error', function (done) {
976+
var gs = globStream('./fixtures/**/*.dmc', { cwd: dir });
977+
978+
var spy = sinon.spy();
979+
980+
gs.on('error', spy);
981+
982+
gs.on('close', function () {
983+
sinon.restore();
984+
expect(spy.called).toEqual(false);
985+
done();
986+
});
987+
988+
gs.destroy();
989+
});
818990
});

‎test/readable.js

-169
This file was deleted.

0 commit comments

Comments
 (0)
Please sign in to comment.