Skip to content

Commit

Permalink
Merge pull request #175 from webpack/bugfix/plan-merging
Browse files Browse the repository at this point in the history 
fix bugs in recursive watching
  • Loading branch information
@sokra
sokra committed Aug 27, 2020
2 parents 9ec3868 + dc01813 commit e8010ae
Show file tree
Hide file tree
Showing 3 changed files with 175 additions and 13 deletions.
22 changes: 14 additions & 8 deletions lib/reducePlan.js
View file
Expand Up @@ -52,7 +52,9 @@ module.exports = (plan, limit) => {
value: undefined
};
treeMap.set(parentPath, parent);
node.parent = parent;
} else {
node.parent = parent;
if (parent.children === undefined) {
parent.children = [node];
} else {
Expand All @@ -68,12 +70,13 @@ module.exports = (plan, limit) => {
// Reduce until limit reached
while (currentCount > limit) {
// Select node that helps reaching the limit most effectively without overmerging
const overLimit = limit - currentCount;
const overLimit = currentCount - limit;
let bestNode = undefined;
let bestCost = Infinity;
for (const node of treeMap.values()) {
if (node.entries <= 1 || !node.children) continue;
if (node.children.length <= 1) continue;
if (node.entries <= 1 || !node.children || !node.parent) continue;
if (node.children.length === 0) continue;
if (node.children.length === 1 && !node.value) continue;
// Try to select the node with has just a bit more entries than we need to reduce
// When just a bit more is over 30% over the limit,
// also consider just a bit less entries then we need to reduce
Expand Down Expand Up @@ -108,11 +111,12 @@ module.exports = (plan, limit) => {
}
// Write down new plan
const newPlan = new Map();
for (const node of treeMap.values()) {
if (!node.active) continue;
for (const rootNode of treeMap.values()) {
if (!rootNode.active) continue;
const map = new Map();
const queue = new Set([node]);
const queue = new Set([rootNode]);
for (const node of queue) {
if (node.active && node !== rootNode) continue;
if (node.value) {
if (Array.isArray(node.value)) {
for (const item of node.value) {
Expand All @@ -123,10 +127,12 @@ module.exports = (plan, limit) => {
}
}
if (node.children) {
for (const child of node.children) queue.add(child);
for (const child of node.children) {
queue.add(child);
}
}
}
newPlan.set(node.filePath, map);
newPlan.set(rootNode.filePath, map);
}
return newPlan;
};
146 changes: 145 additions & 1 deletion test/Assumption.js
View file
Expand Up @@ -9,6 +9,10 @@ var TestHelper = require("./helpers/TestHelper");
var fixtures = path.join(__dirname, "fixtures");
var testHelper = new TestHelper(fixtures);

const IS_OSX = require("os").platform() === "darwin";
const IS_WIN = require("os").platform() === "win32";
const SUPPORTS_RECURSIVE_WATCHING = IS_OSX || IS_WIN;

describe("Assumption", function() {
this.timeout(10000);
var watcherToClose = null;
Expand Down Expand Up @@ -145,6 +149,76 @@ describe("Assumption", function() {
}
});

if (SUPPORTS_RECURSIVE_WATCHING) {
it("should have a file system with correct mtime behavior (fs.watch recursive)", function(done) {
this.timeout(20000);
testHelper.file("a");
var i = 60;
var count = 60;
var before;
var after;
var minDiffBefore = +Infinity;
var maxDiffBefore = -Infinity;
var sumDiffBefore = 0;
var minDiffAfter = +Infinity;
var maxDiffAfter = -Infinity;
var sumDiffAfter = 0;
var watcher = (watcherToClose = fs.watch(fixtures, { recursive: true }));
testHelper.tick(100, function() {
watcher.on("change", function(type, filename) {
const s = fs.statSync(path.join(fixtures, filename));
if (before && after) {
var diffBefore = +s.mtime - before;
if (diffBefore < minDiffBefore) minDiffBefore = diffBefore;
if (diffBefore > maxDiffBefore) maxDiffBefore = diffBefore;
sumDiffBefore += diffBefore;
var diffAfter = +s.mtime - after;
if (diffAfter < minDiffAfter) minDiffAfter = diffAfter;
if (diffAfter > maxDiffAfter) maxDiffAfter = diffAfter;
sumDiffAfter += diffAfter;
before = after = undefined;
if (i-- === 0) {
afterMeasure();
} else {
testHelper.tick(100, checkMtime);
}
}
});
testHelper.tick(100, checkMtime);
});

function checkMtime() {
before = Date.now();
testHelper.file("a");
after = Date.now();
}

function afterMeasure() {
console.log(
"mtime fs.watch({ recursive: true }) accuracy (before): [" +
minDiffBefore +
" ; " +
maxDiffBefore +
"] avg " +
Math.round(sumDiffBefore / count)
);
console.log(
"mtime fs.watch({ recursive: true }) accuracy (after): [" +
minDiffAfter +
" ; " +
maxDiffAfter +
"] avg " +
Math.round(sumDiffAfter / count)
);
minDiffBefore.should.be.aboveOrEqual(-2000);
maxDiffBefore.should.be.below(2000);
minDiffAfter.should.be.aboveOrEqual(-2000);
maxDiffAfter.should.be.below(2000);
done();
}
});
}

it("should not fire events in subdirectories", function(done) {
testHelper.dir("watch-test-directory");
testHelper.tick(500, () => {
Expand All @@ -166,7 +240,77 @@ describe("Assumption", function() {
});
});

if (require("os").platform() !== "darwin") {
if (SUPPORTS_RECURSIVE_WATCHING) {
it("should fire events in subdirectories (recursive)", function(done) {
testHelper.dir("watch-test-directory");
testHelper.file("watch-test-directory/watch-test-file");
testHelper.file("watch-test-directory/existing-file");
testHelper.tick(500, () => {
var watcher = (watcherToClose = fs.watch(fixtures, {
recursive: true
}));
const events = [];
watcher.once("change", () => {
testHelper.tick(1000, function() {
events.should.matchAny(/watch-test-directory[/\\]watch-test-file/);
done();
});
});
watcher.on("change", function(type, filename) {
events.push(filename);
});
watcher.on("error", function(err) {
done(err);
done = function() {};
});
testHelper.tick(500, function() {
testHelper.file("watch-test-directory/watch-test-file");
});
});
});

it("should allow to create/close/create recursive watchers", function(done) {
testHelper.dir("watch-test-directory");
testHelper.file("watch-test-directory/watch-test-file");
testHelper.file("watch-test-directory/existing-file");
testHelper.tick(500, () => {
watcherToClose = fs.watch(fixtures, {
recursive: true
});
watcherToClose.close();
watcherToClose = fs.watch(fixtures, {
recursive: true
});
watcherToClose.close();
watcherToClose = fs.watch(fixtures, {
recursive: true
});
watcherToClose.close();
var watcher = (watcherToClose = fs.watch(fixtures, {
recursive: true
}));
const events = [];
watcher.once("change", () => {
testHelper.tick(1000, function() {
events.should.matchAny(/watch-test-directory[/\\]watch-test-file/);
done();
});
});
watcher.on("change", function(type, filename) {
events.push(filename);
});
watcher.on("error", function(err) {
done(err);
done = function() {};
});
testHelper.tick(500, function() {
testHelper.file("watch-test-directory/watch-test-file");
});
});
});
}

if (!IS_OSX) {
it("should detect removed directory", function(done) {
testHelper.dir("watch-test-dir");
testHelper.tick(() => {
Expand Down
20 changes: 16 additions & 4 deletions test/ManyWatchers.js
View file
Expand Up @@ -11,13 +11,16 @@ const fixtures = path.join(__dirname, "fixtures");
const testHelper = new TestHelper(fixtures);

describe("ManyWatchers", function() {
this.timeout(240000);
this.timeout(600000);
beforeEach(testHelper.before);
afterEach(testHelper.after);

it("should watch more than 4096 directories", done => {
console.time("creating files");
// windows is very slow in creating so many files
// this can take about 1 minute
const files = [];
for (let i = 1; i <= 5000; i++) {
for (let i = 1; i < 5000; i++) {
let highBit = 1;
let j = i;
while (j > 1) {
Expand All @@ -38,24 +41,33 @@ describe("ManyWatchers", function() {
files.push(path.join(fixtures, dir, "file2"));
}
}
testHelper.tick(1000, () => {
testHelper.file("file");
files.push(path.join(fixtures, "file"));
console.timeEnd("creating files");
testHelper.tick(10000, () => {
const w = new Watchpack({
aggregateTimeout: 1000
});
w.on("aggregated", function(changes) {
console.timeEnd("detecting change event");
Array.from(changes).should.be.eql([
path.join(fixtures, "4096/900/file")
]);
w.close();
done();
});
for (let i = 100; i < files.length; i += 987) {
console.time("creating/closing watchers");
// MacOS is very slow in creating and destroying watchers
// This can take about 2 minutes
for (let i = 100; i < files.length; i += 2432) {
for (let j = 0; j < files.length - i; j += 987) {
w.watch({ files: files.slice(j, j + i) });
}
}
w.watch({ files });
console.timeEnd("creating/closing watchers");
testHelper.tick(10000, () => {
console.time("detecting change event");
testHelper.file("4096/900/file");
});
});
Expand Down

0 comments on commit e8010ae

Please sign in to comment.