Skip to content

Commit

Permalink
Allow scope hoisting to process modules in multiple chunks
Browse files Browse the repository at this point in the history
  • Loading branch information
@sokra
sokra committed Jun 14, 2017
1 parent d6a7594 commit 5d4ba56
Show file tree
Hide file tree
Showing 16 changed files with 254 additions and 131 deletions.
18 changes: 10 additions & 8 deletions lib/Compilation.js
View file
Expand Up @@ -906,6 +906,15 @@ class Compilation extends Tapable {
chunks
}];

const filterFn = dep => {
if(chunks.has(dep.chunk)) return false;
for(const chunk of chunks) {
if(chunk.containsModule(dep.module))
return false;
}
return true;
};

while(queue2.length) {
const queueItem = queue2.pop();
chunk = queueItem.chunk;
Expand All @@ -914,14 +923,7 @@ class Compilation extends Tapable {
const deps = chunkDependencies.get(chunk);
if(!deps) continue;

const depsFiltered = deps.filter(dep => {
if(chunks.has(dep.chunk)) return false;
for(const chunk of chunks) {
if(chunk.containsModule(dep.module))
return false;
}
return true;
});
const depsFiltered = deps.filter(filterFn);

for(let i = 0; i < depsFiltered.length; i++) {
const dep = depsFiltered[i];
Expand Down
18 changes: 18 additions & 0 deletions lib/Module.js
View file
Expand Up @@ -127,10 +127,28 @@ class Module extends DependenciesBlock {
return Array.from(this._chunks, fn);
}

getChunks() {
return Array.from(this._chunks);
}

getNumberOfChunks() {
return this._chunks.size;
}

hasEqualsChunks(otherModule) {
if(this._chunks.size !== otherModule._chunks.size) return false;
this._ensureChunksSortedByDebugId();
otherModule._ensureChunksSortedByDebugId();
const a = this._chunks[Symbol.iterator]();
const b = otherModule._chunks[Symbol.iterator]();
while(true) { // eslint-disable-line
const aItem = a.next();
const bItem = b.next();
if(aItem.done) return true;
if(aItem.value !== bItem.value) return false;
}
}

_ensureChunksSorted() {
if(this._chunksIsSorted) return;
this._chunks = new Set(Array.from(this._chunks).sort(byId));
Expand Down
247 changes: 124 additions & 123 deletions lib/optimize/ModuleConcatenationPlugin.js
View file
Expand Up @@ -23,9 +23,9 @@ class ModuleConcatenationPlugin {
});
const bailoutReasonMap = new Map();

function setBailoutReason(module, reason) {
function setBailoutReason(module, prefix, reason) {
bailoutReasonMap.set(module, reason);
module.optimizationBailout.push(reason);
module.optimizationBailout.push(typeof reason === "function" ? (rs) => `${prefix}: ${reason(rs)}` : `${prefix}: ${reason}`);
}

function getBailoutReason(module, requestShortener) {
Expand All @@ -35,141 +35,135 @@ class ModuleConcatenationPlugin {
}

compilation.plugin("optimize-chunk-modules", (chunks, modules) => {
chunks.forEach(chunk => {
const relevantModules = [];
const possibleInners = new Set();
for(const module of chunk.modulesIterable) {
// Only harmony modules are valid for optimization
if(!module.meta || !module.meta.harmonyModule) {
continue;
}
const relevantModules = [];
const possibleInners = new Set();
for(const module of modules) {
// Only harmony modules are valid for optimization
if(!module.meta || !module.meta.harmonyModule) {
continue;
}

// Module must not be in other chunks
// TODO add an option to allow module to be in other entry points
if(module.getNumberOfChunks() !== 1) {
setBailoutReason(module, "ModuleConcatenation: module is in multiple chunks");
continue;
}
// Because of variable renaming we can't use modules with eval
if(module.meta && module.meta.hasEval) {
setBailoutReason(module, "ModuleConcatenation", "eval is used in the module");
continue;
}

// Because of variable renaming we can't use modules with eval
if(module.meta && module.meta.hasEval) {
setBailoutReason(module, "ModuleConcatenation: eval is used in the module");
continue;
}
relevantModules.push(module);

relevantModules.push(module);
// Module must not be the entry points
if(module.getChunks().some(chunk => chunk.entryModule === module)) {
setBailoutReason(module, "ModuleConcatenation (inner)", "module is an entrypoint");
continue;
}

// Module must not be the entry points
if(chunk.entryModule === module) {
setBailoutReason(module, "ModuleConcatenation (inner): module is an entrypoint");
continue;
}
// Exports must be known (and not dynamic)
if(!Array.isArray(module.providedExports)) {
setBailoutReason(module, "ModuleConcatenation (inner)", "exports are not known");
continue;
}

// Exports must be known (and not dynamic)
if(!Array.isArray(module.providedExports)) {
setBailoutReason(module, "ModuleConcatenation (inner): exports are not known");
continue;
}
// Using dependency variables is not possible as this wraps the code in a function
if(module.variables.length > 0) {
setBailoutReason(module, "ModuleConcatenation (inner)", "dependency variables are used (i. e. ProvidePlugin)");
continue;
}

// Using dependency variables is not possible as this wraps the code in a function
if(module.variables.length > 0) {
setBailoutReason(module, "ModuleConcatenation (inner): dependency variables are used (i. e. ProvidePlugin)");
continue;
}
// Module must only be used by Harmony Imports
const nonHarmonyReasons = module.reasons.filter(reason => !(reason.dependency instanceof HarmonyImportDependency));
if(nonHarmonyReasons.length > 0) {
const importingModules = new Set(nonHarmonyReasons.map(r => r.module));
setBailoutReason(module, "ModuleConcatenation (inner)", (requestShortener) => {
const names = Array.from(importingModules).map(m => m.readableIdentifier(requestShortener));
return `module is used with non-harmony imports from ${names.join(", ")}`;
});
continue;
}

// Module must only be used by Harmony Imports
const nonHarmonyReasons = module.reasons.filter(reason => !(reason.dependency instanceof HarmonyImportDependency));
if(nonHarmonyReasons.length > 0) {
const importingModules = new Set(nonHarmonyReasons.map(r => r.module));
setBailoutReason(module, (requestShortener) => {
const names = Array.from(importingModules).map(m => m.readableIdentifier(requestShortener));
return `ModuleConcatenation (inner): module is used with non-harmony imports from ${names.join(", ")}`;
});
continue;
possibleInners.add(module);
}
// sort by depth
// modules with lower depth are more likly suited as roots
// this improves performance, because modules already selected as inner are skipped
relevantModules.sort((a, b) => {
return a.depth - b.depth;
});
const concatConfigurations = [];
const usedAsInner = new Set();
for(const currentRoot of relevantModules) {
// when used by another configuration as inner:
// the other configuration is better and we can skip this one
if(usedAsInner.has(currentRoot))
continue;

// create a configuration with the root
const currentConfiguration = new ConcatConfiguration(currentRoot);

// cache failures to add modules
const failureCache = new Map();

// try to add all imports
for(const imp of this.getImports(currentRoot)) {
const problem = this.tryToAdd(currentConfiguration, imp, possibleInners, failureCache);
if(problem) {
failureCache.set(imp, problem);
currentConfiguration.addWarning(imp, problem);
}

possibleInners.add(module);
}
// sort by depth
// modules with lower depth are more likly suited as roots
// this improves performance, because modules already selected as inner are skipped
relevantModules.sort((a, b) => {
return a.depth - b.depth;
});
const concatConfigurations = [];
const usedAsInner = new Set();
for(const currentRoot of relevantModules) {
// when used by another configuration as inner:
// the other configuration is better and we can skip this one
if(usedAsInner.has(currentRoot))
continue;

// create a configuration with the root
const currentConfiguration = new ConcatConfiguration(currentRoot);

// cache failures to add modules
const failureCache = new Map();

// try to add all imports
for(const imp of this.getImports(currentRoot)) {
const problem = this.tryToAdd(currentConfiguration, imp, possibleInners, failureCache);
if(problem) {
failureCache.set(imp, problem);
currentConfiguration.addWarning(imp, problem);
}
}
if(!currentConfiguration.isEmpty()) {
concatConfigurations.push(currentConfiguration);
for(const module of currentConfiguration.modules) {
if(module !== currentConfiguration.rootModule)
usedAsInner.add(module);
}
if(!currentConfiguration.isEmpty()) {
concatConfigurations.push(currentConfiguration);
for(const module of currentConfiguration.modules) {
if(module !== currentConfiguration.rootModule)
usedAsInner.add(module);
}
}
// HACK: Sort configurations by length and start with the longest one
// to get the biggers groups possible. Used modules are marked with usedModules
// TODO: Allow to reuse existing configuration while trying to add dependencies.
// This would improve performance. O(n^2) -> O(n)
concatConfigurations.sort((a, b) => {
return b.modules.size - a.modules.size;
});
const usedModules = new Set();
for(const concatConfiguration of concatConfigurations) {
if(usedModules.has(concatConfiguration.rootModule))
continue;
const orderedModules = new Set();
this.addInOrder(concatConfiguration.rootModule, concatConfiguration.modules, orderedModules);
const newModule = new ConcatenatedModule(concatConfiguration.rootModule, Array.from(orderedModules));
for(const warning of concatConfiguration.warnings) {
newModule.optimizationBailout.push((requestShortener) => {
const reason = getBailoutReason(warning[0], requestShortener);
const reasonPrefix = reason ? `: ${reason}` : "";
if(warning[0] === warning[1])
return `ModuleConcatenation: Cannot concat with ${warning[0].readableIdentifier(requestShortener)}${reasonPrefix}`;
else
return `ModuleConcatenation: Cannot concat with ${warning[0].readableIdentifier(requestShortener)} because of ${warning[1].readableIdentifier(requestShortener)}${reasonPrefix}`;
});
}
for(const m of orderedModules) {
usedModules.add(m);
chunk.removeModule(m);
}
}
// HACK: Sort configurations by length and start with the longest one
// to get the biggers groups possible. Used modules are marked with usedModules
// TODO: Allow to reuse existing configuration while trying to add dependencies.
// This would improve performance. O(n^2) -> O(n)
concatConfigurations.sort((a, b) => {
return b.modules.size - a.modules.size;
});
const usedModules = new Set();
for(const concatConfiguration of concatConfigurations) {
if(usedModules.has(concatConfiguration.rootModule))
continue;
const orderedModules = new Set();
this.addInOrder(concatConfiguration.rootModule, concatConfiguration.modules, orderedModules);
const newModule = new ConcatenatedModule(concatConfiguration.rootModule, Array.from(orderedModules));
for(const warning of concatConfiguration.warnings) {
newModule.optimizationBailout.push((requestShortener) => {
const reason = getBailoutReason(warning[0], requestShortener);
const reasonPrefix = reason ? `: ${reason}` : "";
if(warning[0] === warning[1])
return `ModuleConcatenation: Cannot concat with ${warning[0].readableIdentifier(requestShortener)}${reasonPrefix}`;
else
return `ModuleConcatenation: Cannot concat with ${warning[0].readableIdentifier(requestShortener)} because of ${warning[1].readableIdentifier(requestShortener)}${reasonPrefix}`;
});
}
const chunks = concatConfiguration.rootModule.getChunks();
for(const m of orderedModules) {
usedModules.add(m);
chunks.forEach(chunk => chunk.removeModule(m));
}
chunks.forEach(chunk => {
chunk.addModule(newModule);
compilation.modules.push(newModule);
if(chunk.entryModule === concatConfiguration.rootModule)
chunk.entryModule = newModule;
newModule.reasons.forEach(reason => reason.dependency.module = newModule);
newModule.dependencies.forEach(dep => {
if(dep.module) {
dep.module.reasons.forEach(reason => {
if(reason.dependency === dep)
reason.module = newModule;
});
}
});
}
compilation.modules = compilation.modules.filter(m => !usedModules.has(m));
});
});
compilation.modules.push(newModule);
newModule.reasons.forEach(reason => reason.dependency.module = newModule);
newModule.dependencies.forEach(dep => {
if(dep.module) {
dep.module.reasons.forEach(reason => {
if(reason.dependency === dep)
reason.module = newModule;
});
}
});
}
compilation.modules = compilation.modules.filter(m => !usedModules.has(m));
});
});
}
Expand Down Expand Up @@ -208,6 +202,13 @@ class ModuleConcatenationPlugin {

// Not possible to add?
if(!possibleModules.has(module)) {
failureCache.set(module, module); // cache failures for performance
return module;
}

// module must be in the same chunks
if(!config.rootModule.hasEqualsChunks(module)) {
failureCache.set(module, module); // cache failures for performance
return module;
}

Expand Down
1 change: 1 addition & 0 deletions test/statsCases/scope-hoisting-multi/common.js
View file
@@ -0,0 +1 @@
export { default } from "./common2";
1 change: 1 addition & 0 deletions test/statsCases/scope-hoisting-multi/common2.js
View file
@@ -0,0 +1 @@
export default "common";
1 change: 1 addition & 0 deletions test/statsCases/scope-hoisting-multi/common_lazy.js
View file
@@ -0,0 +1 @@
export default "common";
1 change: 1 addition & 0 deletions test/statsCases/scope-hoisting-multi/common_lazy_shared.js
View file
@@ -0,0 +1 @@
export default "common";
35 changes: 35 additions & 0 deletions test/statsCases/scope-hoisting-multi/expected.txt
View file
@@ -0,0 +1,35 @@
Hash: 731069e082cf620521ced84ccc10b5c4fc7695db
Child
Hash: 731069e082cf620521ce
Time: Xms
[0] (webpack)/test/statsCases/scope-hoisting-multi/common_lazy_shared.js 25 bytes {0} {1} {2} [built]
[1] (webpack)/test/statsCases/scope-hoisting-multi/vendor.js 25 bytes {5} [built]
[2] (webpack)/test/statsCases/scope-hoisting-multi/common.js 37 bytes {3} {4} [built]
[3] (webpack)/test/statsCases/scope-hoisting-multi/common2.js 25 bytes {3} {4} [built]
[4] (webpack)/test/statsCases/scope-hoisting-multi/common_lazy.js 25 bytes {1} {2} [built]
[5] (webpack)/test/statsCases/scope-hoisting-multi/lazy_shared.js 31 bytes {0} [built]
[6] (webpack)/test/statsCases/scope-hoisting-multi/first.js 207 bytes {3} [built]
[7] (webpack)/test/statsCases/scope-hoisting-multi/module_first.js 31 bytes {3} [built]
[8] (webpack)/test/statsCases/scope-hoisting-multi/lazy_first.js 55 bytes {2} [built]
[9] (webpack)/test/statsCases/scope-hoisting-multi/second.js 177 bytes {4} [built]
[10] (webpack)/test/statsCases/scope-hoisting-multi/lazy_second.js 55 bytes {1} [built]
Child
Hash: d84ccc10b5c4fc7695db
Time: Xms
[0] (webpack)/test/statsCases/scope-hoisting-multi/common_lazy_shared.js 25 bytes {0} {1} {2} [built]
[1] (webpack)/test/statsCases/scope-hoisting-multi/vendor.js 25 bytes {5} [built]
ModuleConcatenation (inner): module is an entrypoint
[2] (webpack)/test/statsCases/scope-hoisting-multi/common.js + 1 modules 62 bytes {4} {3} [built]
[3] (webpack)/test/statsCases/scope-hoisting-multi/common_lazy.js 25 bytes {1} {2} [built]
[4] (webpack)/test/statsCases/scope-hoisting-multi/first.js + 1 modules 238 bytes {4} [built]
ModuleConcatenation (inner): module is an entrypoint
ModuleConcatenation: Cannot concat with (webpack)/test/statsCases/scope-hoisting-multi/vendor.js: module is an entrypoint
ModuleConcatenation: Cannot concat with (webpack)/test/statsCases/scope-hoisting-multi/common.js
[5] (webpack)/test/statsCases/scope-hoisting-multi/lazy_shared.js 31 bytes {0} [built]
ModuleConcatenation (inner): module is used with non-harmony imports from (webpack)/test/statsCases/scope-hoisting-multi/first.js, (webpack)/test/statsCases/scope-hoisting-multi/second.js
[6] (webpack)/test/statsCases/scope-hoisting-multi/second.js 177 bytes {3} [built]
ModuleConcatenation (inner): module is an entrypoint
[7] (webpack)/test/statsCases/scope-hoisting-multi/lazy_second.js 55 bytes {1} [built]
ModuleConcatenation (inner): module is used with non-harmony imports from (webpack)/test/statsCases/scope-hoisting-multi/second.js
[8] (webpack)/test/statsCases/scope-hoisting-multi/lazy_first.js 55 bytes {2} [built]
ModuleConcatenation (inner): module is used with non-harmony imports from (webpack)/test/statsCases/scope-hoisting-multi/first.js
6 changes: 6 additions & 0 deletions test/statsCases/scope-hoisting-multi/first.js
View file
@@ -0,0 +1,6 @@
import v from "./vendor";
import c from "./common";
import x from "./module_first";

import(/* webpackChunkName: "lazy_first" */"./lazy_first");
import(/* webpackChunkName: "lazy_shared" */"./lazy_shared");
2 changes: 2 additions & 0 deletions test/statsCases/scope-hoisting-multi/lazy_first.js
View file
@@ -0,0 +1,2 @@
import "./common_lazy";
import "./common_lazy_shared";

0 comments on commit 5d4ba56

Please sign in to comment.