feat: concurrency option (#466)

Author: cap-Bernardito

README.md

@@ -65,7 +65,7 @@ module.exports = {
         { from: 'other', to: 'public' },
       ],
       options: {
-        ignore: ['*.bin'],
+        concurrency: 100,
       },
     }),
   ],
@@ -562,9 +562,15 @@ module.exports = {
 };
 ```
 
-#### `ignore`
+### Options
 
-Array of globs to ignore (applied to `from`).
+|             Name              |    Type    | Default | Description                                      |
+| :---------------------------: | :--------: | :-----: | :----------------------------------------------- |
+| [`concurrency`](#concurrency) | `{Number}` |  `100`  | Limits the number of simultaneous requests to fs |
+
+#### `concurrency`
+
+limits the number of simultaneous requests to fs
 
 **webpack.config.js**
 
@@ -573,7 +579,7 @@ module.exports = {
   plugins: [
     new CopyPlugin({
       patterns: [...patterns],
-      options: { ignore: ['*.js', '*.css'] },
+      options: { concurrency: 50 },
     }),
   ],
 };

src/index.js

@@ -1,4 +1,5 @@
 import validateOptions from 'schema-utils';
+import pLimit from 'p-limit';
 
 import schema from './options.json';
 import preProcessPattern from './preProcessPattern';
@@ -18,6 +19,7 @@ class CopyPlugin {
 
   apply(compiler) {
     const plugin = { name: 'CopyPlugin' };
+    const limit = pLimit(this.options.concurrency || 100);
 
     compiler.hooks.compilation.tap(plugin, (compilation) => {
       const logger = compilation.getLogger('copy-webpack-plugin');
@@ -33,38 +35,39 @@ class CopyPlugin {
             compilation,
             inputFileSystem: compiler.inputFileSystem,
             output: compiler.options.output.path,
-            concurrency: this.options.concurrency,
           };
 
           try {
             await Promise.all(
-              this.patterns.map(async (pattern) => {
-                const patternAfterPreProcess = await preProcessPattern(
-                  globalRef,
-                  pattern
-                );
+              this.patterns.map((pattern) =>
+                limit(async () => {
+                  const patternAfterPreProcess = await preProcessPattern(
+                    globalRef,
+                    pattern
+                  );
 
-                const files = await processPattern(
-                  globalRef,
-                  patternAfterPreProcess
-                );
+                  const files = await processPattern(
+                    globalRef,
+                    patternAfterPreProcess
+                  );
 
-                if (!files) {
-                  return Promise.resolve();
-                }
+                  if (!files) {
+                    return Promise.resolve();
+                  }
 
-                return Promise.all(
-                  files
-                    .filter(Boolean)
-                    .map((file) =>
-                      postProcessPattern(
-                        globalRef,
-                        patternAfterPreProcess,
-                        file
-                      )
+                  return Promise.all(
+                    files.filter(Boolean).map((file) =>
+                      limit(() => {
+                        return postProcessPattern(
+                          globalRef,
+                          patternAfterPreProcess,
+                          file
+                        );
+                      })
                     )
-                );
-              })
+                  );
+                })
+              )
             );
 
             logger.debug('end to adding additionalAssets');

src/options.json

@@ -76,7 +76,11 @@
     "options": {
       "type": "object",
       "additionalProperties": false,
-      "properties": {}
+      "properties": {
+        "concurrency": {
+          "type": "number"
+        }
+      }
     }
   }
 }

src/processPattern.js

@@ -1,18 +1,15 @@
 import path from 'path';
 
 import globby from 'globby';
-import pLimit from 'p-limit';
 
 import createPatternGlob from './utils/createPatternGlob';
 
 /* eslint-disable no-param-reassign */
 
 export default async function processPattern(globalRef, pattern) {
-  const { logger, output, concurrency, compilation } = globalRef;
+  const { logger, output, compilation } = globalRef;
   createPatternGlob(pattern, globalRef);
 
-  const limit = pLimit(concurrency || 100);
-
   logger.log(
     `begin globbing '${pattern.glob}' with a context of '${pattern.context}'`
   );
@@ -38,42 +35,36 @@ export default async function processPattern(globalRef, pattern) {
     return Promise.resolve();
   }
 
-  return Promise.all(
-    paths.map((from) =>
-      limit(() => {
-        const file = {
-          force: pattern.force,
-          absoluteFrom: path.resolve(pattern.context, from),
-        };
-
-        file.relativeFrom = path.relative(pattern.context, file.absoluteFrom);
-
-        if (pattern.flatten) {
-          file.relativeFrom = path.basename(file.relativeFrom);
-        }
-
-        logger.debug(`found ${from}`);
-
-        // Change the to path to be relative for webpack
-        if (pattern.toType === 'dir') {
-          file.webpackTo = path.join(pattern.to, file.relativeFrom);
-        } else if (pattern.toType === 'file') {
-          file.webpackTo = pattern.to || file.relativeFrom;
-        } else if (pattern.toType === 'template') {
-          file.webpackTo = pattern.to;
-          file.webpackToRegExp = pattern.test;
-        }
-
-        if (path.isAbsolute(file.webpackTo)) {
-          file.webpackTo = path.relative(output, file.webpackTo);
-        }
-
-        logger.log(
-          `determined that '${from}' should write to '${file.webpackTo}'`
-        );
-
-        return file;
-      })
-    )
-  );
+  return paths.map((from) => {
+    const file = {
+      force: pattern.force,
+      absoluteFrom: path.resolve(pattern.context, from),
+    };
+
+    file.relativeFrom = path.relative(pattern.context, file.absoluteFrom);
+
+    if (pattern.flatten) {
+      file.relativeFrom = path.basename(file.relativeFrom);
+    }
+
+    logger.debug(`found ${from}`);
+
+    // Change the to path to be relative for webpack
+    if (pattern.toType === 'dir') {
+      file.webpackTo = path.join(pattern.to, file.relativeFrom);
+    } else if (pattern.toType === 'file') {
+      file.webpackTo = pattern.to || file.relativeFrom;
+    } else if (pattern.toType === 'template') {
+      file.webpackTo = pattern.to;
+      file.webpackToRegExp = pattern.test;
+    }
+
+    if (path.isAbsolute(file.webpackTo)) {
+      file.webpackTo = path.relative(output, file.webpackTo);
+    }
+
+    logger.log(`determined that '${from}' should write to '${file.webpackTo}'`);
+
+    return file;
+  });
 }

test/__snapshots__/validate-options.test.js.snap

@@ -1,9 +1,14 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`validate options should throw an error on the "options" option with "{"concurrency":true}" value 1`] = `
+"Invalid options object. Copy Plugin has been initialized using an options object that does not match the API schema.
+ - options.options.concurrency should be a number."
+`;
+
 exports[`validate options should throw an error on the "options" option with "{"unknown":true}" value 1`] = `
 "Invalid options object. Copy Plugin has been initialized using an options object that does not match the API schema.
  - options.options has an unknown property 'unknown'. These properties are valid:
-   object {}"
+   object { concurrency? }"
 `;
 
 exports[`validate options should throw an error on the "patterns" option with "" value 1`] = `

test/validate-options.test.js

@@ -203,8 +203,8 @@ describe('validate options', () => {
       ],
     },
     options: {
-      success: [{}],
-      failure: [{ unknown: true }],
+      success: [{ concurrency: 50 }],
+      failure: [{ unknown: true }, { concurrency: true }],
     },
     unknown: {
       success: [],