Update output tracing to do separate passes (#30637)

* Update output tracing to do separate passes

* fix windows backslashes

Author: ijjk

packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts

@@ -24,6 +24,7 @@ const TRACE_IGNORES = [
   '**/*/next/dist/server/next.js',
   '**/*/next/dist/bin/next',
 ]
+const root = nodePath.parse(process.cwd()).root
 
 function getModuleFromDependency(
   compilation: any,
@@ -32,6 +33,54 @@ function getModuleFromDependency(
   return compilation.moduleGraph.getModule(dep)
 }
 
+function getFilesMapFromReasons(
+  fileList: Set<string>,
+  reasons: NodeFileTraceReasons
+) {
+  // this uses the reasons tree to collect files specific to a
+  // certain parent allowing us to not have to trace each parent
+  // separately
+  const parentFilesMap = new Map<string, Set<string>>()
+
+  function propagateToParents(
+    parents: Set<string>,
+    file: string,
+    seen = new Set<string>()
+  ) {
+    for (const parent of parents || []) {
+      if (!seen.has(parent)) {
+        seen.add(parent)
+        let parentFiles = parentFilesMap.get(parent)
+
+        if (!parentFiles) {
+          parentFiles = new Set()
+          parentFilesMap.set(parent, parentFiles)
+        }
+        parentFiles.add(file)
+        const parentReason = reasons.get(parent)
+
+        if (parentReason?.parents) {
+          propagateToParents(parentReason.parents, file, seen)
+        }
+      }
+    }
+  }
+
+  for (const file of fileList!) {
+    const reason = reasons!.get(file)
+
+    if (
+      !reason ||
+      !reason.parents ||
+      (reason.type === 'initial' && reason.parents.size === 0)
+    ) {
+      continue
+    }
+    propagateToParents(reason.parents, file)
+  }
+  return parentFilesMap
+}
+
 export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
   private appDir: string
   private entryTraces: Map<string, Set<string>>
@@ -59,36 +108,107 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
 
   // Here we output all traced assets and webpack chunks to a
   // ${page}.js.nft.json file
-  createTraceAssets(compilation: any, assets: any, span: Span) {
+  async createTraceAssets(
+    compilation: any,
+    assets: any,
+    span: Span,
+    readlink: any,
+    stat: any,
+    doResolve: any
+  ) {
     const outputPath = compilation.outputOptions.path
 
-    const nodeFileTraceSpan = span.traceChild('create-trace-assets')
-    nodeFileTraceSpan.traceFn(() => {
+    await span.traceChild('create-trace-assets').traceAsyncFn(async () => {
+      const entryFilesMap = new Map<any, Set<string>>()
+      const chunksToTrace = new Set<string>()
+
       for (const entrypoint of compilation.entrypoints.values()) {
         const entryFiles = new Set<string>()
 
         for (const chunk of entrypoint
           .getEntrypointChunk()
           .getAllReferencedChunks()) {
           for (const file of chunk.files) {
-            entryFiles.add(nodePath.join(outputPath, file))
+            const filePath = nodePath.join(outputPath, file)
+            chunksToTrace.add(filePath)
+            entryFiles.add(filePath)
           }
           for (const file of chunk.auxiliaryFiles) {
-            entryFiles.add(nodePath.join(outputPath, file))
+            const filePath = nodePath.join(outputPath, file)
+            chunksToTrace.add(filePath)
+            entryFiles.add(filePath)
           }
         }
-        // don't include the entry itself in the trace
-        entryFiles.delete(nodePath.join(outputPath, `../${entrypoint.name}.js`))
+        entryFilesMap.set(entrypoint, entryFiles)
+      }
+
+      const result = await nodeFileTrace([...chunksToTrace], {
+        base: root,
+        processCwd: this.appDir,
+        readFile: async (path) => {
+          if (chunksToTrace.has(path)) {
+            const source =
+              assets[
+                nodePath.relative(outputPath, path).replace(/\\/g, '/')
+              ]?.source?.()
+            if (source) return source
+          }
+          try {
+            return await new Promise((resolve, reject) => {
+              ;(
+                compilation.inputFileSystem
+                  .readFile as typeof import('fs').readFile
+              )(path, (err, data) => {
+                if (err) return reject(err)
+                resolve(data)
+              })
+            })
+          } catch (e) {
+            if (isError(e) && (e.code === 'ENOENT' || e.code === 'EISDIR')) {
+              return null
+            }
+            throw e
+          }
+        },
+        readlink,
+        stat,
+        resolve: doResolve
+          ? (id, parent, job, isCjs) => {
+              return doResolve(id, parent, job, !isCjs)
+            }
+          : undefined,
+        ignore: [...TRACE_IGNORES, ...this.excludeFiles],
+        mixedModules: true,
+      })
+      const reasons = result.reasons
+      const fileList = result.fileList
+      result.esmFileList.forEach((file) => fileList.add(file))
+
+      const parentFilesMap = getFilesMapFromReasons(fileList, reasons)
+
+      for (const [entrypoint, entryFiles] of entryFilesMap) {
         const traceOutputName = `../${entrypoint.name}.js.nft.json`
         const traceOutputPath = nodePath.dirname(
           nodePath.join(outputPath, traceOutputName)
         )
+        const allEntryFiles = new Set<string>()
+
+        entryFiles.forEach((file) => {
+          parentFilesMap
+            .get(nodePath.relative(root, file))
+            ?.forEach((child) => {
+              allEntryFiles.add(nodePath.join(root, child))
+            })
+        })
+        // don't include the entry itself in the trace
+        entryFiles.delete(nodePath.join(outputPath, `../${entrypoint.name}.js`))
 
         assets[traceOutputName] = new sources.RawSource(
           JSON.stringify({
             version: TRACE_OUTPUT_VERSION,
             files: [
               ...entryFiles,
+              ...allEntryFiles,
               ...(this.entryTraces.get(entrypoint.name) || []),
             ].map((file) => {
               return nodePath
@@ -104,12 +224,14 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
   tapfinishModules(
     compilation: webpack5.Compilation,
     traceEntrypointsPluginSpan: Span,
-    doResolve?: (
+    doResolve: (
       request: string,
       parent: string,
       job: import('@vercel/nft/out/node-file-trace').Job,
       isEsmRequested: boolean
-    ) => Promise<string>
+    ) => Promise<string>,
+    readlink: any,
+    stat: any
   ) {
     compilation.hooks.finishModules.tapAsync(
       PLUGIN_NAME,
@@ -168,71 +290,9 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
               if (source) {
                 return source.buffer()
               }
-
-              try {
-                return await new Promise((resolve, reject) => {
-                  ;(
-                    compilation.inputFileSystem
-                      .readFile as typeof import('fs').readFile
-                  )(path, (err, data) => {
-                    if (err) return reject(err)
-                    resolve(data)
-                  })
-                })
-              } catch (e) {
-                if (
-                  isError(e) &&
-                  (e.code === 'ENOENT' || e.code === 'EISDIR')
-                ) {
-                  return null
-                }
-                throw e
-              }
-            }
-            const readlink = async (path: string): Promise<string | null> => {
-              try {
-                return await new Promise((resolve, reject) => {
-                  ;(
-                    compilation.inputFileSystem
-                      .readlink as typeof import('fs').readlink
-                  )(path, (err, link) => {
-                    if (err) return reject(err)
-                    resolve(link)
-                  })
-                })
-              } catch (e) {
-                if (
-                  isError(e) &&
-                  (e.code === 'EINVAL' ||
-                    e.code === 'ENOENT' ||
-                    e.code === 'UNKNOWN')
-                ) {
-                  return null
-                }
-                throw e
-              }
-            }
-            const stat = async (
-              path: string
-            ): Promise<import('fs').Stats | null> => {
-              try {
-                return await new Promise((resolve, reject) => {
-                  ;(
-                    compilation.inputFileSystem.stat as typeof import('fs').stat
-                  )(path, (err, stats) => {
-                    if (err) return reject(err)
-                    resolve(stats)
-                  })
-                })
-              } catch (e) {
-                if (
-                  isError(e) &&
-                  (e.code === 'ENOENT' || e.code === 'ENOTDIR')
-                ) {
-                  return null
-                }
-                throw e
-              }
+              // we don't want to analyze non-transpiled
+              // files here, that is done against webpack output
+              return ''
             }
 
             const entryPaths = Array.from(entryModMap.keys())
@@ -262,8 +322,6 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
             })
             let fileList: Set<string>
             let reasons: NodeFileTraceReasons
-            const root = nodePath.parse(process.cwd()).root
-
             await finishModulesSpan
               .traceChild('node-file-trace', {
                 traceEntryCount: entriesToTrace.length + '',
@@ -276,11 +334,15 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
                   readlink,
                   stat,
                   resolve: doResolve
-                    ? (id, parent, job, isCjs) =>
-                        // @ts-ignore
-                        doResolve(id, parent, job, !isCjs)
+                    ? async (id, parent, job, isCjs) => {
+                        return doResolve(id, parent, job, !isCjs)
+                      }
                     : undefined,
-                  ignore: [...TRACE_IGNORES, ...this.excludeFiles],
+                  ignore: [
+                    ...TRACE_IGNORES,
+                    ...this.excludeFiles,
+                    '**/node_modules/**',
+                  ],
                   mixedModules: true,
                 })
                 // @ts-ignore
@@ -289,50 +351,10 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
                 reasons = result.reasons
               })
 
-            // this uses the reasons tree to collect files specific to a certain
-            // parent allowing us to not have to trace each parent separately
-            const parentFilesMap = new Map<string, Set<string>>()
-
-            function propagateToParents(
-              parents: Set<string>,
-              file: string,
-              seen = new Set<string>()
-            ) {
-              for (const parent of parents || []) {
-                if (!seen.has(parent)) {
-                  seen.add(parent)
-                  let parentFiles = parentFilesMap.get(parent)
-
-                  if (!parentFiles) {
-                    parentFiles = new Set()
-                    parentFilesMap.set(parent, parentFiles)
-                  }
-                  parentFiles.add(file)
-                  const parentReason = reasons.get(parent)
-
-                  if (parentReason?.parents) {
-                    propagateToParents(parentReason.parents, file, seen)
-                  }
-                }
-              }
-            }
-
             await finishModulesSpan
               .traceChild('collect-traced-files')
               .traceAsyncFn(() => {
-                for (const file of fileList!) {
-                  const reason = reasons!.get(file)
-
-                  if (
-                    !reason ||
-                    !reason.parents ||
-                    (reason.type === 'initial' && reason.parents.size === 0)
-                  ) {
-                    continue
-                  }
-                  propagateToParents(reason.parents, file)
-                }
-
+                const parentFilesMap = getFilesMapFromReasons(fileList, reasons)
                 entryPaths.forEach((entry) => {
                   const entryName = entryNameMap.get(entry)!
                   const normalizedEntry = nodePath.relative(root, entry)
@@ -371,24 +393,69 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
 
   apply(compiler: webpack5.Compiler) {
     compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
+      const readlink = async (path: string): Promise<string | null> => {
+        try {
+          return await new Promise((resolve, reject) => {
+            ;(
+              compilation.inputFileSystem
+                .readlink as typeof import('fs').readlink
+            )(path, (err, link) => {
+              if (err) return reject(err)
+              resolve(link)
+            })
+          })
+        } catch (e) {
+          if (
+            isError(e) &&
+            (e.code === 'EINVAL' || e.code === 'ENOENT' || e.code === 'UNKNOWN')
+          ) {
+            return null
+          }
+          throw e
+        }
+      }
+      const stat = async (path: string): Promise<import('fs').Stats | null> => {
+        try {
+          return await new Promise((resolve, reject) => {
+            ;(compilation.inputFileSystem.stat as typeof import('fs').stat)(
+              path,
+              (err, stats) => {
+                if (err) return reject(err)
+                resolve(stats)
+              }
+            )
+          })
+        } catch (e) {
+          if (isError(e) && (e.code === 'ENOENT' || e.code === 'ENOTDIR')) {
+            return null
+          }
+          throw e
+        }
+      }
+
       const compilationSpan = spans.get(compilation) || spans.get(compiler)!
       const traceEntrypointsPluginSpan = compilationSpan.traceChild(
         'next-trace-entrypoint-plugin'
       )
       traceEntrypointsPluginSpan.traceFn(() => {
         // @ts-ignore TODO: Remove ignore when webpack 5 is stable
-        compilation.hooks.processAssets.tap(
+        compilation.hooks.processAssets.tapAsync(
           {
             name: PLUGIN_NAME,
             // @ts-ignore TODO: Remove ignore when webpack 5 is stable
             stage: webpack.Compilation.PROCESS_ASSETS_STAGE_SUMMARIZE,
           },
-          (assets: any) => {
+          (assets: any, callback: any) => {
             this.createTraceAssets(
               compilation,
               assets,
-              traceEntrypointsPluginSpan
+              traceEntrypointsPluginSpan,
+              readlink,
+              stat,
+              doResolve
             )
+              .then(() => callback())
+              .catch((err) => callback(err))
           }
         )
         let resolver = compilation.resolverFactory.get('normal')
@@ -540,7 +607,9 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance {
         this.tapfinishModules(
           compilation,
           traceEntrypointsPluginSpan,
-          doResolve
+          doResolve,
+          readlink,
+          stat
         )
       })
     })