performance improvement of static generation (#25035)

### move all access to built pages into worker pool

to allow parallelizing and avoid loading the bundles in the main thread

This improves performance of the static check step a bit and helps reducing memory load in main thread

### enable splitChunks for server build in webpack 5

This improves performance for static generation by loading less code due to reduced duplication

Author: sokra

packages/next/build/entries.ts

@@ -58,7 +58,7 @@ export type WebpackEntrypoints = {
     | string[]
     | {
         import: string | string[]
-        dependOn: string | string[]
+        dependOn?: string | string[]
       }
 }
 

packages/next/build/index.ts

@@ -81,9 +81,6 @@ import {
   detectConflictingPaths,
   computeFromManifest,
   getJsPageSizeInKb,
-  getNamedExports,
-  hasCustomGetInitialProps,
-  isPageStatic,
   PageInfo,
   printCustomRoutes,
   printTreeView,
@@ -266,7 +263,7 @@ export default async function build(
       )
     const pageKeys = Object.keys(mappedPages)
     const conflictingPublicFiles: string[] = []
-    const hasCustomErrorPage = mappedPages['/_error'].startsWith(
+    const hasCustomErrorPage: boolean = mappedPages['/_error'].startsWith(
       'private-next-pages'
     )
     const hasPages404 = Boolean(
@@ -656,219 +653,228 @@ export default async function build(
       await promises.readFile(buildManifestPath, 'utf8')
     ) as BuildManifest
 
-    let customAppGetInitialProps: boolean | undefined
-    let namedExports: Array<string> | undefined
-    let isNextImageImported: boolean | undefined
     const analysisBegin = process.hrtime()
-    let hasSsrAmpPages = false
 
     const staticCheckSpan = nextBuildSpan.traceChild('static-check')
-    const { hasNonStaticErrorPage } = await staticCheckSpan.traceAsyncFn(
-      async () => {
-        process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
+    const {
+      customAppGetInitialProps,
+      namedExports,
+      isNextImageImported,
+      hasSsrAmpPages,
+      hasNonStaticErrorPage,
+    } = await staticCheckSpan.traceAsyncFn(async () => {
+      process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
+
+      const staticCheckWorkers = new Worker(staticCheckWorker, {
+        numWorkers: config.experimental.cpus,
+        enableWorkerThreads: config.experimental.workerThreads,
+      }) as Worker & typeof import('./utils')
+
+      staticCheckWorkers.getStdout().pipe(process.stdout)
+      staticCheckWorkers.getStderr().pipe(process.stderr)
+
+      const runtimeEnvConfig = {
+        publicRuntimeConfig: config.publicRuntimeConfig,
+        serverRuntimeConfig: config.serverRuntimeConfig,
+      }
 
-        const staticCheckWorkers = new Worker(staticCheckWorker, {
-          numWorkers: config.experimental.cpus,
-          enableWorkerThreads: config.experimental.workerThreads,
-        }) as Worker & { isPageStatic: typeof isPageStatic }
+      const nonStaticErrorPageSpan = staticCheckSpan.traceChild(
+        'check-static-error-page'
+      )
+      const nonStaticErrorPagePromise = nonStaticErrorPageSpan.traceAsyncFn(
+        async () =>
+          hasCustomErrorPage &&
+          (await staticCheckWorkers.hasCustomGetInitialProps(
+            '/_error',
+            distDir,
+            isLikeServerless,
+            runtimeEnvConfig,
+            false
+          ))
+      )
+      // we don't output _app in serverless mode so use _app export
+      // from _error instead
+      const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
 
-        staticCheckWorkers.getStdout().pipe(process.stdout)
-        staticCheckWorkers.getStderr().pipe(process.stderr)
+      const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps(
+        appPageToCheck,
+        distDir,
+        isLikeServerless,
+        runtimeEnvConfig,
+        true
+      )
 
-        const runtimeEnvConfig = {
-          publicRuntimeConfig: config.publicRuntimeConfig,
-          serverRuntimeConfig: config.serverRuntimeConfig,
-        }
+      const namedExportsPromise = staticCheckWorkers.getNamedExports(
+        appPageToCheck,
+        distDir,
+        isLikeServerless,
+        runtimeEnvConfig
+      )
 
-        const nonStaticErrorPageSpan = staticCheckSpan.traceChild(
-          'check-static-error-page'
-        )
-        const nonStaticErrorPage = await nonStaticErrorPageSpan.traceAsyncFn(
-          async () =>
-            hasCustomErrorPage &&
-            (await hasCustomGetInitialProps(
-              '/_error',
+      // eslint-disable-next-line no-shadow
+      let isNextImageImported: boolean | undefined
+      // eslint-disable-next-line no-shadow
+      let hasSsrAmpPages = false
+
+      const computedManifestData = await computeFromManifest(
+        buildManifest,
+        distDir,
+        config.experimental.gzipSize
+      )
+      await Promise.all(
+        pageKeys.map(async (page) => {
+          const checkPageSpan = staticCheckSpan.traceChild('check-page', {
+            page,
+          })
+          return checkPageSpan.traceAsyncFn(async () => {
+            const actualPage = normalizePagePath(page)
+            const [selfSize, allSize] = await getJsPageSizeInKb(
+              actualPage,
               distDir,
-              isLikeServerless,
-              runtimeEnvConfig,
-              false
-            ))
-        )
-        // we don't output _app in serverless mode so use _app export
-        // from _error instead
-        const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
+              buildManifest,
+              config.experimental.gzipSize,
+              computedManifestData
+            )
 
-        customAppGetInitialProps = await hasCustomGetInitialProps(
-          appPageToCheck,
-          distDir,
-          isLikeServerless,
-          runtimeEnvConfig,
-          true
-        )
+            let isSsg = false
+            let isStatic = false
+            let isHybridAmp = false
+            let ssgPageRoutes: string[] | null = null
 
-        namedExports = await getNamedExports(
-          appPageToCheck,
-          distDir,
-          isLikeServerless,
-          runtimeEnvConfig
-        )
+            const nonReservedPage = !page.match(
+              /^\/(_app|_error|_document|api(\/|$))/
+            )
 
-        if (customAppGetInitialProps) {
-          console.warn(
-            chalk.bold.yellow(`Warning: `) +
-              chalk.yellow(
-                `You have opted-out of Automatic Static Optimization due to \`getInitialProps\` in \`pages/_app\`. This does not opt-out pages with \`getStaticProps\``
-              )
-          )
-          console.warn(
-            'Read more: https://nextjs.org/docs/messages/opt-out-auto-static-optimization\n'
-          )
-        }
+            if (nonReservedPage) {
+              try {
+                let isPageStaticSpan = checkPageSpan.traceChild(
+                  'is-page-static'
+                )
+                let workerResult = await isPageStaticSpan.traceAsyncFn(() => {
+                  return staticCheckWorkers.isPageStatic(
+                    page,
+                    distDir,
+                    isLikeServerless,
+                    runtimeEnvConfig,
+                    config.i18n?.locales,
+                    config.i18n?.defaultLocale,
+                    isPageStaticSpan.id
+                  )
+                })
 
-        const computedManifestData = await computeFromManifest(
-          buildManifest,
-          distDir,
-          config.experimental.gzipSize
-        )
-        await Promise.all(
-          pageKeys.map(async (page) => {
-            const checkPageSpan = staticCheckSpan.traceChild('check-page', {
-              page,
-            })
-            return checkPageSpan.traceAsyncFn(async () => {
-              const actualPage = normalizePagePath(page)
-              const [selfSize, allSize] = await getJsPageSizeInKb(
-                actualPage,
-                distDir,
-                buildManifest,
-                config.experimental.gzipSize,
-                computedManifestData
-              )
+                if (
+                  workerResult.isStatic === false &&
+                  (workerResult.isHybridAmp || workerResult.isAmpOnly)
+                ) {
+                  hasSsrAmpPages = true
+                }
 
-              let isSsg = false
-              let isStatic = false
-              let isHybridAmp = false
-              let ssgPageRoutes: string[] | null = null
+                if (workerResult.isHybridAmp) {
+                  isHybridAmp = true
+                  hybridAmpPages.add(page)
+                }
 
-              const nonReservedPage = !page.match(
-                /^\/(_app|_error|_document|api(\/|$))/
-              )
+                if (workerResult.isNextImageImported) {
+                  isNextImageImported = true
+                }
 
-              if (nonReservedPage) {
-                try {
-                  let isPageStaticSpan = checkPageSpan.traceChild(
-                    'is-page-static'
-                  )
-                  let workerResult = await isPageStaticSpan.traceAsyncFn(() => {
-                    return staticCheckWorkers.isPageStatic(
-                      page,
-                      distDir,
-                      isLikeServerless,
-                      runtimeEnvConfig,
-                      config.i18n?.locales,
-                      config.i18n?.defaultLocale,
-                      isPageStaticSpan.id
-                    )
-                  })
+                if (workerResult.hasStaticProps) {
+                  ssgPages.add(page)
+                  isSsg = true
 
                   if (
-                    workerResult.isStatic === false &&
-                    (workerResult.isHybridAmp || workerResult.isAmpOnly)
+                    workerResult.prerenderRoutes &&
+                    workerResult.encodedPrerenderRoutes
                   ) {
-                    hasSsrAmpPages = true
-                  }
-
-                  if (workerResult.isHybridAmp) {
-                    isHybridAmp = true
-                    hybridAmpPages.add(page)
-                  }
-
-                  if (workerResult.isNextImageImported) {
-                    isNextImageImported = true
-                  }
-
-                  if (workerResult.hasStaticProps) {
-                    ssgPages.add(page)
-                    isSsg = true
-
-                    if (
-                      workerResult.prerenderRoutes &&
+                    additionalSsgPaths.set(page, workerResult.prerenderRoutes)
+                    additionalSsgPathsEncoded.set(
+                      page,
                       workerResult.encodedPrerenderRoutes
-                    ) {
-                      additionalSsgPaths.set(page, workerResult.prerenderRoutes)
-                      additionalSsgPathsEncoded.set(
-                        page,
-                        workerResult.encodedPrerenderRoutes
-                      )
-                      ssgPageRoutes = workerResult.prerenderRoutes
-                    }
-
-                    if (workerResult.prerenderFallback === 'blocking') {
-                      ssgBlockingFallbackPages.add(page)
-                    } else if (workerResult.prerenderFallback === true) {
-                      ssgStaticFallbackPages.add(page)
-                    }
-                  } else if (workerResult.hasServerProps) {
-                    serverPropsPages.add(page)
-                  } else if (
-                    workerResult.isStatic &&
-                    customAppGetInitialProps === false
-                  ) {
-                    staticPages.add(page)
-                    isStatic = true
+                    )
+                    ssgPageRoutes = workerResult.prerenderRoutes
                   }
 
-                  if (hasPages404 && page === '/404') {
-                    if (
-                      !workerResult.isStatic &&
-                      !workerResult.hasStaticProps
-                    ) {
-                      throw new Error(
-                        `\`pages/404\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}`
-                      )
-                    }
-                    // we need to ensure the 404 lambda is present since we use
-                    // it when _app has getInitialProps
-                    if (
-                      customAppGetInitialProps &&
-                      !workerResult.hasStaticProps
-                    ) {
-                      staticPages.delete(page)
-                    }
+                  if (workerResult.prerenderFallback === 'blocking') {
+                    ssgBlockingFallbackPages.add(page)
+                  } else if (workerResult.prerenderFallback === true) {
+                    ssgStaticFallbackPages.add(page)
                   }
+                } else if (workerResult.hasServerProps) {
+                  serverPropsPages.add(page)
+                } else if (
+                  workerResult.isStatic &&
+                  (await customAppGetInitialPropsPromise) === false
+                ) {
+                  staticPages.add(page)
+                  isStatic = true
+                }
 
+                if (hasPages404 && page === '/404') {
+                  if (!workerResult.isStatic && !workerResult.hasStaticProps) {
+                    throw new Error(
+                      `\`pages/404\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}`
+                    )
+                  }
+                  // we need to ensure the 404 lambda is present since we use
+                  // it when _app has getInitialProps
                   if (
-                    STATIC_STATUS_PAGES.includes(page) &&
-                    !workerResult.isStatic &&
+                    (await customAppGetInitialPropsPromise) &&
                     !workerResult.hasStaticProps
                   ) {
-                    throw new Error(
-                      `\`pages${page}\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}`
-                    )
+                    staticPages.delete(page)
                   }
-                } catch (err) {
-                  if (err.message !== 'INVALID_DEFAULT_EXPORT') throw err
-                  invalidPages.add(page)
                 }
+
+                if (
+                  STATIC_STATUS_PAGES.includes(page) &&
+                  !workerResult.isStatic &&
+                  !workerResult.hasStaticProps
+                ) {
+                  throw new Error(
+                    `\`pages${page}\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}`
+                  )
+                }
+              } catch (err) {
+                if (err.message !== 'INVALID_DEFAULT_EXPORT') throw err
+                invalidPages.add(page)
               }
+            }
 
-              pageInfos.set(page, {
-                size: selfSize,
-                totalSize: allSize,
-                static: isStatic,
-                isSsg,
-                isHybridAmp,
-                ssgPageRoutes,
-                initialRevalidateSeconds: false,
-              })
+            pageInfos.set(page, {
+              size: selfSize,
+              totalSize: allSize,
+              static: isStatic,
+              isSsg,
+              isHybridAmp,
+              ssgPageRoutes,
+              initialRevalidateSeconds: false,
             })
           })
-        )
-        staticCheckWorkers.end()
-
-        return { hasNonStaticErrorPage: nonStaticErrorPage }
+        })
+      )
+      const returnValue = {
+        customAppGetInitialProps: await customAppGetInitialPropsPromise,
+        namedExports: await namedExportsPromise,
+        isNextImageImported,
+        hasSsrAmpPages,
+        hasNonStaticErrorPage: await nonStaticErrorPagePromise,
       }
-    )
+
+      staticCheckWorkers.end()
+      return returnValue
+    })
+
+    if (customAppGetInitialProps) {
+      console.warn(
+        chalk.bold.yellow(`Warning: `) +
+          chalk.yellow(
+            `You have opted-out of Automatic Static Optimization due to \`getInitialProps\` in \`pages/_app\`. This does not opt-out pages with \`getStaticProps\``
+          )
+      )
+      console.warn(
+        'Read more: https://nextjs.org/docs/messages/opt-out-auto-static-optimization\n'
+      )
+    }
 
     if (!hasSsrAmpPages) {
       requiredServerFiles.ignore.push(

packages/next/build/webpack-config.ts

@@ -822,7 +822,17 @@ export default async function getBaseWebpackConfig(
       ...(isWebpack5 ? { emitOnErrors: !dev } : { noEmitOnErrors: dev }),
       checkWasmTypes: false,
       nodeEnv: false,
-      splitChunks: isServer ? false : splitChunksConfig,
+      splitChunks: isServer
+        ? isWebpack5
+          ? {
+              // allow to split entrypoints
+              chunks: 'all',
+              // size of files is not so relevant for server build
+              // we want to prefer deduplication to load less code
+              minSize: 1000,
+            }
+          : false
+        : splitChunksConfig,
       runtimeChunk: isServer
         ? isWebpack5 && !isLikeServerless
           ? { name: 'webpack-runtime' }

packages/next/build/webpack/plugins/pages-manifest-plugin.ts

@@ -38,7 +38,7 @@ export default class PagesManifestPlugin implements webpack.Plugin {
             !file.includes('webpack-runtime') && file.endsWith('.js')
         )
 
-      if (files.length > 1) {
+      if (!isWebpack5 && files.length > 1) {
         console.log(
           `Found more than one file in server entrypoint ${entrypoint.name}`,
           files
@@ -47,7 +47,7 @@ export default class PagesManifestPlugin implements webpack.Plugin {
       }
 
       // Write filename, replace any backslashes in path (on windows) with forwardslashes for cross-platform consistency.
-      pages[pagePath] = files[0]
+      pages[pagePath] = files[files.length - 1]
 
       if (isWebpack5 && !this.dev) {
         pages[pagePath] = pages[pagePath].slice(3)

packages/next/client/next-dev.js

@@ -48,7 +48,7 @@ initNext({ webpackHMR })
         const { pages } = JSON.parse(event.data)
         const router = window.next.router
 
-        if (pages.includes(router.pathname)) {
+        if (!router.clc && pages.includes(router.pathname)) {
           console.log('Refreshing page data due to server-side change')
 
           buildIndicatorHandler('building')