Add handling for static generation in app (#40561)

x-ref: [internal notes](https://www.notion.so/vercel/App-Static-Generation-dc5f1e0916684501b586e56a5b7b9483)

Author: ijjk

packages/next/build/index.ts

@@ -97,6 +97,7 @@ import {
   printTreeView,
   copyTracedFiles,
   isReservedPage,
+  AppConfig,
 } from './utils'
 import getBaseWebpackConfig from './webpack-config'
 import { PagesManifest } from './webpack/plugins/pages-manifest-plugin'
@@ -1065,6 +1066,11 @@ export default async function build(
       const serverPropsPages = new Set<string>()
       const additionalSsgPaths = new Map<string, Array<string>>()
       const additionalSsgPathsEncoded = new Map<string, Array<string>>()
+      const appStaticPaths = new Map<string, Array<string>>()
+      const appStaticPathsEncoded = new Map<string, Array<string>>()
+      const appNormalizedPaths = new Map<string, string>()
+      const appDynamicParamPaths = new Set<string>()
+      const appDefaultConfigs = new Map<string, AppConfig>()
       const pageTraceIncludes = new Map<string, Array<string>>()
       const pageTraceExcludes = new Map<string, Array<string>>()
       const pageInfos = new Map<string, PageInfo>()
@@ -1087,6 +1093,26 @@ export default async function build(
         : require.resolve('./utils')
       let infoPrinted = false
 
+      let appPathsManifest: Record<string, string> = {}
+      const appPathRoutes: Record<string, string> = {}
+
+      if (appDir) {
+        appPathsManifest = JSON.parse(
+          await promises.readFile(
+            path.join(distDir, serverDir, APP_PATHS_MANIFEST),
+            'utf8'
+          )
+        )
+
+        Object.keys(appPathsManifest).forEach((entry) => {
+          appPathRoutes[entry] = normalizeAppPath(entry) || '/'
+        })
+        await promises.writeFile(
+          path.join(distDir, APP_PATH_ROUTES_MANIFEST),
+          JSON.stringify(appPathRoutes, null, 2)
+        )
+      }
+
       process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
 
       const staticWorkers = new Worker(staticWorker, {
@@ -1255,33 +1281,49 @@ export default async function build(
                 let isHybridAmp = false
                 let ssgPageRoutes: string[] | null = null
 
-                const pagePath =
-                  pageType === 'pages'
-                    ? pagesPaths.find(
-                        (p) =>
-                          p.startsWith(actualPage + '.') ||
-                          p.startsWith(actualPage + '/index.')
+                let pagePath = ''
+
+                if (pageType === 'pages') {
+                  pagePath =
+                    pagesPaths.find(
+                      (p) =>
+                        p.startsWith(actualPage + '.') ||
+                        p.startsWith(actualPage + '/index.')
+                    ) || ''
+                }
+                let originalAppPath: string | undefined
+
+                if (pageType === 'app' && mappedAppPages) {
+                  for (const [originalPath, normalizedPath] of Object.entries(
+                    appPathRoutes
+                  )) {
+                    if (normalizedPath === page) {
+                      pagePath = mappedAppPages[originalPath].replace(
+                        /^private-next-app-dir/,
+                        ''
                       )
-                    : appPaths?.find((p) => p.startsWith(actualPage + '/page.'))
+                      originalAppPath = originalPath
+                      break
+                    }
+                  }
+                }
 
-                const staticInfo =
-                  pagesDir && pageType === 'pages' && pagePath
-                    ? await getPageStaticInfo({
-                        pageFilePath: join(pagesDir, pagePath),
-                        nextConfig: config,
-                      })
-                    : {}
-                const pageRuntime = staticInfo.runtime
+                const staticInfo = pagePath
+                  ? await getPageStaticInfo({
+                      pageFilePath: join(
+                        (pageType === 'pages' ? pagesDir : appDir) || '',
+                        pagePath
+                      ),
+                      nextConfig: config,
+                    })
+                  : undefined
+
+                const pageRuntime = staticInfo?.runtime
                 isServerComponent =
                   pageType === 'app' &&
-                  staticInfo.rsc !== RSC_MODULE_TYPES.client
+                  staticInfo?.rsc !== RSC_MODULE_TYPES.client
 
-                if (
-                  // Only calculate page static information if the page is not an
-                  // app page.
-                  pageType !== 'app' &&
-                  !isReservedPage(page)
-                ) {
+                if (!isReservedPage(page)) {
                   try {
                     let edgeInfo: any
 
@@ -1291,8 +1333,10 @@ export default async function build(
                         serverDir,
                         MIDDLEWARE_MANIFEST
                       ))
+                      const manifestKey =
+                        pageType === 'pages' ? page : join(page, 'page')
 
-                      edgeInfo = manifest.functions[page]
+                      edgeInfo = manifest.functions[manifestKey]
                     }
 
                     let isPageStaticSpan =
@@ -1301,6 +1345,7 @@ export default async function build(
                       () => {
                         return staticWorkers.isPageStatic({
                           page,
+                          originalAppPath,
                           distDir,
                           serverless: isLikeServerless,
                           configFileName,
@@ -1311,10 +1356,50 @@ export default async function build(
                           parentId: isPageStaticSpan.id,
                           pageRuntime,
                           edgeInfo,
+                          pageType,
+                          hasServerComponents,
                         })
                       }
                     )
 
+                    if (pageType === 'app' && originalAppPath) {
+                      appNormalizedPaths.set(originalAppPath, page)
+
+                      // TODO-APP: handle prerendering with edge
+                      // runtime
+                      if (pageRuntime === 'experimental-edge') {
+                        return
+                      }
+
+                      if (
+                        workerResult.encodedPrerenderRoutes &&
+                        workerResult.prerenderRoutes
+                      ) {
+                        appStaticPaths.set(
+                          originalAppPath,
+                          workerResult.prerenderRoutes
+                        )
+                        appStaticPathsEncoded.set(
+                          originalAppPath,
+                          workerResult.encodedPrerenderRoutes
+                        )
+                      }
+                      if (!isDynamicRoute(page)) {
+                        appStaticPaths.set(originalAppPath, [page])
+                        appStaticPathsEncoded.set(originalAppPath, [page])
+                      }
+                      if (workerResult.prerenderFallback) {
+                        // whether or not to allow requests for paths not
+                        // returned from generateStaticParams
+                        appDynamicParamPaths.add(originalAppPath)
+                      }
+                      appDefaultConfigs.set(
+                        originalAppPath,
+                        workerResult.appConfig || {}
+                      )
+                      return
+                    }
+
                     if (pageRuntime === SERVER_RUNTIME.edge) {
                       if (workerResult.hasStaticProps) {
                         console.warn(
@@ -1821,24 +1906,6 @@ export default async function build(
         'utf8'
       )
 
-      if (appDir) {
-        const appPathsManifest = JSON.parse(
-          await promises.readFile(
-            path.join(distDir, serverDir, APP_PATHS_MANIFEST),
-            'utf8'
-          )
-        )
-        const appPathRoutes: Record<string, string> = {}
-
-        Object.keys(appPathsManifest).forEach((entry) => {
-          appPathRoutes[entry] = normalizeAppPath(entry) || '/'
-        })
-        await promises.writeFile(
-          path.join(distDir, APP_PATH_ROUTES_MANIFEST),
-          JSON.stringify(appPathRoutes, null, 2)
-        )
-      }
-
       const middlewareManifest: MiddlewareManifest = JSON.parse(
         await promises.readFile(
           path.join(distDir, serverDir, MIDDLEWARE_MANIFEST),
@@ -1865,6 +1932,7 @@ export default async function build(
       }
 
       const finalPrerenderRoutes: { [route: string]: SsgRoute } = {}
+      const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {}
       const tbdPrerenderRoutes: string[] = []
       let ssgNotFoundPaths: string[] = []
 
@@ -1889,7 +1957,16 @@ export default async function build(
 
       const combinedPages = [...staticPages, ...ssgPages]
 
-      if (combinedPages.length > 0 || useStatic404 || useDefaultStatic500) {
+      // we need to trigger automatic exporting when we have
+      // - static 404/500
+      // - getStaticProps paths
+      // - experimental app is enabled
+      if (
+        combinedPages.length > 0 ||
+        useStatic404 ||
+        useDefaultStatic500 ||
+        config.experimental.appDir
+      ) {
         const staticGenerationSpan =
           nextBuildSpan.traceChild('static-generation')
         await staticGenerationSpan.traceAsyncFn(async () => {
@@ -1986,6 +2063,20 @@ export default async function build(
                 }
               }
 
+              // TODO: output manifest specific to app paths and their
+              // revalidate periods and dynamicParams settings
+              appStaticPaths.forEach((routes, originalAppPath) => {
+                const encodedRoutes = appStaticPathsEncoded.get(originalAppPath)
+
+                routes.forEach((route, routeIdx) => {
+                  defaultMap[route] = {
+                    page: originalAppPath,
+                    query: { __nextSsgPath: encodedRoutes?.[routeIdx] },
+                    _isAppDir: true,
+                  }
+                })
+              })
+
               if (i18n) {
                 for (const page of [
                   ...staticPages,
@@ -2035,6 +2126,58 @@ export default async function build(
             await promises.unlink(serverBundle)
           }
 
+          for (const [originalAppPath, routes] of appStaticPaths) {
+            const page = appNormalizedPaths.get(originalAppPath) || ''
+            const appConfig = appDefaultConfigs.get(originalAppPath) || {}
+            let hasDynamicData = appConfig.revalidate === 0
+
+            routes.forEach((route) => {
+              let revalidate = exportConfig.initialPageRevalidationMap[route]
+
+              if (typeof revalidate === 'undefined') {
+                revalidate =
+                  typeof appConfig.revalidate !== 'undefined'
+                    ? appConfig.revalidate
+                    : false
+              }
+              if (revalidate !== 0) {
+                const normalizedRoute = normalizePagePath(route)
+                const dataRoute = path.posix.join(`${normalizedRoute}.rsc`)
+                finalPrerenderRoutes[route] = {
+                  initialRevalidateSeconds: revalidate,
+                  srcRoute: page,
+                  dataRoute,
+                }
+              } else {
+                hasDynamicData = true
+              }
+            })
+
+            if (!hasDynamicData && isDynamicRoute(originalAppPath)) {
+              const normalizedRoute = normalizePagePath(page)
+              const dataRoute = path.posix.join(`${normalizedRoute}.rsc`)
+
+              // TODO: create a separate manifest to allow enforcing
+              // dynamicParams for non-static paths?
+              finalDynamicRoutes[page] = {
+                routeRegex: normalizeRouteRegex(
+                  getNamedRouteRegex(page).re.source
+                ),
+                dataRoute,
+                // if dynamicParams are enabled treat as fallback:
+                // 'blocking' if not it's fallback: false
+                fallback: appDynamicParamPaths.has(originalAppPath)
+                  ? null
+                  : false,
+                dataRouteRegex: normalizeRouteRegex(
+                  getNamedRouteRegex(
+                    dataRoute.replace(/\.rsc$/, '')
+                  ).re.source.replace(/\(\?:\\\/\)\?\$$/, '\\.rsc$')
+                ),
+              }
+            }
+          }
+
           const moveExportedPage = async (
             originPage: string,
             page: string,
@@ -2347,8 +2490,7 @@ export default async function build(
         telemetry.record(eventPackageUsedInGetServerSideProps(telemetryPlugin))
       }
 
-      if (ssgPages.size > 0) {
-        const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {}
+      if (ssgPages.size > 0 || appDir) {
         tbdPrerenderRoutes.forEach((tbdRoute) => {
           const normalizedRoute = normalizePagePath(tbdRoute)
           const dataRoute = path.posix.join(

packages/next/build/utils.ts

@@ -813,13 +813,21 @@ export async function getJsPageSizeInKb(
   return [-1, -1]
 }
 
-export async function buildStaticPaths(
-  page: string,
-  getStaticPaths: GetStaticPaths,
-  configFileName: string,
-  locales?: string[],
+export async function buildStaticPaths({
+  page,
+  getStaticPaths,
+  staticPathsResult,
+  configFileName,
+  locales,
+  defaultLocale,
+}: {
+  page: string
+  getStaticPaths?: GetStaticPaths
+  staticPathsResult?: UnwrapPromise<ReturnType<GetStaticPaths>>
+  configFileName: string
+  locales?: string[]
   defaultLocale?: string
-): Promise<
+}): Promise<
   Omit<UnwrapPromise<ReturnType<GetStaticPaths>>, 'paths'> & {
     paths: string[]
     encodedPaths: string[]
@@ -833,7 +841,15 @@ export async function buildStaticPaths(
   // Get the default list of allowed params.
   const _validParamKeys = Object.keys(_routeMatcher(page))
 
-  const staticPathsResult = await getStaticPaths({ locales, defaultLocale })
+  if (!staticPathsResult) {
+    if (getStaticPaths) {
+      staticPathsResult = await getStaticPaths({ locales, defaultLocale })
+    } else {
+      throw new Error(
+        `invariant: attempted to buildStaticPaths without "staticPathsResult" or "getStaticPaths" ${page}`
+      )
+    }
+  }
 
   const expectedReturnVal =
     `Expected: { paths: [], fallback: boolean }\n` +
@@ -1013,6 +1029,137 @@ export async function buildStaticPaths(
   }
 }
 
+export type AppConfig = {
+  revalidate?: number | false
+  dynamicParams?: true | false
+  dynamic?: 'auto' | 'error' | 'force-static'
+  fetchCache?: 'force-cache' | 'only-cache'
+  preferredRegion?: string
+}
+type GenerateParams = Array<{
+  config: AppConfig
+  segmentPath: string
+  getStaticPaths?: GetStaticPaths
+  generateStaticParams?: any
+  isLayout?: boolean
+}>
+
+export const collectGenerateParams = (
+  segment: any,
+  parentSegments: string[] = [],
+  generateParams: GenerateParams = []
+): GenerateParams => {
+  if (!Array.isArray(segment)) return generateParams
+  const isLayout = !!segment[2]?.layout
+  const mod = isLayout ? segment[2]?.layout?.() : segment[2]?.page?.()
+
+  const result = {
+    isLayout,
+    segmentPath: `/${parentSegments.join('/')}${
+      segment[0] && parentSegments.length > 0 ? '/' : ''
+    }${segment[0]}`,
+    config: mod?.config,
+    getStaticPaths: mod?.getStaticPaths,
+    generateStaticParams: mod?.generateStaticParams,
+  }
+
+  if (segment[0]) {
+    parentSegments.push(segment[0])
+  }
+
+  if (result.config || result.generateStaticParams || result.getStaticPaths) {
+    generateParams.push(result)
+  }
+  return collectGenerateParams(
+    segment[1]?.children,
+    parentSegments,
+    generateParams
+  )
+}
+
+export async function buildAppStaticPaths({
+  page,
+  configFileName,
+  generateParams,
+}: {
+  page: string
+  configFileName: string
+  generateParams: GenerateParams
+}) {
+  const pageEntry = generateParams[generateParams.length - 1]
+
+  // if the page has legacy getStaticPaths we call it like normal
+  if (typeof pageEntry?.getStaticPaths === 'function') {
+    return buildStaticPaths({
+      page,
+      configFileName,
+      getStaticPaths: pageEntry.getStaticPaths,
+    })
+  } else {
+    // if generateStaticParams is being used we iterate over them
+    // collecting them from each level
+    type Params = Array<Record<string, string | string[]>>
+    let hadGenerateParams = false
+
+    const buildParams = async (
+      paramsItems: Params = [{}],
+      idx = 0
+    ): Promise<Params> => {
+      const curGenerate = generateParams[idx]
+
+      if (idx === generateParams.length) {
+        return paramsItems
+      }
+      if (
+        typeof curGenerate.generateStaticParams !== 'function' &&
+        idx < generateParams.length
+      ) {
+        return buildParams(paramsItems, idx + 1)
+      }
+      hadGenerateParams = true
+
+      const newParams = []
+
+      for (const params of paramsItems) {
+        const result = await curGenerate.generateStaticParams({ params })
+        // TODO: validate the result is valid here or wait for
+        // buildStaticPaths to validate?
+        for (const item of result.params) {
+          newParams.push({ ...params, ...item })
+        }
+      }
+
+      if (idx < generateParams.length) {
+        return buildParams(newParams, idx + 1)
+      }
+      return newParams
+    }
+    const builtParams = await buildParams()
+    const fallback = !generateParams.some(
+      // TODO: check complementary configs that can impact
+      // dynamicParams behavior
+      (generate) => generate.config?.dynamicParams === false
+    )
+
+    if (!hadGenerateParams) {
+      return {
+        paths: undefined,
+        fallback: undefined,
+        encodedPaths: undefined,
+      }
+    }
+
+    return buildStaticPaths({
+      staticPathsResult: {
+        fallback,
+        paths: builtParams.map((params) => ({ params })),
+      },
+      page,
+      configFileName,
+    })
+  }
+}
+
 export async function isPageStatic({
   page,
   distDir,
@@ -1025,6 +1172,9 @@ export async function isPageStatic({
   parentId,
   pageRuntime,
   edgeInfo,
+  pageType,
+  hasServerComponents,
+  originalAppPath,
 }: {
   page: string
   distDir: string
@@ -1036,7 +1186,10 @@ export async function isPageStatic({
   defaultLocale?: string
   parentId?: any
   edgeInfo?: any
+  pageType?: 'pages' | 'app'
   pageRuntime: ServerRuntime
+  hasServerComponents?: boolean
+  originalAppPath?: string
 }): Promise<{
   isStatic?: boolean
   isAmpOnly?: boolean
@@ -1049,6 +1202,7 @@ export async function isPageStatic({
   isNextImageImported?: boolean
   traceIncludes?: string[]
   traceExcludes?: string[]
+  appConfig?: AppConfig
 }> {
   const isPageStaticSpan = trace('is-page-static-utils', parentId)
   return isPageStaticSpan
@@ -1057,6 +1211,10 @@ export async function isPageStatic({
       setHttpAgentOptions(httpAgentOptions)
 
       let componentsResult: LoadComponentsReturnType
+      let prerenderRoutes: Array<string> | undefined
+      let encodedPrerenderRoutes: Array<string> | undefined
+      let prerenderFallback: boolean | 'blocking' | undefined
+      let appConfig: AppConfig = {}
 
       if (pageRuntime === SERVER_RUNTIME.edge) {
         const runtime = await getRuntimeContext({
@@ -1084,16 +1242,74 @@ export async function isPageStatic({
       } else {
         componentsResult = await loadComponents({
           distDir,
-          pathname: page,
+          pathname: originalAppPath || page,
           serverless,
-          hasServerComponents: false,
-          isAppPath: false,
+          hasServerComponents: !!hasServerComponents,
+          isAppPath: pageType === 'app',
         })
       }
-      const Comp = componentsResult.Component
+      const Comp = componentsResult.Component || {}
+      let staticPathsResult:
+        | UnwrapPromise<ReturnType<GetStaticPaths>>
+        | undefined
+
+      if (pageType === 'app') {
+        const tree = componentsResult.ComponentMod.tree
+        const generateParams = collectGenerateParams(tree)
+
+        appConfig = generateParams.reduce(
+          (builtConfig: AppConfig, curGenParams): AppConfig => {
+            const {
+              dynamic,
+              fetchCache,
+              preferredRegion,
+              revalidate: curRevalidate,
+            } = curGenParams?.config || {}
+
+            // TODO: should conflicting configs here throw an error
+            // e.g. if layout defines one region but page defines another
+            if (typeof builtConfig.preferredRegion === 'undefined') {
+              builtConfig.preferredRegion = preferredRegion
+            }
+            if (typeof builtConfig.dynamic === 'undefined') {
+              builtConfig.dynamic = dynamic
+            }
+            if (typeof builtConfig.fetchCache === 'undefined') {
+              builtConfig.fetchCache = fetchCache
+            }
+
+            // any revalidate number overrides false
+            // shorter revalidate overrides longer (initially)
+            if (typeof builtConfig.revalidate === 'undefined') {
+              builtConfig.revalidate = curRevalidate
+            }
+            if (
+              typeof curRevalidate === 'number' &&
+              (typeof builtConfig.revalidate !== 'number' ||
+                curRevalidate < builtConfig.revalidate)
+            ) {
+              builtConfig.revalidate = curRevalidate
+            }
+            return builtConfig
+          },
+          {}
+        )
 
-      if (!Comp || !isValidElementType(Comp) || typeof Comp === 'string') {
-        throw new Error('INVALID_DEFAULT_EXPORT')
+        if (isDynamicRoute(page)) {
+          ;({
+            paths: prerenderRoutes,
+            fallback: prerenderFallback,
+            encodedPaths: encodedPrerenderRoutes,
+          } = await buildAppStaticPaths({
+            page,
+            configFileName,
+            generateParams,
+          }))
+        }
+      } else {
+        if (!Comp || !isValidElementType(Comp) || typeof Comp === 'string') {
+          throw new Error('INVALID_DEFAULT_EXPORT')
+        }
       }
 
       const hasGetInitialProps = !!(Comp as any).getInitialProps
@@ -1163,21 +1379,19 @@ export async function isPageStatic({
         )
       }
 
-      let prerenderRoutes: Array<string> | undefined
-      let encodedPrerenderRoutes: Array<string> | undefined
-      let prerenderFallback: boolean | 'blocking' | undefined
-      if (hasStaticProps && hasStaticPaths) {
+      if ((hasStaticProps && hasStaticPaths) || staticPathsResult) {
         ;({
           paths: prerenderRoutes,
           fallback: prerenderFallback,
           encodedPaths: encodedPrerenderRoutes,
-        } = await buildStaticPaths(
+        } = await buildStaticPaths({
           page,
-          componentsResult.getStaticPaths!,
-          configFileName,
           locales,
-          defaultLocale
-        ))
+          defaultLocale,
+          configFileName,
+          staticPathsResult,
+          getStaticPaths: componentsResult.getStaticPaths!,
+        }))
       }
 
       const isNextImageImported = (global as any).__NEXT_IMAGE_IMPORTED
@@ -1194,6 +1408,7 @@ export async function isPageStatic({
         isNextImageImported,
         traceIncludes: config.unstable_includeFiles || [],
         traceExcludes: config.unstable_excludeFiles || [],
+        appConfig,
       }
     })
     .catch((err) => {