chore: don't use extra fallback manifest and instead store html and boolean wether that is fallback html in single blob

pieh · pieh · commit e5de50e28c78 · 2024-10-16T15:10:11.000+02:00
diff --git a/src/build/content/prerendered.ts b/src/build/content/prerendered.ts
@@ -1,16 +1,13 @@
 import { existsSync } from 'node:fs'
 import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
-import { join as posixJoin } from 'node:path/posix'
 
 import { trace } from '@opentelemetry/api'
 import { wrapTracer } from '@opentelemetry/api/experimental'
 import { glob } from 'fast-glob'
 import pLimit from 'p-limit'
 import { satisfies } from 'semver'
 
-import { FS_BLOBS_MANIFEST } from '../../run/constants.js'
-import { type FSBlobsManifest } from '../../run/next.cjs'
 import { encodeBlobKey } from '../../shared/blobkey.js'
 import type {
   CachedFetchValueForMultipleVersions,
@@ -160,11 +157,6 @@ export const copyPrerenderedContent = async (ctx: PluginContext): Promise<void>
           })
         : false
 
-      const fsBlobsManifest: FSBlobsManifest = {
-        fallbackPaths: [],
-        outputRoot: ctx.distDir,
-      }
-
       await Promise.all([
         ...Object.entries(manifest.routes).map(
           ([route, meta]): Promise<void> =>
@@ -214,41 +206,15 @@ export const copyPrerenderedContent = async (ctx: PluginContext): Promise<void>
               await writeCacheEntry(key, value, lastModified, ctx)
             }),
         ),
-        ...Object.entries(manifest.dynamicRoutes).map(async ([route, meta]) => {
-          // fallback can be `string | false | null`
-          //  - `string` - when user use pages router with `fallback: true`, and then it's html file path
-          //  - `null` - when user use pages router with `fallback: 'block'` or app router with `export const dynamicParams = true`
-          //  - `false` - when user use pages router with `fallback: false` or app router with `export const dynamicParams = false`
-          if (typeof meta.fallback === 'string') {
-            // https://github.com/vercel/next.js/pull/68603 started using route cache to serve fallbacks
-            // so we have to seed blobs with fallback entries
-
-            // create cache entry for pages router with `fallback: true` case
-            await limitConcurrentPrerenderContentHandling(async () => {
-              // dynamic routes don't have entries for each locale so we have to generate them
-              // ourselves. If i18n is not used we use empty string as "locale" to be able to use
-              // same handling wether i18n is used or not
-              const locales = ctx.buildConfig.i18n?.locales ?? ['']
+        ...ctx.getFallbacks(manifest).map(async (route) => {
+          const key = routeToFilePath(route)
+          const value = await buildPagesCacheValue(
+            join(ctx.publishDir, 'server/pages', key),
+            shouldUseEnumKind,
+            true, // there is no corresponding json file for fallback, so we are skipping it for this entry
+          )
 
-              const lastModified = Date.now()
-              for (const locale of locales) {
-                const key = routeToFilePath(posixJoin(locale, route))
-                const value = await buildPagesCacheValue(
-                  join(ctx.publishDir, 'server/pages', key),
-                  shouldUseEnumKind,
-                  true, // there is no corresponding json file for fallback, so we are skipping it for this entry
-                )
-                // Netlify Forms are not support and require a workaround
-                if (value.kind === 'PAGE' || value.kind === 'PAGES' || value.kind === 'APP_PAGE') {
-                  verifyNetlifyForms(ctx, value.html)
-                }
-
-                await writeCacheEntry(key, value, lastModified, ctx)
-
-                fsBlobsManifest.fallbackPaths.push(`${key}.html`)
-              }
-            })
-          }
+          await writeCacheEntry(key, value, Date.now(), ctx)
         }),
       ])
 
@@ -263,10 +229,6 @@ export const copyPrerenderedContent = async (ctx: PluginContext): Promise<void>
         )
         await writeCacheEntry(key, value, lastModified, ctx)
       }
-      await writeFile(
-        join(ctx.serverHandlerDir, FS_BLOBS_MANIFEST),
-        JSON.stringify(fsBlobsManifest),
-      )
     } catch (error) {
       ctx.failBuild('Failed assembling prerendered content for upload', error)
     }
diff --git a/src/build/content/static.test.ts b/src/build/content/static.test.ts
@@ -25,6 +25,7 @@ const createFsFixtureWithBasePath = (
 ) => {
   return createFsFixture(
     {
+      [join(ctx.publishDir, 'prerender-manifest.json')]: JSON.stringify({ dynamicRoutes: [] }),
       ...fixture,
       [join(ctx.publishDir, 'routes-manifest.json')]: JSON.stringify({ basePath }),
       [join(ctx.publishDir, 'required-server-files.json')]: JSON.stringify({
diff --git a/src/build/content/static.ts b/src/build/content/static.ts
@@ -6,6 +6,7 @@ import { trace } from '@opentelemetry/api'
 import { wrapTracer } from '@opentelemetry/api/experimental'
 import glob from 'fast-glob'
 
+import type { HtmlBlob } from '../../run/next.cjs'
 import { encodeBlobKey } from '../../shared/blobkey.js'
 import { PluginContext } from '../plugin-context.js'
 import { verifyNetlifyForms } from '../verification.js'
@@ -25,6 +26,8 @@ export const copyStaticContent = async (ctx: PluginContext): Promise<void> => {
       extglob: true,
     })
 
+    const fallbacks = ctx.getFallbacks(await ctx.getPrerenderManifest())
+
     try {
       await mkdir(destDir, { recursive: true })
       await Promise.all(
@@ -33,7 +36,14 @@ export const copyStaticContent = async (ctx: PluginContext): Promise<void> => {
           .map(async (path): Promise<void> => {
             const html = await readFile(join(srcDir, path), 'utf-8')
             verifyNetlifyForms(ctx, html)
-            await writeFile(join(destDir, await encodeBlobKey(path)), html, 'utf-8')
+
+            const isFallback = fallbacks.includes(path.slice(0, -5))
+
+            await writeFile(
+              join(destDir, await encodeBlobKey(path)),
+              JSON.stringify({ html, isFallback } satisfies HtmlBlob),
+              'utf-8',
+            )
           }),
       )
     } catch (error) {
diff --git a/src/build/plugin-context.ts b/src/build/plugin-context.ts
@@ -334,6 +334,42 @@ export class PluginContext {
     return this.#nextVersion
   }
 
+  #fallbacks: string[] | null = null
+  /**
+   * Get an array of localized fallback routes
+   *
+   * Example return value for non-i18n site: `['blog/[slug]']`
+   *
+   * Example return value for i18n site: `['en/blog/[slug]', 'fr/blog/[slug]']`
+   */
+  getFallbacks(prerenderManifest: PrerenderManifest): string[] {
+    if (!this.#fallbacks) {
+      // dynamic routes don't have entries for each locale so we have to generate them
+      // ourselves. If i18n is not used we use empty string as "locale" to be able to use
+      // same handling wether i18n is used or not
+      const locales = this.buildConfig.i18n?.locales ?? ['']
+
+      this.#fallbacks = Object.entries(prerenderManifest.dynamicRoutes).reduce(
+        (fallbacks, [route, meta]) => {
+          // fallback can be `string | false | null`
+          //  - `string` - when user use pages router with `fallback: true`, and then it's html file path
+          //  - `null` - when user use pages router with `fallback: 'block'` or app router with `export const dynamicParams = true`
+          //  - `false` - when user use pages router with `fallback: false` or app router with `export const dynamicParams = false`
+          if (typeof meta.fallback === 'string') {
+            for (const locale of locales) {
+              const localizedRoute = posixJoin(locale, route.replace(/^\/+/g, ''))
+              fallbacks.push(localizedRoute)
+            }
+          }
+          return fallbacks
+        },
+        [] as string[],
+      )
+    }
+
+    return this.#fallbacks
+  }
+
   /** Fails a build with a message and an optional error */
   failBuild(message: string, error?: unknown): never {
     return this.utils.build.failBuild(message, error instanceof Error ? { error } : undefined)
diff --git a/src/run/constants.ts b/src/run/constants.ts
@@ -5,4 +5,3 @@ export const MODULE_DIR = fileURLToPath(new URL('.', import.meta.url))
 export const PLUGIN_DIR = resolve(`${MODULE_DIR}../../..`)
 // a file where we store the required-server-files config object in to access during runtime
 export const RUN_CONFIG = 'run-config.json'
-export const FS_BLOBS_MANIFEST = 'fs-blobs-manifest.json'
diff --git a/src/run/next.cts b/src/run/next.cts
@@ -1,5 +1,5 @@
-import fs, { readFile } from 'fs/promises'
-import { join, relative, resolve } from 'path'
+import fs from 'fs/promises'
+import { relative, resolve } from 'path'
 
 // @ts-expect-error no types installed
 import { patchFs } from 'fs-monkey'
@@ -80,25 +80,9 @@ console.timeEnd('import next server')
 
 type FS = typeof import('fs')
 
-export type FSBlobsManifest = {
-  fallbackPaths: string[]
-  outputRoot: string
-}
-
-function normalizeStaticAssetPath(path: string) {
-  return path.startsWith('/') ? path : `/${path}`
-}
-
-let fsBlobsManifestPromise: Promise<FSBlobsManifest> | undefined
-const getFSBlobsManifest = (): Promise<FSBlobsManifest> => {
-  if (!fsBlobsManifestPromise) {
-    fsBlobsManifestPromise = (async () => {
-      const { FS_BLOBS_MANIFEST, PLUGIN_DIR } = await import('./constants.js')
-      return JSON.parse(await readFile(resolve(PLUGIN_DIR, FS_BLOBS_MANIFEST), 'utf-8'))
-    })()
-  }
-
-  return fsBlobsManifestPromise
+export type HtmlBlob = {
+  html: string
+  isFallback: boolean
 }
 
 export async function getMockedRequestHandlers(...args: Parameters<typeof getRequestHandlers>) {
@@ -117,20 +101,20 @@ export async function getMockedRequestHandlers(...args: Parameters<typeof getReq
       } catch (error) {
         // only try to get .html files from the blob store
         if (typeof path === 'string' && path.endsWith('.html')) {
-          const fsBlobsManifest = await getFSBlobsManifest()
-
           const store = getRegionalBlobStore()
-          const relPath = relative(resolve(join(fsBlobsManifest.outputRoot, '/server/pages')), path)
-          const file = await store.get(await encodeBlobKey(relPath))
+          const relPath = relative(resolve('.next/server/pages'), path)
+          const file = (await store.get(await encodeBlobKey(relPath), {
+            type: 'json',
+          })) as HtmlBlob | null
           if (file !== null) {
-            if (!fsBlobsManifest.fallbackPaths.includes(normalizeStaticAssetPath(relPath))) {
+            if (!file.isFallback) {
               const requestContext = getRequestContext()
               if (requestContext) {
                 requestContext.usedFsReadForNonFallback = true
               }
             }
 
-            return file
+            return file.html
           }
         }
 

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ const createFsFixtureWithBasePath = (`
`25`	`25`	`) => {`
`26`	`26`	`return createFsFixture(`
`27`	`27`	`{`
	`28`	`+ [join(ctx.publishDir, 'prerender-manifest.json')]: JSON.stringify({ dynamicRoutes: [] }),`
`28`	`29`	`...fixture,`
`29`	`30`	`[join(ctx.publishDir, 'routes-manifest.json')]: JSON.stringify({ basePath }),`
`30`	`31`	`[join(ctx.publishDir, 'required-server-files.json')]: JSON.stringify({`