fix: only set permament caching header when reading html file when its not during server initialization AND when read html is Next produced fully static html

Author: pieh

src/build/content/static.ts

@@ -27,21 +27,23 @@ export const copyStaticContent = async (ctx: PluginContext): Promise<void> => {
     })
 
     const fallbacks = ctx.getFallbacks(await ctx.getPrerenderManifest())
+    const fullyStaticPages = await ctx.getFullyStaticHtmlPages()
 
     try {
       await mkdir(destDir, { recursive: true })
       await Promise.all(
         paths
-          .filter((path) => !paths.includes(`${path.slice(0, -5)}.json`))
+          .filter((path) => !path.endsWith('.json') && !paths.includes(`${path.slice(0, -5)}.json`))
           .map(async (path): Promise<void> => {
             const html = await readFile(join(srcDir, path), 'utf-8')
             verifyNetlifyForms(ctx, html)
 
             const isFallback = fallbacks.includes(path.slice(0, -5))
+            const isFullyStaticPage = !isFallback && fullyStaticPages.includes(path)
 
             await writeFile(
               join(destDir, await encodeBlobKey(path)),
-              JSON.stringify({ html, isFallback } satisfies HtmlBlob),
+              JSON.stringify({ html, isFullyStaticPage } satisfies HtmlBlob),
               'utf-8',
             )
           }),

src/build/plugin-context.ts

@@ -12,6 +12,7 @@ import type {
 } from '@netlify/build'
 import type { PrerenderManifest, RoutesManifest } from 'next/dist/build/index.js'
 import type { MiddlewareManifest } from 'next/dist/build/webpack/plugins/middleware-plugin.js'
+import type { PagesManifest } from 'next/dist/build/webpack/plugins/pages-manifest-plugin.js'
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
 import { satisfies } from 'semver'
 
@@ -370,6 +371,35 @@ export class PluginContext {
     return this.#fallbacks
   }
 
+  #fullyStaticHtmlPages: string[] | null = null
+  /**
+   * Get an array of fully static pages router pages (no `getServerSideProps` or `getStaticProps`).
+   * Those are being served as-is without involving CacheHandler, so we need to keep track of them
+   * to make sure we apply permanent caching headers for responses that use them.
+   */
+  async getFullyStaticHtmlPages(): Promise<string[]> {
+    if (!this.#fullyStaticHtmlPages) {
+      const pagesManifest = JSON.parse(
+        await readFile(join(this.publishDir, 'server/pages-manifest.json'), 'utf-8'),
+      ) as PagesManifest
+
+      this.#fullyStaticHtmlPages = Object.values(pagesManifest)
+        .filter(
+          (filePath) =>
+            // Limit handling to pages router files (App Router pages should not be included in pages-manifest.json
+            // as they have their own app-paths-manifest.json)
+            filePath.startsWith('pages/') &&
+            // Fully static pages will have entries in the pages-manifest.json pointing to .html files.
+            // Pages with data fetching exports will point to .js files.
+            filePath.endsWith('.html'),
+        )
+        // values will be prefixed with `pages/`, so removing it here for consistency with other methods
+        // like `getFallbacks` that return the route without the prefix
+        .map((filePath) => relative('pages', filePath))
+    }
+    return this.#fullyStaticHtmlPages
+  }
+
   /** Fails a build with a message and an optional error */
   failBuild(message: string, error?: unknown): never {
     return this.utils.build.failBuild(message, error instanceof Error ? { error } : undefined)

src/run/next.cts

@@ -1,5 +1,6 @@
-import fs from 'fs/promises'
-import { relative, resolve } from 'path'
+import { AsyncLocalStorage } from 'node:async_hooks'
+import fs from 'node:fs/promises'
+import { relative, resolve } from 'node:path'
 
 // @ts-expect-error no types installed
 import { patchFs } from 'fs-monkey'
@@ -79,6 +80,13 @@ ResponseCache.prototype.get = function get(...getArgs: unknown[]) {
 type FS = typeof import('fs')
 
 export async function getMockedRequestHandler(...args: Parameters<typeof getRequestHandlers>) {
+  const initContext = { initializingServer: true }
+  /**
+   * Using async local storage to identify operations happening as part of server initialization
+   * and not part of handling of current request.
+   */
+  const initAsyncLocalStorage = new AsyncLocalStorage<typeof initContext>()
+
   const tracer = getTracer()
   return tracer.withActiveSpan('mocked request handler', async () => {
     const ofs = { ...fs }
@@ -96,9 +104,21 @@ export async function getMockedRequestHandler(...args: Parameters<typeof getRequ
           const relPath = relative(resolve('.next/server/pages'), path)
           const file = await cacheStore.get<HtmlBlob>(relPath, 'staticHtml.get')
           if (file !== null) {
-            if (!file.isFallback) {
+            if (file.isFullyStaticPage) {
               const requestContext = getRequestContext()
-              if (requestContext) {
+              // On server initialization Next.js attempt to preload all pages
+              // which might result in reading .html files from the file system
+              // for fully static pages. We don't want to capture those cases.
+              // Note that Next.js does NOT cache read html files so on actual requests
+              // that those will be served, it will read those AGAIN and then we do
+              // want to capture fact of reading them.
+              const { initializingServer } = initAsyncLocalStorage.getStore() ?? {}
+              if (!initializingServer && requestContext) {
+                console.log('setting usedFsReadForNonFallback to true', {
+                  requestContext,
+                  initContext: initAsyncLocalStorage.getStore(),
+                  initializingServer,
+                })
                 requestContext.usedFsReadForNonFallback = true
               }
             }
@@ -120,7 +140,12 @@ export async function getMockedRequestHandler(...args: Parameters<typeof getRequ
       require('fs').promises,
     )
 
-    const requestHandlers = await getRequestHandlers(...args)
+    const requestHandlers = await initAsyncLocalStorage.run(initContext, async () => {
+      // we need to await getRequestHandlers(...) promise in this callback to ensure that initAsyncLocalStorage
+      // is available in async / background work
+      return await getRequestHandlers(...args)
+    })
+
     // depending on Next.js version requestHandlers might be an array of object
     // see https://github.com/vercel/next.js/commit/08e7410f15706379994b54c3195d674909a8d533#diff-37243d614f1f5d3f7ea50bbf2af263f6b1a9a4f70e84427977781e07b02f57f1R742
     return Array.isArray(requestHandlers) ? requestHandlers[0] : requestHandlers.requestHandler

src/shared/blob-types.cts

@@ -4,7 +4,7 @@ export type TagManifest = { revalidatedAt: number }
 
 export type HtmlBlob = {
   html: string
-  isFallback: boolean
+  isFullyStaticPage: boolean
 }
 
 export type BlobType = NetlifyCacheHandlerValue | TagManifest | HtmlBlob
@@ -24,9 +24,9 @@ export const isHtmlBlob = (value: BlobType): value is HtmlBlob => {
     typeof value === 'object' &&
     value !== null &&
     'html' in value &&
-    'isFallback' in value &&
+    'isFullyStaticPage' in value &&
     typeof value.html === 'string' &&
-    typeof value.isFallback === 'boolean' &&
+    typeof value.isFullyStaticPage === 'boolean' &&
     Object.keys(value).length === 2
   )
 }