feat: Use none bundler for SSR Routes (#2084)

* feat: use "none bundler" for SSR

* chore: bump prerelease version

* chore: fix linting

* chore: enable new path for e2e tests

* fix: include static manifest

* fix: include app-dir files from `app-paths-manifest.json`

* fix: add fallback for when no app-paths-manifest was found

* Update packages/runtime/src/index.ts

---------

Co-authored-by: Eduardo Bouças <[email protected]>
Co-authored-by: Rob Stanford <[email protected]>
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

Author: 4 people

.eslintrc.js

@@ -51,6 +51,7 @@ module.exports = {
         // https://github.com/typescript-eslint/typescript-eslint/issues/2483
         'no-shadow': 'off',
         '@typescript-eslint/no-shadow': 'error',
+        'import/max-dependencies': 'off',
       },
     },
     {

demos/canary/netlify.toml

@@ -3,6 +3,10 @@ command = "next build"
 publish = ".next"
 ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ../..; fi;"
 
+[build.environment]
+NEXT_SPLIT_API_ROUTES = "true"
+NEXT_BUNDLE_BASED_ON_NFT_FILES = "true"
+
 [[plugins]]
 package = "@netlify/plugin-nextjs"
 

demos/default/netlify.toml

@@ -10,6 +10,8 @@ CYPRESS_CACHE_FOLDER = "../node_modules/.CypressBinary"
 # set TERM variable for terminal output
 TERM = "xterm"
 NODE_VERSION = "16.15.1"
+NEXT_SPLIT_API_ROUTES = "true"
+NEXT_BUNDLE_BASED_ON_NFT_FILES = "true"
 
 [[headers]]
 for = "/_next/image/*"

demos/middleware/netlify.toml

@@ -3,6 +3,10 @@ command = "npm run build"
 publish = ".next"
 ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ../..; fi;"
 
+[build.environment]
+NEXT_SPLIT_API_ROUTES = "true"
+NEXT_BUNDLE_BASED_ON_NFT_FILES = "true"
+
 [[plugins]]
 package = "@netlify/plugin-nextjs"
 

demos/nx-next-monorepo-demo/netlify.toml

@@ -5,6 +5,7 @@ ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff
 
 [build.environment]
 NEXT_SPLIT_API_ROUTES = "true"
+NEXT_BUNDLE_BASED_ON_NFT_FILES = "true"
 
 [dev]
 command = "npm run start"

demos/static-root/netlify.toml

@@ -5,6 +5,7 @@ ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff
 
 [build.environment]
 NEXT_SPLIT_API_ROUTES = "true"
+NEXT_BUNDLE_BASED_ON_NFT_FILES = "true"
 
 [[plugins]]
 package = "@netlify/plugin-nextjs"

packages/runtime/src/helpers/config.ts

@@ -8,7 +8,7 @@ import slash from 'slash'
 
 import { HANDLER_FUNCTION_NAME, IMAGE_FUNCTION_NAME, ODB_FUNCTION_NAME } from '../constants'
 
-import type { APILambda } from './functions'
+import type { APILambda, SSRLambda } from './functions'
 import type { RoutesManifest } from './types'
 import { escapeStringRegexp } from './utils'
 
@@ -107,12 +107,14 @@ export const configureHandlerFunctions = async ({
   publish,
   ignore = [],
   apiLambdas,
+  ssrLambdas,
   splitApiRoutes,
 }: {
   netlifyConfig: NetlifyConfig
   publish: string
   ignore: Array<string>
   apiLambdas: APILambda[]
+  ssrLambdas: SSRLambda[]
   splitApiRoutes: boolean
 }) => {
   const config = await getRequiredServerFiles(publish)
@@ -170,17 +172,23 @@ export const configureHandlerFunctions = async ({
     })
   }
 
-  configureFunction(HANDLER_FUNCTION_NAME)
-  configureFunction(ODB_FUNCTION_NAME)
+  const configureLambda = (lambda: APILambda) => {
+    const { functionName, includedFiles } = lambda
+    netlifyConfig.functions[functionName] ||= { included_files: [] }
+    netlifyConfig.functions[functionName].node_bundler = 'none'
+    netlifyConfig.functions[functionName].included_files ||= []
+    netlifyConfig.functions[functionName].included_files.push(...includedFiles.map(escapeGlob))
+  }
+
+  if (ssrLambdas.length === 0) {
+    configureFunction(HANDLER_FUNCTION_NAME)
+    configureFunction(ODB_FUNCTION_NAME)
+  } else {
+    ssrLambdas.forEach(configureLambda)
+  }
 
   if (splitApiRoutes) {
-    for (const apiLambda of apiLambdas) {
-      const { functionName, includedFiles } = apiLambda
-      netlifyConfig.functions[functionName] ||= { included_files: [] }
-      netlifyConfig.functions[functionName].node_bundler = 'none'
-      netlifyConfig.functions[functionName].included_files ||= []
-      netlifyConfig.functions[functionName].included_files.push(...includedFiles.map(escapeGlob))
-    }
+    apiLambdas.forEach(configureLambda)
   } else {
     configureFunction('_api_*')
   }

packages/runtime/src/helpers/flags.ts

@@ -29,3 +29,10 @@ export const splitApiRoutes = (featureFlags: Record<string, unknown>, publish: s
 
   return isEnabled
 }
+
+export const bundleBasedOnNftFiles = (featureFlags: Record<string, unknown>): boolean => {
+  const isEnabled =
+    destr(process.env.NEXT_BUNDLE_BASED_ON_NFT_FILES) ?? featureFlags.next_bundle_based_on_nft_files ?? false
+
+  return isEnabled
+}

packages/runtime/src/helpers/functions.ts

@@ -3,9 +3,10 @@ import bridgeFile from '@vercel/node-bridge'
 import chalk from 'chalk'
 import destr from 'destr'
 import { copyFile, ensureDir, existsSync, readJSON, writeFile, writeJSON, stat } from 'fs-extra'
+import { PrerenderManifest } from 'next/dist/build'
 import type { ImageConfigComplete, RemotePattern } from 'next/dist/shared/lib/image-config'
 import { outdent } from 'outdent'
-import { join, relative, resolve, dirname } from 'pathe'
+import { join, relative, resolve, dirname, basename, extname } from 'pathe'
 import glob from 'tiny-glob'
 
 import {
@@ -32,27 +33,35 @@ import { pack } from './pack'
 import { ApiRouteType } from './types'
 import { getFunctionNameForPage } from './utils'
 
-export interface ApiRouteConfig {
+export interface RouteConfig {
   functionName: string
   functionTitle?: string
   route: string
-  config: ApiConfig
   compiled: string
   includedFiles: string[]
 }
 
-export interface APILambda {
+export interface ApiRouteConfig extends RouteConfig {
+  config: ApiConfig
+}
+
+export interface SSRLambda {
   functionName: string
   functionTitle: string
-  routes: ApiRouteConfig[]
+  routes: RouteConfig[]
   includedFiles: string[]
+}
+
+export interface APILambda extends SSRLambda {
+  routes: ApiRouteConfig[]
   type?: ApiRouteType
 }
 
 export const generateFunctions = async (
   { FUNCTIONS_SRC = DEFAULT_FUNCTIONS_SRC, INTERNAL_FUNCTIONS_SRC, PUBLISH_DIR }: NetlifyPluginConstants,
   appDir: string,
   apiLambdas: APILambda[],
+  ssrLambdas: SSRLambda[],
 ): Promise<void> => {
   const publish = resolve(PUBLISH_DIR)
   const functionsDir = resolve(INTERNAL_FUNCTIONS_SRC || FUNCTIONS_SRC)
@@ -144,6 +153,12 @@ export const generateFunctions = async (
       join(functionsDir, functionName, 'handlerUtils.js'),
     )
     await writeFunctionConfiguration({ functionName, functionTitle, functionsDir })
+
+    const nfInternalFiles = await glob(join(functionsDir, functionName, '**'))
+    const lambda = ssrLambdas.find((l) => l.functionName === functionName)
+    if (lambda) {
+      lambda.includedFiles.push(...nfInternalFiles)
+    }
   }
 
   await writeHandler(HANDLER_FUNCTION_NAME, HANDLER_FUNCTION_TITLE, false)
@@ -295,13 +310,17 @@ export const traceNPMPackage = async (packageName: string, publish: string) => {
   }
 }
 
-export const getAPIPRouteCommonDependencies = async (publish: string) => {
+export const getCommonDependencies = async (publish: string) => {
   const deps = await Promise.all([
     traceRequiredServerFiles(publish),
     traceNextServer(publish),
 
     // used by our own bridge.js
     traceNPMPackage('follow-redirects', publish),
+
+    // using package.json because otherwise, we'd find some /dist/... path
+    traceNPMPackage('@netlify/functions/package.json', publish),
+    traceNPMPackage('is-promise', publish),
   ])
 
   return deps.flat(1)
@@ -329,12 +348,106 @@ const getBundleWeight = async (patterns: string[]) => {
   return sum(sizes.flat(1))
 }
 
+const changeExtension = (file: string, extension: string) => {
+  const base = basename(file, extname(file))
+  return join(dirname(file), base + extension)
+}
+
+const getSSRDependencies = async (publish: string): Promise<string[]> => {
+  const prerenderManifest: PrerenderManifest = await readJSON(join(publish, 'prerender-manifest.json'))
+
+  return [
+    ...Object.entries(prerenderManifest.routes).flatMap(([route, ssgRoute]) => {
+      if (ssgRoute.initialRevalidateSeconds === false) {
+        return []
+      }
+
+      if (ssgRoute.dataRoute.endsWith('.rsc')) {
+        return [
+          join(publish, 'server', 'app', ssgRoute.dataRoute),
+          join(publish, 'server', 'app', changeExtension(ssgRoute.dataRoute, '.html')),
+        ]
+      }
+
+      const trimmedPath = route === '/' ? 'index' : route.slice(1)
+      return [
+        join(publish, 'server', 'pages', `${trimmedPath}.html`),
+        join(publish, 'server', 'pages', `${trimmedPath}.json`),
+      ]
+    }),
+    join(publish, '**', '*.html'),
+    join(publish, 'static-manifest.json'),
+  ]
+}
+
+export const getSSRLambdas = async (publish: string): Promise<SSRLambda[]> => {
+  const commonDependencies = await getCommonDependencies(publish)
+  const ssrRoutes = await getSSRRoutes(publish)
+
+  // TODO: for now, they're the same - but we should separate them
+  const nonOdbRoutes = ssrRoutes
+  const odbRoutes = ssrRoutes
+
+  const ssrDependencies = await getSSRDependencies(publish)
+
+  return [
+    {
+      functionName: HANDLER_FUNCTION_NAME,
+      functionTitle: HANDLER_FUNCTION_TITLE,
+      includedFiles: [
+        ...commonDependencies,
+        ...ssrDependencies,
+        ...nonOdbRoutes.flatMap((route) => route.includedFiles),
+      ],
+      routes: nonOdbRoutes,
+    },
+    {
+      functionName: ODB_FUNCTION_NAME,
+      functionTitle: ODB_FUNCTION_TITLE,
+      includedFiles: [...commonDependencies, ...ssrDependencies, ...odbRoutes.flatMap((route) => route.includedFiles)],
+      routes: odbRoutes,
+    },
+  ]
+}
+
+const getSSRRoutes = async (publish: string): Promise<RouteConfig[]> => {
+  const pageManifest = (await readJSON(join(publish, 'server', 'pages-manifest.json'))) as Record<string, string>
+  const pageManifestRoutes = Object.entries(pageManifest).filter(
+    ([page, compiled]) => !page.startsWith('/api/') && !compiled.endsWith('.html'),
+  )
+
+  const appPathsManifest: Record<string, string> = await readJSON(
+    join(publish, 'server', 'app-paths-manifest.json'),
+  ).catch(() => ({}))
+  const appRoutes = Object.entries(appPathsManifest)
+
+  const routes = [...pageManifestRoutes, ...appRoutes]
+
+  return await Promise.all(
+    routes.map(async ([route, compiled]) => {
+      const functionName = getFunctionNameForPage(route)
+
+      const compiledPath = join(publish, 'server', compiled)
+
+      const routeDependencies = await getDependenciesOfFile(compiledPath)
+      const includedFiles = [compiledPath, ...routeDependencies]
+
+      return {
+        functionName,
+        route,
+        compiled,
+        includedFiles,
+      }
+    }),
+  )
+}
+
 export const getAPILambdas = async (
   publish: string,
   baseDir: string,
   pageExtensions: string[],
 ): Promise<APILambda[]> => {
-  const commonDependencies = await getAPIPRouteCommonDependencies(publish)
+  const commonDependencies = await getCommonDependencies(publish)
 
   const threshold = LAMBDA_WARNING_SIZE - (await getBundleWeight(commonDependencies))
 

packages/runtime/src/index.ts

@@ -18,7 +18,7 @@ import {
 import { onPreDev } from './helpers/dev'
 import { writeEdgeFunctions, loadMiddlewareManifest, cleanupEdgeFunctions } from './helpers/edge'
 import { moveStaticPages, movePublicFiles, removeMetadataFiles } from './helpers/files'
-import { splitApiRoutes } from './helpers/flags'
+import { bundleBasedOnNftFiles, splitApiRoutes } from './helpers/flags'
 import {
   generateFunctions,
   setupImageFunction,
@@ -28,6 +28,7 @@ import {
   packSingleFunction,
   getExtendedApiRouteConfigs,
   APILambda,
+  getSSRLambdas,
 } from './helpers/functions'
 import { generateRedirects, generateStaticRedirects } from './helpers/redirects'
 import { shouldSkip, isNextAuthInstalled, getCustomImageResponseHeaders, getRemotePatterns } from './helpers/utils'
@@ -171,14 +172,17 @@ const plugin: NetlifyPlugin = {
           extendedRoutes.map(packSingleFunction),
         )
 
-    await generateFunctions(constants, appDir, apiLambdas)
+    const ssrLambdas = bundleBasedOnNftFiles(featureFlags) ? await getSSRLambdas(publish) : []
+
+    await generateFunctions(constants, appDir, apiLambdas, ssrLambdas)
     await generatePagesResolver(constants)
 
     await configureHandlerFunctions({
       netlifyConfig,
       ignore,
       publish: relative(process.cwd(), publish),
       apiLambdas,
+      ssrLambdas,
       splitApiRoutes: splitApiRoutes(featureFlags, publish),
     })
 

packages/runtime/src/templates/getHandler.ts

@@ -206,6 +206,8 @@ export const getHandler = ({
     throw new Error('Could not find Next.js server')
   }
 
+  process.env.NODE_ENV = 'production';
+
   const { Server } = require("http");
   const { promises } = require("fs");
   // We copy the file here rather than requiring from the node module