feat: add blob storage

.eslintrc.js

@@ -93,4 +93,5 @@ module.exports = {
       },
     },
   ],
+  ignorePatterns: ['packages/runtime/src/blob.js'],
 }

packages/runtime/.gitignore

@@ -0,0 +1 @@
+src/blob.js

packages/runtime/package.json

@@ -11,7 +11,8 @@
     "manifest.yml"
   ],
   "dependencies": {
-    "@netlify/esbuild": "0.14.39",
+    "@netlify/blobs": "^1.6.1",
+    "@netlify/esbuild": "0.14.39-1",
     "@netlify/functions": "^1.6.0",
     "@netlify/ipx": "^1.4.4",
     "@vercel/node-bridge": "^2.1.0",
@@ -23,6 +24,7 @@
     "fs-extra": "^10.0.0",
     "globby": "^11.0.4",
     "merge-stream": "^2.0.0",
+    "mime-types": "^2.1.35",
     "moize": "^6.1.0",
     "node-fetch": "^2.6.6",
     "node-stream-zip": "^1.15.0",
@@ -52,6 +54,7 @@
     "publish:install": "npm ci",
     "publish:test": "cd .. && npm ci && npm test",
     "clean": "rimraf lib dist-types",
+    "prebuild": "npx esbuild src/netliblob.mts --bundle --format=cjs --platform=node --outfile=src/blob.js",
     "build": "tsc",
     "watch": "tsc --watch",
     "prepare": "npm run build"

packages/runtime/src/helpers/config.ts

@@ -1,6 +1,6 @@
 import type { NetlifyConfig } from '@netlify/build/types'
 import destr from 'destr'
-import { readJSON, writeJSON } from 'fs-extra'
+import { existsSync, readJSON, writeJSON } from 'fs-extra'
 import type { Header } from 'next/dist/lib/load-custom-routes'
 import type { NextConfigComplete } from 'next/dist/server/config-shared'
 import { join, dirname, relative } from 'pathe'
@@ -34,18 +34,52 @@ const defaultFailBuild = (message: string, { error }): never => {
   throw new Error(`${message}\n${error && error.stack}`)
 }
 
+const addIncrementalCacheHandlerPath = (
+  incrementalCacheHandlerPath: string,
+  experimentalConfig: NextConfig['experimental'],
+) => {
+  // This check is needed for now because if blob storage isn't available, this file will not have been created
+  if (incrementalCacheHandlerPath !== undefined && existsSync(incrementalCacheHandlerPath)) {
+    experimentalConfig.incrementalCacheHandlerPath = incrementalCacheHandlerPath
+  }
+
+  return experimentalConfig
+}
+
 export const getNextConfig = async function getNextConfig({
   publish,
   failBuild = defaultFailBuild,
+  incrementalCacheHandlerPath,
+}: {
+  publish: string
+  failBuild?: (message: string, { error }) => never
+  incrementalCacheHandlerPath?: string
 }): Promise<NextConfig> {
   try {
-    const { config, appDir, ignore }: RequiredServerFiles = await readJSON(join(publish, 'required-server-files.json'))
+    const requiredServerFiles: RequiredServerFiles = await readJSON(join(publish, 'required-server-files.json'))
+    const { config, appDir, ignore } = requiredServerFiles
+
     if (!config) {
       // eslint-disable-next-line @typescript-eslint/ban-ts-comment
       // @ts-ignore
       return failBuild('Error loading your Next config')
     }
 
+    if ('incrementalCacheHandlerPath' in config.experimental) {
+      console.warn(
+        "Your Next.js configuration has the experimental incrementalCacheHandlerPath option set. It will be overridden by Netlify's internal cache.",
+      )
+    }
+
+    // For more info, see https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath
+    // ./cache-handler.js will be copied to the root or the .next build folder
+    await writeJSON(join(publish, 'required-server-files.json'), {
+      ...requiredServerFiles,
+      config: {
+        ...config,
+        experimental: addIncrementalCacheHandlerPath(incrementalCacheHandlerPath, config.experimental),
+      },
+    })
     const routesManifest: RoutesManifest = await readJSON(join(publish, ROUTES_MANIFEST_FILE))
 
     // If you need access to other manifest files, you can add them here as well
@@ -115,7 +149,7 @@ export const configureHandlerFunctions = async ({
   ignore: Array<string>
   apiLambdas: APILambda[]
   ssrLambdas: SSRLambda[]
-  splitApiRoutes: boolean
+  splitApiRoutes: unknown
 }) => {
   const config = await getRequiredServerFiles(publish)
   const files = config.files || []

packages/runtime/src/helpers/flags.ts

@@ -17,7 +17,7 @@ import { join } from 'pathe'
  *
  * Disabled by default. Can be overriden using the NEXT_SPLIT_API_ROUTES env var.
  */
-export const splitApiRoutes = (featureFlags: Record<string, unknown>, publish: string): boolean => {
+export const splitApiRoutes = (featureFlags: Record<string, unknown>, publish: string) => {
   const isEnabled = destr(process.env.NEXT_SPLIT_API_ROUTES) ?? featureFlags.next_split_api_routes ?? false
 
   if (isEnabled && !existsSync(join(publish, 'next-server.js.nft.json'))) {
@@ -30,7 +30,7 @@ export const splitApiRoutes = (featureFlags: Record<string, unknown>, publish: s
   return isEnabled
 }
 
-export const bundleBasedOnNftFiles = (featureFlags: Record<string, unknown>): boolean => {
+export const bundleBasedOnNftFiles = (featureFlags: Record<string, unknown>) => {
   const isEnabled =
     destr(process.env.NEXT_BUNDLE_BASED_ON_NFT_FILES) ?? featureFlags.next_bundle_based_on_nft_files ?? false
 

packages/runtime/src/helpers/functions.ts

@@ -1,11 +1,17 @@
+import { cpus } from 'os'
+
+import { Blobs } from '@netlify/blobs/dist/src/main'
 import type { NetlifyConfig, NetlifyPluginConstants } from '@netlify/build/types'
+import { build } from '@netlify/esbuild'
 import bridgeFile from '@vercel/node-bridge'
 import chalk from 'chalk'
 import destr from 'destr'
-import { copyFile, ensureDir, existsSync, readJSON, writeFile, writeJSON, stat } from 'fs-extra'
-import { PrerenderManifest } from 'next/dist/build'
+import { copyFile, ensureDir, existsSync, readJSON, writeFile, writeJSON, stat, readFile } from 'fs-extra'
+import mime from 'mime-types'
+import type { PrerenderManifest } from 'next/dist/build'
 import type { ImageConfigComplete, RemotePattern } from 'next/dist/shared/lib/image-config'
 import { outdent } from 'outdent'
+import pLimit from 'p-limit'
 import { join, relative, resolve, dirname, basename, extname } from 'pathe'
 import glob from 'tiny-glob'
 
@@ -21,12 +27,13 @@ import {
   API_FUNCTION_NAME,
   LAMBDA_WARNING_SIZE,
 } from '../constants'
+import { BlobISRPage, getHashedKey } from '../templates/blobStorage'
 import { getApiHandler } from '../templates/getApiHandler'
 import { getHandler } from '../templates/getHandler'
 import { getResolverForPages, getResolverForSourceFiles } from '../templates/getPageResolver'
 
 import { ApiConfig, extractConfigFromFile, isEdgeConfig } from './analysis'
-import { getRequiredServerFiles } from './config'
+import { getRequiredServerFiles, NextConfig } from './config'
 import { getDependenciesOfFile, getServerFile, getSourceFileForPage } from './files'
 import { writeFunctionConfiguration } from './functionsMetaData'
 import { pack } from './pack'
@@ -157,6 +164,16 @@ export const generateFunctions = async (
       join(__dirname, '..', '..', 'lib', 'templates', 'handlerUtils.js'),
       join(functionsDir, functionName, 'handlerUtils.js'),
     )
+    // need to copy the blob storage helper over to be available on request time
+    // the odb needs access to the blob storage
+    // we have to bundle it to not miss any files on the odb then
+    await build({
+      entryPoints: [join(__dirname, '..', '..', 'lib', 'templates', 'blobStorage.js')],
+      outfile: join(functionsDir, functionName, 'blobStorage.js'),
+      bundle: true,
+      platform: 'node',
+    })
+
     await writeFunctionConfiguration({ functionName, functionTitle, functionsDir })
 
     const nfInternalFiles = await glob(join(functionsDir, functionName, '**'))
@@ -364,42 +381,161 @@ const changeExtension = (file: string, extension: string) => {
   return join(dirname(file), base + extension)
 }
 
-const getSSRDependencies = async (publish: string): Promise<string[]> => {
+const getPrerenderManifest = async (publish: string) => {
   const prerenderManifest: PrerenderManifest = await readJSON(join(publish, 'prerender-manifest.json'))
 
-  return [
-    ...Object.entries(prerenderManifest.routes).flatMap(([route, ssgRoute]) => {
-      if (ssgRoute.initialRevalidateSeconds === false) {
-        return []
-      }
+  return prerenderManifest
+}
 
-      if (ssgRoute.dataRoute.endsWith('.rsc')) {
-        return [
-          join(publish, 'server', 'app', ssgRoute.dataRoute),
-          join(publish, 'server', 'app', changeExtension(ssgRoute.dataRoute, '.html')),
-        ]
+/**
+ * Warms up the cache with prerendered content
+ *
+ * @param options
+ * @param options.netliBlob - the blob storage instance
+ * @param options.prerenderManifest - the prerender manifest
+ * @param options.publish - the publish directory
+ *
+ */
+const setPrerenderedBlobStoreContent = async ({
+  netliBlob,
+  prerenderManifest,
+  publish,
+  i18n,
+}: {
+  i18n: NextConfig['i18n']
+  netliBlob: Blobs
+  prerenderManifest: PrerenderManifest
+  publish: string
+}): Promise<void> => {
+  // *.rsc, *.json and *.html files can be found in the .next build artifacts folder,
+  //
+  // e.g. app router build artifacts from the default demo site
+  // demos/default/.next/server/app/blog/rob/second-post.html
+  // demos/default/.next/server/app/blog/rob/second-post.rsc
+  //
+  // e.g. pages router build artifacts from the default demo site
+  // demos/default/.next/server/pages/en/getStaticProps/1.html
+  // demos/default/.next/server/pages/en/getStaticProps/1.json
+  const limit = pLimit(Math.max(2, cpus().length))
+
+  const blobCalls = Object.entries(prerenderManifest.routes).map(([route, ssgRoute]) =>
+    limit(async () => {
+      const routerTypeSubPath = ssgRoute.dataRoute.endsWith('.rsc') ? 'app' : 'pages'
+      const dataFilePath = join(publish, 'server', routerTypeSubPath, ssgRoute.dataRoute)
+
+      try {
+        // Page data for an app router page is an RSC serialized format, i.e. a string,
+        // or a JSON file for the pages router.
+        const pageData =
+          routerTypeSubPath === 'app'
+            ? await readFile(dataFilePath, 'utf8')
+            : JSON.parse(await readFile(join(publish, 'server', routerTypeSubPath, `${route}.json`), 'utf8'))
+
+        const htmlFilePath = join(publish, 'server', routerTypeSubPath, `${route}.html`)
+        const html = await readFile(htmlFilePath, 'utf8')
+
+        // TODO: once implemented in blob storage API
+        // We need to remove the leading slash from the route so that the call to the blob storage
+        // does not generate a 405 error.
+        // It's currently under consideration to support this in the blob storage API.
+        const pageRoute = route.replace(new RegExp(`^${i18n.defaultLocale}/`), '')
+        const pageBlob: BlobISRPage = {
+          value: html,
+          headers: {
+            'content-type': 'text/html',
+          },
+          lastModified: Date.now(),
+        }
+        let { dataRoute } = ssgRoute
+        const dataBlob: BlobISRPage = {
+          value: pageData,
+          headers: {
+            'content-type': mime.lookup(dataFilePath),
+          },
+          lastModified: Date.now(),
+        }
+
+        // for the index route we have to replace it with the language as this is the url that will be requested
+        if (pageRoute === i18n.defaultLocale) {
+          dataRoute = dataRoute.replace(/index\.json$/, `${i18n.defaultLocale}.json`)
+        }
+
+        console.log('[SET KEY]:', pageRoute, getHashedKey(pageRoute))
+        console.log('[SET KEY]:', dataRoute, getHashedKey(pageRoute), { ssgRoute })
+        return Promise.all([
+          netliBlob.setJSON(getHashedKey(pageRoute), pageBlob),
+          netliBlob.setJSON(getHashedKey(dataRoute), dataBlob),
+        ])
+      } catch {
+        // noop
+        // gracefully fall back to not having it in the blob storage and the ISR ODB handler needs to let the
+        // request fall through to the next server to generate the page nothing we can serve then.
       }
+    }),
+  )
 
-      const trimmedPath = route === '/' ? 'index' : route.slice(1)
+  await Promise.all(blobCalls)
+}
+
+const getPrerenderedContent = (prerenderManifest: PrerenderManifest, publish: string): string[] => [
+  ...Object.entries(prerenderManifest.routes).flatMap(([route, ssgRoute]) => {
+    if (ssgRoute.initialRevalidateSeconds === false) {
+      return []
+    }
+
+    if (ssgRoute.dataRoute.endsWith('.rsc')) {
       return [
-        join(publish, 'server', 'pages', `${trimmedPath}.html`),
-        join(publish, 'server', 'pages', `${trimmedPath}.json`),
+        join(publish, 'server', 'app', ssgRoute.dataRoute),
+        join(publish, 'server', 'app', changeExtension(ssgRoute.dataRoute, '.html')),
       ]
-    }),
-    join(publish, '**', '*.html'),
-    join(publish, 'static-manifest.json'),
-  ]
-}
+    }
 
-export const getSSRLambdas = async (publish: string): Promise<SSRLambda[]> => {
+    const trimmedPath = route === '/' ? 'index' : route.slice(1)
+    return [
+      join(publish, 'server', 'pages', `${trimmedPath}.html`),
+      join(publish, 'server', 'pages', `${trimmedPath}.json`),
+    ]
+  }),
+  join(publish, '**', '*.html'),
+  join(publish, 'static-manifest.json'),
+]
+
+// TODO: get a build feature flag set up for blob storage
+export const getSSRLambdas = async ({
+  publish,
+  i18n,
+  netliBlob,
+}: {
+  i18n: NextConfig['i18n']
+  publish: string
+  netliBlob?: Blobs
+}): Promise<SSRLambda[]> => {
   const commonDependencies = await getCommonDependencies(publish)
   const ssrRoutes = await getSSRRoutes(publish)
 
   // TODO: for now, they're the same - but we should separate them
   const nonOdbRoutes = ssrRoutes
   const odbRoutes = ssrRoutes
 
-  const ssrDependencies = await getSSRDependencies(publish)
+  const prerenderManifest = await getPrerenderManifest(publish)
+  let ssrDependencies: Awaited<ReturnType<typeof getPrerenderedContent>>
+
+  if (netliBlob) {
+    console.log('using blob storage')
+    ssrDependencies = []
+
+    try {
+      console.log('warming up the cache with prerendered content')
+      await setPrerenderedBlobStoreContent({ netliBlob, prerenderManifest, publish, i18n })
+    } catch (error) {
+      console.error('Unable to store prerendered content in blob storage', error)
+
+      throw error
+    }
+  } else {
+    // We only want prerendered content stored in the lambda if we aren't using blob storage
+    ssrDependencies = getPrerenderedContent(prerenderManifest, publish)
+  }
 
   return [
     {