feat: split api routes into separate functions (#1495)

* feat: add static analysis helper

* feat: split api routes into separate functions

* chore: fix syntax of api handlers for better static analysis

* fix: broken test

* chore: change config shape

* chore: fix test

* chore: windows paths 😠

* chore: add e2e tests for extended API routes

* chore: fix cypress 404 test

* chore: fix import

* chore: lint

* chore: remove unused function

* chore: use enum for api route type

* chore: snapidoo

* chore: lockfile

* chore: debug preview test

* chore: use const enum

* feat: add warning logs for advanced api routes

* chore: support jsx

Author: ascorbic

cypress/integration/default/api.spec.ts

@@ -0,0 +1,10 @@
+describe('Extended API routes', () => {
+  it('returns HTTP 202 Accepted for background route', () => {
+    cy.request('/api/hello-background').then((response) => {
+      expect(response.status).to.equal(202)
+    })
+  })
+  it('correctly returns 404 for scheduled route', () => {
+    cy.request({ url: '/api/hello-scheduled', failOnStatusCode: false }).its('status').should('equal', 404)
+  })
+})

cypress/integration/default/preview.spec.ts

@@ -1,8 +1,10 @@
 describe('Preview Mode', () => {
   it('enters and exits preview mode', () => {
+    Cypress.Cookies.debug(true)
+    cy.getCookies().then((cookie) => cy.log('cookies', cookie))
     // preview mode is off by default
     cy.visit('/previewTest')
-    cy.findByText('Is preview? No', {selector: 'h1'})
+    cy.findByText('Is preview? No', { selector: 'h1' })
 
     // enter preview mode
     cy.request('/api/enterPreview').then((response) => {

demos/default/pages/api/hello-background.ts

@@ -0,0 +1,9 @@
+export default (req, res) => {
+  res.setHeader('Content-Type', 'application/json')
+  res.status(200)
+  res.json({ message: 'hello world :)' })
+}
+
+export const config = {
+  type: 'experimental-background',
+}

demos/default/pages/api/hello-scheduled.js

@@ -0,0 +1,10 @@
+export default (req, res) => {
+  res.setHeader('Content-Type', 'application/json')
+  res.status(200)
+  res.json({ message: 'hello world :)' })
+}
+
+export const config = {
+  type: 'experimental-scheduled',
+  schedule: '@hourly',
+}

packages/runtime/src/helpers/analysis.ts

@@ -0,0 +1,111 @@
+import fs, { existsSync } from 'fs'
+
+import { extractExportedConstValue, UnsupportedValueError } from 'next/dist/build/analysis/extract-const-value'
+import { parseModule } from 'next/dist/build/analysis/parse-module'
+import { relative } from 'pathe'
+
+// I have no idea what eslint is up to here but it gives an error
+// eslint-disable-next-line no-shadow
+export const enum ApiRouteType {
+  SCHEDULED = 'experimental-scheduled',
+  BACKGROUND = 'experimental-background',
+}
+
+export interface ApiStandardConfig {
+  type?: never
+  runtime?: 'nodejs' | 'experimental-edge'
+  schedule?: never
+}
+
+export interface ApiScheduledConfig {
+  type: ApiRouteType.SCHEDULED
+  runtime?: 'nodejs'
+  schedule: string
+}
+
+export interface ApiBackgroundConfig {
+  type: ApiRouteType.BACKGROUND
+  runtime?: 'nodejs'
+  schedule?: never
+}
+
+export type ApiConfig = ApiStandardConfig | ApiScheduledConfig | ApiBackgroundConfig
+
+export const validateConfigValue = (config: ApiConfig, apiFilePath: string): config is ApiConfig => {
+  if (config.type === ApiRouteType.SCHEDULED) {
+    if (!config.schedule) {
+      console.error(
+        `Invalid config value in ${relative(process.cwd(), apiFilePath)}: schedule is required when type is "${
+          ApiRouteType.SCHEDULED
+        }"`,
+      )
+      return false
+    }
+    if ((config as ApiConfig).runtime === 'experimental-edge') {
+      console.error(
+        `Invalid config value in ${relative(
+          process.cwd(),
+          apiFilePath,
+        )}: edge runtime is not supported for scheduled functions`,
+      )
+      return false
+    }
+    return true
+  }
+
+  if (!config.type || config.type === ApiRouteType.BACKGROUND) {
+    if (config.schedule) {
+      console.error(
+        `Invalid config value in ${relative(process.cwd(), apiFilePath)}: schedule is not allowed unless type is "${
+          ApiRouteType.SCHEDULED
+        }"`,
+      )
+      return false
+    }
+    if (config.type && (config as ApiConfig).runtime === 'experimental-edge') {
+      console.error(
+        `Invalid config value in ${relative(
+          process.cwd(),
+          apiFilePath,
+        )}: edge runtime is not supported for background functions`,
+      )
+      return false
+    }
+    return true
+  }
+  console.error(
+    `Invalid config value in ${relative(process.cwd(), apiFilePath)}: type ${
+      (config as ApiConfig).type
+    } is not supported`,
+  )
+  return false
+}
+
+/**
+ * Uses Next's swc static analysis to extract the config values from a file.
+ */
+export const extractConfigFromFile = async (apiFilePath: string): Promise<ApiConfig> => {
+  if (!apiFilePath || !existsSync(apiFilePath)) {
+    return {}
+  }
+  const fileContent = await fs.promises.readFile(apiFilePath, 'utf8')
+  // No need to parse if there's no "config"
+  if (!fileContent.includes('config')) {
+    return {}
+  }
+  const ast = await parseModule(apiFilePath, fileContent)
+
+  let config: ApiConfig
+  try {
+    config = extractExportedConstValue(ast, 'config')
+  } catch (error) {
+    if (error instanceof UnsupportedValueError) {
+      console.warn(`Unsupported config value in ${relative(process.cwd(), apiFilePath)}`)
+    }
+    return {}
+  }
+  if (validateConfigValue(config, apiFilePath)) {
+    return config
+  }
+  throw new Error(`Unsupported config value in ${relative(process.cwd(), apiFilePath)}`)
+}

packages/runtime/src/helpers/config.ts

@@ -92,7 +92,7 @@ export const configureHandlerFunctions = async ({ netlifyConfig, publish, ignore
   }
 
   /* eslint-enable no-underscore-dangle */
-  ;[HANDLER_FUNCTION_NAME, ODB_FUNCTION_NAME].forEach((functionName) => {
+  ;[HANDLER_FUNCTION_NAME, ODB_FUNCTION_NAME, '_api_*'].forEach((functionName) => {
     netlifyConfig.functions[functionName] ||= { included_files: [], external_node_modules: [] }
     netlifyConfig.functions[functionName].node_bundler = 'nft'
     netlifyConfig.functions[functionName].included_files ||= []

packages/runtime/src/helpers/edge.ts

@@ -5,7 +5,7 @@ import { resolve, join } from 'path'
 import type { NetlifyConfig, NetlifyPluginConstants } from '@netlify/build'
 import { greenBright } from 'chalk'
 import destr from 'destr'
-import { copy, copyFile, emptyDir, ensureDir, readJSON, readJson, writeJSON, writeJson } from 'fs-extra'
+import { copy, copyFile, emptyDir, ensureDir, readJson, writeJSON, writeJson } from 'fs-extra'
 import type { MiddlewareManifest } from 'next/dist/build/webpack/plugins/middleware-plugin'
 import type { RouteHas } from 'next/dist/lib/load-custom-routes'
 import { outdent } from 'outdent'
@@ -284,9 +284,4 @@ export const writeEdgeFunctions = async (netlifyConfig: NetlifyConfig) => {
   await writeJson(join(edgeFunctionRoot, 'manifest.json'), manifest)
 }
 
-export const enableEdgeInNextConfig = async (publish: string) => {
-  const configFile = join(publish, 'required-server-files.json')
-  const config = await readJSON(configFile)
-  await writeJSON(configFile, config)
-}
 /* eslint-enable max-lines */

packages/runtime/src/helpers/files.ts

@@ -8,7 +8,7 @@ import globby from 'globby'
 import { PrerenderManifest } from 'next/dist/build'
 import { outdent } from 'outdent'
 import pLimit from 'p-limit'
-import { join } from 'pathe'
+import { join, resolve } from 'pathe'
 import slash from 'slash'
 
 import { MINIMUM_REVALIDATE_SECONDS, DIVIDER } from '../constants'
@@ -18,6 +18,7 @@ import { Rewrites, RoutesManifest } from './types'
 import { findModuleFromBase } from './utils'
 
 const TEST_ROUTE = /(|\/)\[[^/]+?](\/|\.html|$)/
+const SOURCE_FILE_EXTENSIONS = ['js', 'jsx', 'ts', 'tsx']
 
 export const isDynamicRoute = (route) => TEST_ROUTE.test(route)
 
@@ -333,6 +334,31 @@ const getServerFile = (root: string, includeBase = true) => {
   return findModuleFromBase({ candidates, paths: [root] })
 }
 
+/**
+ * Find the source file for a given page route
+ */
+export const getSourceFileForPage = (page: string, root: string) => {
+  for (const extension of SOURCE_FILE_EXTENSIONS) {
+    const file = join(root, `${page}.${extension}`)
+    if (existsSync(file)) {
+      return file
+    }
+  }
+  console.log('Could not find source file for page', page)
+}
+
+/**
+ * Reads the node file trace file for a given file, and resolves the dependencies
+ */
+export const getDependenciesOfFile = async (file: string) => {
+  const nft = `${file}.nft.json`
+  if (!existsSync(nft)) {
+    return []
+  }
+  const dependencies = await readJson(nft, 'utf8')
+  return dependencies.files.map((dep) => resolve(file, dep))
+}
+
 const baseServerReplacements: Array<[string, string]> = [
   // force manual revalidate during cache fetches
   [

packages/runtime/src/helpers/functions.ts

@@ -1,30 +1,67 @@
+/* eslint-disable max-lines */
 import type { NetlifyConfig, NetlifyPluginConstants } from '@netlify/build'
 import bridgeFile from '@vercel/node-bridge'
+import chalk from 'chalk'
 import destr from 'destr'
-import { copyFile, ensureDir, writeFile, writeJSON } from 'fs-extra'
+import { copyFile, ensureDir, existsSync, readJSON, writeFile, writeJSON } from 'fs-extra'
 import type { ImageConfigComplete, RemotePattern } from 'next/dist/shared/lib/image-config'
+import { outdent } from 'outdent'
 import { join, relative, resolve } from 'pathe'
 
 import { HANDLER_FUNCTION_NAME, ODB_FUNCTION_NAME, IMAGE_FUNCTION_NAME, DEFAULT_FUNCTIONS_SRC } from '../constants'
+import { getApiHandler } from '../templates/getApiHandler'
 import { getHandler } from '../templates/getHandler'
-import { getPageResolver } from '../templates/getPageResolver'
+import { getPageResolver, getSinglePageResolver } from '../templates/getPageResolver'
+
+import { ApiConfig, ApiRouteType, extractConfigFromFile } from './analysis'
+import { getSourceFileForPage } from './files'
+import { getFunctionNameForPage } from './utils'
+
+export interface ApiRouteConfig {
+  route: string
+  config: ApiConfig
+  compiled: string
+}
 
 export const generateFunctions = async (
   { FUNCTIONS_SRC = DEFAULT_FUNCTIONS_SRC, INTERNAL_FUNCTIONS_SRC, PUBLISH_DIR }: NetlifyPluginConstants,
   appDir: string,
+  apiRoutes: Array<ApiRouteConfig>,
 ): Promise<void> => {
-  const functionsDir = INTERNAL_FUNCTIONS_SRC || FUNCTIONS_SRC
-  const functionDir = join(process.cwd(), functionsDir, HANDLER_FUNCTION_NAME)
-  const publishDir = relative(functionDir, resolve(PUBLISH_DIR))
+  const publish = resolve(PUBLISH_DIR)
+  const functionsDir = resolve(INTERNAL_FUNCTIONS_SRC || FUNCTIONS_SRC)
+  console.log({ functionsDir })
+  const functionDir = join(functionsDir, HANDLER_FUNCTION_NAME)
+  const publishDir = relative(functionDir, publish)
 
-  const writeHandler = async (func: string, isODB: boolean) => {
+  for (const { route, config, compiled } of apiRoutes) {
+    const apiHandlerSource = await getApiHandler({
+      page: route,
+      config,
+    })
+    const functionName = getFunctionNameForPage(route, config.type === ApiRouteType.BACKGROUND)
+    await ensureDir(join(functionsDir, functionName))
+    await writeFile(join(functionsDir, functionName, `${functionName}.js`), apiHandlerSource)
+    await copyFile(bridgeFile, join(functionsDir, functionName, 'bridge.js'))
+    await copyFile(
+      join(__dirname, '..', '..', 'lib', 'templates', 'handlerUtils.js'),
+      join(functionsDir, functionName, 'handlerUtils.js'),
+    )
+    const resolverSource = await getSinglePageResolver({
+      functionsDir,
+      sourceFile: join(publish, 'server', compiled),
+    })
+    await writeFile(join(functionsDir, functionName, 'pages.js'), resolverSource)
+  }
+
+  const writeHandler = async (functionName: string, isODB: boolean) => {
     const handlerSource = await getHandler({ isODB, publishDir, appDir: relative(functionDir, appDir) })
-    await ensureDir(join(functionsDir, func))
-    await writeFile(join(functionsDir, func, `${func}.js`), handlerSource)
-    await copyFile(bridgeFile, join(functionsDir, func, 'bridge.js'))
+    await ensureDir(join(functionsDir, functionName))
+    await writeFile(join(functionsDir, functionName, `${functionName}.js`), handlerSource)
+    await copyFile(bridgeFile, join(functionsDir, functionName, 'bridge.js'))
     await copyFile(
       join(__dirname, '..', '..', 'lib', 'templates', 'handlerUtils.js'),
-      join(functionsDir, func, 'handlerUtils.js'),
+      join(functionsDir, functionName, 'handlerUtils.js'),
     )
   }
 
@@ -124,3 +161,58 @@ export const setupImageFunction = async ({
     })
   }
 }
+
+/**
+ * Look for API routes, and extract the config from the source file.
+ */
+export const getApiRouteConfigs = async (publish: string, baseDir: string): Promise<Array<ApiRouteConfig>> => {
+  const pages = await readJSON(join(publish, 'server', 'pages-manifest.json'))
+  const apiRoutes = Object.keys(pages).filter((page) => page.startsWith('/api/'))
+  const pagesDir = join(baseDir, 'pages')
+  return Promise.all(
+    apiRoutes.map(async (apiRoute) => {
+      const filePath = getSourceFileForPage(apiRoute, pagesDir)
+      return { route: apiRoute, config: await extractConfigFromFile(filePath), compiled: pages[apiRoute] }
+    }),
+  )
+}
+
+interface FunctionsManifest {
+  functions: Array<{ name: string; schedule?: string }>
+}
+
+/**
+ * Warn the user of the caveats if they're using background or scheduled API routes
+ */
+
+export const warnOnApiRoutes = async ({
+  FUNCTIONS_DIST,
+}: Pick<NetlifyPluginConstants, 'FUNCTIONS_DIST'>): Promise<void> => {
+  const functionsManifestPath = join(FUNCTIONS_DIST, 'manifest.json')
+  if (!existsSync(functionsManifestPath)) {
+    return
+  }
+
+  const { functions }: FunctionsManifest = await readJSON(functionsManifestPath)
+
+  if (functions.some((func) => func.name.endsWith('-background'))) {
+    console.warn(
+      outdent`
+        ${chalk.yellowBright`Using background API routes`}
+        If your account type does not support background functions, the deploy will fail.
+        During local development, background API routes will run as regular API routes, but in production they will immediately return an empty "202 Accepted" response.
+      `,
+    )
+  }
+
+  if (functions.some((func) => func.schedule)) {
+    console.warn(
+      outdent`
+        ${chalk.yellowBright`Using scheduled API routes`} 
+        These are run on a schedule when deployed to production.
+        You can test them locally by loading them in your browser but this will not be available when deployed, and any returned value is ignored.
+      `,
+    )
+  }
+}
+/* eslint-enable max-lines */

packages/runtime/src/helpers/redirects.ts

@@ -10,6 +10,7 @@ import { join } from 'pathe'
 import { HANDLER_FUNCTION_PATH, HIDDEN_PATHS, ODB_FUNCTION_PATH } from '../constants'
 
 import { getMiddleware } from './files'
+import { ApiRouteConfig } from './functions'
 import { RoutesManifest } from './types'
 import {
   getApiRewrites,
@@ -228,10 +229,12 @@ export const generateRedirects = async ({
   netlifyConfig,
   nextConfig: { i18n, basePath, trailingSlash, appDir },
   buildId,
+  apiRoutes,
 }: {
   netlifyConfig: NetlifyConfig
   nextConfig: Pick<NextConfig, 'i18n' | 'basePath' | 'trailingSlash' | 'appDir'>
   buildId: string
+  apiRoutes: Array<ApiRouteConfig>
 }) => {
   const { dynamicRoutes: prerenderedDynamicRoutes, routes: prerenderedStaticRoutes }: PrerenderManifest =
     await readJSON(join(netlifyConfig.build.publish, 'prerender-manifest.json'))
@@ -249,7 +252,7 @@ export const generateRedirects = async ({
   // This is only used in prod, so dev uses `next dev` directly
   netlifyConfig.redirects.push(
     // API routes always need to be served from the regular function
-    ...getApiRewrites(basePath),
+    ...getApiRewrites(basePath, apiRoutes),
     // Preview mode gets forced to the function, to bypass pre-rendered pages, but static files need to be skipped
     ...(await getPreviewRewrites({ basePath, appDir })),
   )