feat(gatsby): add node manifest page data digest (#32837)

* process node manifests as part of page-data.json writing so we can add a page-data digest

* move for processNodeManifests to work with gatsby build as well as inc-builds

Author: TylerBarnes

integration-tests/node-manifest/__tests__/create-node-manifest.test.js

@@ -5,6 +5,9 @@ const urling = require(`urling`)
 const rimraf = require(`rimraf`)
 const path = require(`path`)
 const fs = require(`fs-extra`)
+const {
+  getPageDataDigestForPagePath,
+} = require(`gatsby/dist/utils/node-manifest`)
 
 const manifestDir = path.join(
   process.cwd(),
@@ -72,6 +75,17 @@ describe(`Node Manifest API in "gatsby ${gatsbyCommandName}"`, () => {
         )
       ).toBe(true)
     })
+
+    // this doesn't work in gatsby develop since page-data.json files aren't written out
+    it(`Adds a correct page-data.json digest to the manifest`, async () => {
+      const pageDataDigest = await getPageDataDigestForPagePath(
+        `/one`,
+        process.cwd()
+      )
+      const manifest = await getManifestContents(`1`)
+
+      expect(pageDataDigest).toEqual(manifest.pageDataDigest)
+    })
   }
 
   it(`Creates a node manifest with a null page path when createNodeManifest is called but a page is not created for the provided node in the Gatsby site`, async () => {

packages/gatsby/src/commands/build-html.ts

@@ -18,6 +18,7 @@ import { IProgram, Stage } from "./types"
 import { PackageJson } from "../.."
 import type { GatsbyWorkerPool } from "../utils/worker/pool"
 import { IPageDataWithQueryResult } from "../utils/page-data"
+import { processNodeManifests } from "../utils/node-manifest"
 
 type IActivity = any // TODO
 
@@ -459,5 +460,8 @@ export async function buildHTMLPagesAndDeleteStaleArtifacts({
     deletePageDataActivityTimer.end()
   }
 
+  // we process node manifests in this location because we need to make sure all page-data.json files are written for gatsby as well as inc-builds (both call builHTMLPagesAndDeleteStaleArtifacts). Node manifests include a digest of the corresponding page-data.json file and at this point we can be sure page-data has been written out for the latest updates in gatsby build AND inc builds.
+  await processNodeManifests()
+
   return { toRegenerate, toDelete }
 }

packages/gatsby/src/query/index.ts

@@ -11,7 +11,6 @@ import {
 } from "../utils/websocket-manager"
 import { GraphQLRunner } from "./graphql-runner"
 import { IGroupedQueryIds } from "../services"
-import { processNodeManifests } from "../utils/node-manifest"
 
 if (process.env.GATSBY_EXPERIMENTAL_QUERY_CONCURRENCY) {
   console.info(
@@ -238,13 +237,6 @@ export async function processPageQueries(
     graphqlTracing,
   })
 
-  if (process.env.NODE_ENV !== `development`) {
-    /**
-     * only process node manifests here when not in develop. for gatsby develop we process node manifests in src/query/query-watcher.ts everytime queries are re-run. Because we process node manifests in this location for gatsby build we have all the information needed to create the manifests. In query-watcher during gatsby build we might not have all information about created pages and queries.
-     */
-    await processNodeManifests()
-  }
-
   return processedQueries
 }
 

packages/gatsby/src/utils/node-manifest.ts

@@ -6,6 +6,8 @@ import { store } from "../redux/"
 import { internalActions } from "../redux/actions"
 import path from "path"
 import fs from "fs-extra"
+import { readPageData } from "./page-data"
+import { createContentDigest } from "gatsby-core-utils"
 
 interface INodeManifestPage {
   path?: string
@@ -20,6 +22,7 @@ interface INodeManifestOut {
     id: string
   }
   foundPageBy: FoundPageBy
+  pageDataDigest: string | null
 }
 
 type FoundPageBy =
@@ -202,6 +205,41 @@ export function warnAboutNodeManifestMappingProblems({
   }
 }
 
+/**
+ * Retrieves the content digest of a page-data.json file for use in creating node manifest files.
+ */
+export async function getPageDataDigestForPagePath(
+  pagePath?: string,
+  directory?: string
+): Promise<string | null> {
+  if (
+    // if no page was created for the node we're creating a manifest for, there wont be a page path.
+    !pagePath ||
+    // we only add page data digests to node manifests in production because page-data.json may not exist in development.
+    (process.env.NODE_ENV !== `production` && process.env.NODE_ENV !== `test`)
+  ) {
+    return null
+  }
+
+  try {
+    const publicDirectory = path.join(
+      directory || store.getState().program.directory,
+      `public`
+    )
+    const pageData = await readPageData(publicDirectory, pagePath)
+
+    const pageDataDigest = createContentDigest(pageData)
+
+    return pageDataDigest
+  } catch (e) {
+    reporter.warn(
+      `No page-data.json found for ${pagePath} while processing node manifests.`
+    )
+
+    return null
+  }
+}
+
 /**
  * Prepares and then writes out an individual node manifest file to be used for routing to previews. Manifest files are added via the public unstable_createNodeManifest action
  */
@@ -229,10 +267,15 @@ export async function processNodeManifest(
     foundPageBy,
   })
 
+  const pageDataDigest = await getPageDataDigestForPagePath(
+    nodeManifestPage.path
+  )
+
   const finalManifest: INodeManifestOut = {
     node: inputManifest.node,
     page: nodeManifestPage,
     foundPageBy,
+    pageDataDigest,
   }
 
   const gatsbySiteDirectory = store.getState().program.directory

packages/gatsby/src/utils/page-data.ts

@@ -279,6 +279,7 @@ export async function flush(): Promise<void> {
   }
 
   writePageDataActivity.end()
+
   isFlushing = false
 
   return