fix(gatsby): persist pages between runs (#28590)

* fix(gatsby): add pages to saved redux state

* chore: update test snapshots

* unskip context change scenario for data tracking tests

* mock Date.now so persistance tests don't rely on actual time (needed for updatedAt and snapshot testing)

* refactor persistance tests a bit to allow for creating different pages per test scenario

* shard pages state (on top of existing sharding for nodes)

* drop page count check (there actually might be valid cases for 0 pages)

* only show page context size warning for pages (and not nodes)

* fix lint

* adjust snapshot

* test(artifacts): add case for changing page context

* fix(gatsby): garbage collect stateful pages (#28760)

* add stateful page to artifacts tests

* fix(gatsby): garbage collect stateful pages

* Revert "fix: clear tracked queries when deleting stale page-data files (#29431)" (#30848)

This reverts commit 478cf68.

Co-authored-by: Reda Bacha <[email protected]>

Author: pieh

integration-tests/artifacts/__tests__/index.js

@@ -230,6 +230,40 @@ function assertHTMLCorrectness(runNumber) {
       })
     })
   })
+
+  describe(`/changing-context/`, () => {
+    let pageDataContent
+    let htmlContent
+    beforeAll(() => {
+      pageDataContent = fs.readJsonSync(
+        path.join(
+          process.cwd(),
+          `public`,
+          `page-data`,
+          `changing-context`,
+          `page-data.json`
+        )
+      )
+
+      htmlContent = fs.readFileSync(
+        path.join(process.cwd(), `public`, `changing-context`, `index.html`),
+        `utf-8`
+      )
+    })
+
+    it(`html is correctly generated using fresh page context`, () => {
+      // remove <!-- --> from html content string as that's impl details of react ssr
+      expect(htmlContent.replace(/<!-- -->/g, ``)).toContain(
+        `Dummy page for runNumber: ${runNumber}`
+      )
+    })
+
+    it(`page-data is correctly generated using fresh page context`, () => {
+      expect(pageDataContent.result.pageContext).toEqual({
+        dummyId: `runNumber: ${runNumber}`,
+      })
+    })
+  })
 }
 
 function assertNodeCorrectness(runNumber) {
@@ -486,6 +520,7 @@ describe(`Second run (different pages created, data changed)`, () => {
     `/static-query-result-tracking/should-invalidate/`,
     `/page-query-template-change/`,
     `/stale-pages/sometimes-i-have-trailing-slash-sometimes-i-dont/`,
+    `/changing-context/`,
   ]
 
   const expectedPagesToRemainFromPreviousBuild = [
@@ -495,6 +530,7 @@ describe(`Second run (different pages created, data changed)`, () => {
     `/static-query-result-tracking/stable/`,
     `/static-query-result-tracking/rerun-query-but-dont-recreate-html/`,
     `/page-that-will-have-trailing-slash-removed`,
+    `/stateful-page-not-recreated-in-third-run/`,
   ]
 
   const expectedPages = [
@@ -579,6 +615,7 @@ describe(`Third run (js change, all pages are recreated)`, () => {
     `/stale-pages/only-in-first/`,
     `/page-query-dynamic-1/`,
     `/page-query-dynamic-2/`,
+    `/stateful-page-not-recreated-in-third-run/`,
   ]
 
   let changedFileOriginalContent
@@ -664,6 +701,7 @@ describe(`Fourth run (gatsby-browser change - cache get invalidated)`, () => {
   const expectedPages = [
     `/stale-pages/only-not-in-first`,
     `/page-query-dynamic-4/`,
+    `/stateful-page-not-recreated-in-third-run/`,
   ]
 
   const unexpectedPages = [

integration-tests/artifacts/gatsby-node.js

@@ -164,6 +164,14 @@ exports.createPages = async ({ actions, graphql }) => {
     }`
   )
 
+  actions.createPage({
+    path: `/changing-context/`,
+    component: require.resolve(`./src/templates/dummy`),
+    context: {
+      dummyId: `runNumber: ${runNumber}`,
+    },
+  })
+
   const { data } = await graphql(`
     {
       allDepPageQuery {
@@ -189,6 +197,18 @@ exports.createPages = async ({ actions, graphql }) => {
   }
 }
 
+exports.createPagesStatefully = async ({ actions }) => {
+  if (runNumber !== 3) {
+    actions.createPage({
+      path: `/stateful-page-not-recreated-in-third-run/`,
+      component: require.resolve(`./src/templates/dummy`),
+      context: {
+        dummyId: `stateful-page`,
+      },
+    })
+  }
+}
+
 exports.onPreBuild = () => {
   console.log(`[test] onPreBuild`)
   changedBrowserCompilationHash = `not-changed`

packages/gatsby/src/bootstrap/index.ts

@@ -6,7 +6,6 @@ import {
   sourceNodes,
   buildSchema,
   createPages,
-  createPagesStatefully,
   extractQueries,
   writeOutRedirects,
   postBootstrap,
@@ -32,9 +31,12 @@ export async function bootstrap(
 
   const parentSpan = tracer.startSpan(`bootstrap`, spanArgs)
 
-  const bootstrapContext: IBuildContext = {
+  const bootstrapContext: IBuildContext & {
+    shouldRunCreatePagesStatefully: boolean
+  } = {
     ...initialContext,
     parentSpan,
+    shouldRunCreatePagesStatefully: true,
   }
 
   const context = {
@@ -54,8 +56,6 @@ export async function bootstrap(
 
   await createPages(context)
 
-  await createPagesStatefully(context)
-
   await handleStalePageData()
 
   await rebuildSchemaWithSitePage(context)

packages/gatsby/src/query/__tests__/data-tracking.js

@@ -879,7 +879,7 @@ describe(`query caching between builds`, () => {
     }, 99999)
   })
 
-  describe.skip(`Changing page context invalidates page queries`, () => {
+  describe(`Changing page context invalidates page queries`, () => {
     beforeAll(() => {
       let pageChangeCounter = 1
       let nodeChangeCounter = 1

packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap

@@ -62,6 +62,23 @@ Object {
     },
   },
   "pageDataStats": Map {},
+  "pages": Map {
+    "/my-sweet-new-page/" => Object {
+      "component": "/Users/username/dev/site/src/templates/my-sweet-new-page.js",
+      "componentChunkName": "component---users-username-dev-site-src-templates-my-sweet-new-page-js",
+      "componentPath": "/Users/username/dev/site/src/templates/my-sweet-new-page.js",
+      "context": Object {
+        "id": "123456",
+      },
+      "internalComponentName": "Component/my-sweet-new-page/",
+      "isCreatedByStatefulCreatePages": false,
+      "matchPath": undefined,
+      "path": "/my-sweet-new-page/",
+      "pluginCreatorId": "",
+      "pluginCreator___NODE": "",
+      "updatedAt": 1,
+    },
+  },
   "pendingPageDataWrites": Object {
     "pagePaths": Set {},
   },