perf(gatsby,gatsby-plugin-gatsby-cloud): prioritize content/asset fetching early, drop script preloads so js doesn't compete with assets (#35408)

Co-authored-by: Michal Piechowiak <[email protected]>

Author: marvinjude

e2e-tests/path-prefix/cypress/integration/asset-prefix.js

@@ -11,10 +11,6 @@ describe(`assetPrefix`, () => {
   })
 
   describe(`runtime`, () => {
-    it(`prefixes preloads`, () => {
-      assetPrefixMatcher(cy.get(`head link[rel="preload"]`))
-    })
-
     it(`prefixes styles`, () => {
       assetPrefixMatcher(cy.get(`head style[data-href]`), `data-href`)
     })

e2e-tests/production-runtime/cypress/integration/preload.js

@@ -0,0 +1,6 @@
+describe(`Preloads`, () => {
+  it(`should not have preloads in head`, () => {
+    cy.visit(`/`).waitForRouteChange()
+    cy.get(`head link[rel="preload"]`).should("not.exist")
+  })
+})

e2e-tests/production-runtime/cypress/integration/scripts.js

@@ -0,0 +1,8 @@
+describe(`Preloads`, () => {
+  it(`should not have page scripts in HTML`, () => {
+    cy.visit(`/`).waitForRouteChange()
+    cy.get(`body script`).each(script => {
+      cy.wrap(script).should(`not.have.attr`, `src`, /component---/)
+    })
+  })
+})

integration-tests/artifacts/__tests__/index.js

@@ -533,7 +533,7 @@ describe(`First run (baseline)`, () => {
       )
     })
 
-    describe(`should add <link> for webpack's magic comments`, () => {
+    describe(`should add <link> for webpack's magic comments inside "app" bundle`, () => {
       let htmlContent
       beforeAll(() => {
         htmlContent = fs.readFileSync(
@@ -547,14 +547,26 @@ describe(`First run (baseline)`, () => {
         )
       })
 
-      it(`has prefetch link`, () => {
+      it(`has prefetch link (imported in "app")`, () => {
         expect(htmlContent).toMatch(
-          /<link\s+as="script"\s+rel="prefetch"\s+href="\/magic-comment-prefetch-\w+.js"\s*\/>/g
+          /<link\s+as="script"\s+rel="prefetch"\s+href="\/magic-comment-app-prefetch-\w+.js"\s*\/>/g
         )
       })
 
-      it(`has preload link`, () => {
+      it(`has preload link (imported in "app")`, () => {
         expect(htmlContent).toMatch(
+          /<link\s+as="script"\s+rel="preload"\s+href="\/magic-comment-app-preload-\w+.js"\s*\/>/g
+        )
+      })
+
+      it(`doesn't have prefetch link (imported in template)`, () => {
+        expect(htmlContent).not.toMatch(
+          /<link\s+as="script"\s+rel="prefetch"\s+href="\/magic-comment-prefetch-\w+.js"\s*\/>/g
+        )
+      })
+
+      it(`doesn't have preload link (imported in template)`, () => {
+        expect(htmlContent).not.toMatch(
           /<link\s+as="script"\s+rel="preload"\s+href="\/magic-comment-preload-\w+.js"\s*\/>/g
         )
       })

integration-tests/artifacts/gatsby-browser.js

@@ -6,6 +6,18 @@ const Github = require(`./src/components/github`).default
 // TODO: Uncomment imported.css to test issue https://github.com/gatsbyjs/gatsby/issues/33450
 // require("./imported.css")
 
+import(
+  /* webpackChunkName: "magic-comment-app-prefetch", webpackPrefetch: true */ `./src/components/magic-comments/app-prefetch`
+).then(moduleForPrefetch => {
+  console.log({ forPrefetch: moduleForPrefetch.forPrefetch() })
+})
+
+import(
+  /* webpackChunkName: "magic-comment-app-preload", webpackPreload: true */ `./src/components/magic-comments/app-preload`
+).then(moduleForPreload => {
+  console.log({ forPreload: moduleForPreload.forPreload() })
+})
+
 exports.wrapRootElement = ({ element }) => {
   return (
     <>

integration-tests/artifacts/src/components/magic-comments/app-prefetch.js

@@ -0,0 +1,3 @@
+export function forAppPrefetch() {
+  return `export-for-app-prefetch`
+}

integration-tests/artifacts/src/components/magic-comments/app-preload.js

@@ -0,0 +1,3 @@
+export function forAppPreload() {
+  return `export-for-app-preload`
+}

packages/gatsby-plugin-gatsby-cloud/src/__tests__/__snapshots__/build-headers-program.js.snap

@@ -218,10 +218,8 @@ describe(`build-headers-program`, () => {
       `utf8`
     )
     expect(output).toMatchSnapshot()
-    expect(output).toMatch(/app-data\.json/)
-    expect(output).toMatch(/page-data\.json/)
-    // we should only check app-data once which leads to 1 time
-    expect(fs.existsSync).toBeCalledTimes(1)
+    expect(output).not.toMatch(/app-data\.json/)
+    expect(output).not.toMatch(/page-data\.json/)
   })
 
   it(`with manifest['pages-manifest']`, async () => {
@@ -254,7 +252,7 @@ describe(`build-headers-program`, () => {
     expect(output).toMatchSnapshot()
     expect(output).toMatch(/\/pages-manifest-ab11f09e0ca7ecd3b43e\.js/g)
     expect(output).not.toMatch(/\/app-data\.json/g)
-    expect(output).toMatch(/\/page-data\.json/g)
+    expect(output).not.toMatch(/\/page-data\.json/g)
     expect(output).not.toMatch(/\/undefined/g)
   })
 

packages/gatsby-plugin-gatsby-cloud/src/__tests__/build-headers-program.js

@@ -1,17 +1,12 @@
 import _ from "lodash"
 import { createWriteStream, existsSync } from "fs-extra"
-import { parse, posix } from "path"
-import kebabHash from "kebab-hash"
-import { fixedPagePath } from "gatsby-core-utils"
 import { IMMUTABLE_CACHING_HEADER } from "./constants"
 
 import {
-  COMMON_BUNDLES,
   SECURITY_HEADERS,
   CACHING_HEADERS,
   LINK_REGEX,
   HEADERS_FILENAME,
-  PAGE_DATA_DIR,
 } from "./constants"
 import { emitHeaders } from "./ipc"
 
@@ -20,131 +15,10 @@ function getHeaderName(header) {
   return matches && matches[1]
 }
 
-function validHeaders(headers, reporter) {
-  if (!headers || !_.isObject(headers)) {
-    return false
-  }
-
-  return _.every(
-    headers,
-    (headersList, path) =>
-      _.isArray(headersList) &&
-      _.every(headersList, header => {
-        if (_.isString(header)) {
-          if (!getHeaderName(header)) {
-            // TODO panic on builds on v3
-            reporter.warn(
-              `[gatsby-plugin-gatsby-cloud] ${path} contains an invalid header (${header}). Please check your plugin configuration`
-            )
-          }
-
-          return true
-        }
-
-        return false
-      })
-  )
-}
-
-function linkTemplate(assetPath, type = `script`) {
-  return `Link: <${assetPath}>; rel=preload; as=${type}${
-    type === `fetch` ? `; crossorigin` : ``
-  }; nopush`
-}
-
-function pathChunkName(path) {
-  const name = path === `/` ? `index` : kebabHash(path)
-  return `path---${name}`
-}
-
-function getPageDataPath(path) {
-  return posix.join(`page-data`, fixedPagePath(path), `page-data.json`)
-}
-
-function getScriptPath(file, manifest) {
-  const chunk = manifest[file]
-
-  if (!chunk) {
-    return []
-  }
-
-  // convert to array if it's not already
-  const chunks = _.isArray(chunk) ? chunk : [chunk]
-
-  return chunks.filter(script => {
-    const parsed = parse(script)
-    // handle only .js, .css content is inlined already
-    // and doesn't need to be pushed
-    return parsed.ext === `.js`
-  })
-}
-
-function linkHeaders(files, pathPrefix, assetPrefix) {
-  const linkHeaders = []
-  for (const resourceType in files) {
-    files[resourceType].forEach(file => {
-      linkHeaders.push(
-        linkTemplate(
-          `${assetPrefix ? assetPrefix + `/` : ``}${pathPrefix}/${file}`,
-          resourceType
-        )
-      )
-    })
-  }
-
-  return linkHeaders
-}
-
 function headersPath(pathPrefix, path) {
   return `${pathPrefix}${path}`
 }
 
-function preloadHeadersByPage({
-  pages,
-  manifest,
-  pathPrefix,
-  publicFolder,
-  assetPrefix,
-}) {
-  const linksByPage = {}
-
-  const appDataPath = publicFolder(PAGE_DATA_DIR, `app-data.json`)
-  const hasAppData = existsSync(appDataPath)
-
-  pages.forEach(page => {
-    const scripts = _.flatMap(COMMON_BUNDLES, file =>
-      getScriptPath(file, manifest)
-    )
-    scripts.push(...getScriptPath(pathChunkName(page.path), manifest))
-    scripts.push(...getScriptPath(page.componentChunkName, manifest))
-
-    const json = []
-    if (hasAppData) {
-      json.push(posix.join(PAGE_DATA_DIR, `app-data.json`))
-    }
-
-    // page-data gets inline for SSR, so we won't be doing page-data request
-    // and we shouldn't add preload link header for it.
-    if (page.mode !== `SSR`) {
-      json.push(getPageDataPath(page.path))
-    }
-
-    const filesByResourceType = {
-      script: scripts.filter(Boolean),
-      fetch: json,
-    }
-
-    const pathKey = headersPath(pathPrefix, page.path)
-    linksByPage[pathKey] = linkHeaders(
-      filesByResourceType,
-      pathPrefix,
-      assetPrefix
-    )
-  })
-
-  return linksByPage
-}
-
 function defaultMerge(...headers) {
   function unionMerge(objValue, srcValue) {
     if (_.isArray(objValue)) {
@@ -200,21 +74,6 @@ function transformLink(manifest, publicFolder, pathPrefix) {
     })
 }
 
-function stringifyHeaders(headers) {
-  return _.reduce(
-    headers,
-    (text, headerList, path) => {
-      const headersString = _.reduce(
-        headerList,
-        (accum, header) => `${accum}  ${header}\n`,
-        ``
-      )
-      return `${text}${path}\n${headersString}`
-    },
-    ``
-  )
-}
-
 // program methods
 
 const mapUserLinkHeaders =
@@ -247,26 +106,6 @@ const mapUserLinkAllPageHeaders =
     return defaultMerge(headers, duplicateHeadersByPage)
   }
 
-const applyLinkHeaders =
-  (pluginData, { mergeLinkHeaders }) =>
-  headers => {
-    if (!mergeLinkHeaders) {
-      return headers
-    }
-
-    const { pages, manifest, pathPrefix, publicFolder, assetPrefix } =
-      pluginData
-    const perPageHeaders = preloadHeadersByPage({
-      pages,
-      manifest,
-      pathPrefix,
-      publicFolder,
-      assetPrefix,
-    })
-
-    return defaultMerge(headers, perPageHeaders)
-  }
-
 const applySecurityHeaders =
   ({ mergeSecurityHeaders }) =>
   headers => {
@@ -371,7 +210,6 @@ export default function buildHeadersProgram(pluginData, pluginOptions) {
     applySecurityHeaders(pluginOptions),
     applyCachingHeaders(pluginData, pluginOptions),
     mapUserLinkAllPageHeaders(pluginData, pluginOptions),
-    applyLinkHeaders(pluginData, pluginOptions),
     applyTransfromHeaders(pluginOptions),
     saveHeaders(pluginData)
   )(pluginOptions.headers)

packages/gatsby-plugin-gatsby-cloud/src/build-headers-program.js

@@ -14,21 +14,21 @@ exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComp
 
 exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `
 Object {
-  "html": "<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/app-data.json\\" crossorigin=\\"anonymous\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/about/page-data.json\\" crossorigin=\\"anonymous\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";window.___webpackCompilationHash=\\"1234567890abcdef1234\\";/*]]>*/</script><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script></body></html>",
+  "html": "<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";window.___webpackCompilationHash=\\"1234567890abcdef1234\\";/*]]>*/</script><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script></body></html>",
   "unsafeBuiltinsUsage": Array [],
 }
 `;
 
 exports[`static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `
 Object {
-  "html": "<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/about/page-data.json\\" crossorigin=\\"anonymous\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/app-data.json\\" crossorigin=\\"anonymous\\"/></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";window.___webpackCompilationHash=\\"1234567890abcdef1234\\";/*]]>*/</script><div> div3 </div><div> div2 </div><div> div1 </div></body></html>",
+  "html": "<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";window.___webpackCompilationHash=\\"1234567890abcdef1234\\";/*]]>*/</script><div> div3 </div><div> div2 </div><div> div1 </div></body></html>",
   "unsafeBuiltinsUsage": Array [],
 }
 `;
 
 exports[`static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `
 Object {
-  "html": "<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/about/page-data.json\\" crossorigin=\\"anonymous\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/app-data.json\\" crossorigin=\\"anonymous\\"/></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";window.___webpackCompilationHash=\\"1234567890abcdef1234\\";/*]]>*/</script><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script></body></html>",
+  "html": "<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";window.___webpackCompilationHash=\\"1234567890abcdef1234\\";/*]]>*/</script><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script></body></html>",
   "unsafeBuiltinsUsage": Array [],
 }
 `;