feat(gatsby-core-utils): improve fetch-remote-file (#34758)

* feat(gatsby-core-utils): improve fetch-remote-file

* feat: use cacheKey in fetch-remote-file

* use async file operations

* Improve tests + low hanging fruit

* revert wordpress

* feat: make sure 304 works

* fix jest config wordpress

* add integration test

* update tests

Author: wardpeet

integration-tests/gatsby-pipeline/__tests__/fetch-remote-file/index.js

@@ -0,0 +1,66 @@
+/**
+ * We want to make sure that fetch-remote-file is working with multi workers.
+ */
+
+const execa = require(`execa`)
+const path = require(`path`)
+const glob = require(`glob`)
+const fs = require(`fs-extra`)
+const md5File = require(`md5-file`)
+const basePath = path.resolve(__dirname, `../../`)
+
+const cleanDirs = () =>
+  Promise.all([
+    fs.emptyDir(`${basePath}/public`),
+    fs.emptyDir(`${basePath}/.cache`),
+  ])
+
+describe(`fetch-remote-file`, () => {
+  beforeAll(async () => {
+    await cleanDirs()
+    await execa(`yarn`, [`build`], {
+      cwd: basePath,
+      // we want to force 1 query per worker
+      env: { NODE_ENV: `production`, GATSBY_PARALLEL_QUERY_CHUNK_SIZE: `1` },
+    })
+  }, 60 * 1000)
+
+  it("should have the correct md5", async () => {
+    expect(
+      await md5File(
+        path.join(
+          __dirname,
+          "../..",
+          "public/images/50c58a791de3c2303e62084d731799eb/photoA.jpg"
+        )
+      )
+    ).toEqual("a9e57a66a10b2d26a1999a4685d7c9ef")
+    expect(
+      await md5File(
+        path.join(
+          __dirname,
+          "../..",
+          "public/images/4910e745c3c453b8795d6ba65c79d99b/photoB.jpg"
+        )
+      )
+    ).toEqual("c305dc5c5db45cc773231a507af5116d")
+    expect(
+      await md5File(
+        path.join(
+          __dirname,
+          "../..",
+          "public/images/fb673e75e9534b3cc2d2e24085386d48/photoC.jpg"
+        )
+      )
+    ).toEqual("4ba953ba27236727d7abe7d5b8916432")
+  })
+
+  /**
+   * this is a bit of a cheeky test but we just want to make sure we're actually running on multiple workers
+   */
+  it("should have conflict between workers", async () => {
+    const files = await fs.readdir(path.join(__dirname, "../../.cache/workers"))
+
+    expect(files.length).toBeGreaterThan(1)
+  })
+})

integration-tests/gatsby-pipeline/gatsby-node.js

@@ -0,0 +1,69 @@
+const { fetchRemoteFile } = require("gatsby-core-utils/fetch-remote-file")
+const { slash } = require("gatsby-core-utils")
+const path = require("path")
+const fs = require("fs-extra")
+
+/** @type{import('gatsby').createSchemaCustomization} */
+exports.createSchemaCustomization = ({ actions, schema, cache, reporter }) => {
+  actions.createTypes(
+    schema.buildObjectType({
+      name: "MyRemoteFile",
+      fields: {
+        url: "String!",
+        publicUrl: {
+          type: "String!",
+          async resolve(source) {
+            const filePath = await fetchRemoteFile({
+              name: path.basename(source.name, path.extname(source.name)),
+              ext: path.extname(source.name),
+              url: source.url,
+              directory: "./public/images",
+            })
+
+            const dir = path.join(global.__GATSBY.root, ".cache", "workers")
+            await fs.ensureDir(dir)
+            await fs.createFile(
+              `${path.join(dir, `worker-${process.env.GATSBY_WORKER_ID}`)}`
+            )
+
+            const workers = (await cache.get("workers")) ?? []
+            workers.push(process.env.GATSBY_WORKER_ID)
+
+            return `${slash(filePath.replace(/^public/, ""))}`
+          },
+        },
+      },
+      interfaces: ["Node"],
+    })
+  )
+}
+
+/** @type {imporg('gatsby').sourceNodes} */
+exports.sourceNodes = ({ actions, createNodeId, createContentDigest }) => {
+  const items = [
+    {
+      name: "photoA.jpg",
+      url: "https://images.unsplash.com/photo-1517849845537-4d257902454a?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
+    },
+    {
+      name: "photoB.jpg",
+      url: "https://images.unsplash.com/photo-1552053831-71594a27632d?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
+    },
+    {
+      name: "photoC.jpg",
+      url: "https://images.unsplash.com/photo-1561037404-61cd46aa615b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
+    },
+  ]
+
+  items.forEach((item, index) => {
+    actions.createNode({
+      id: createNodeId(`remote-file-${index}`),
+      name: item.name,
+      url: item.url,
+      internal: {
+        type: "MyRemoteFile",
+        contentDigest: createContentDigest(item.url),
+      },
+    })
+  })
+}

integration-tests/gatsby-pipeline/src/pages/fetch-remote-a.js

@@ -0,0 +1,31 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteA = ({ data }) => {
+  return (
+    <Layout>
+      <SEO title="Fetch Remote A" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteA
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

integration-tests/gatsby-pipeline/src/pages/fetch-remote-b.js

@@ -0,0 +1,32 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    <Layout>
+      <SEO title="Fetch Remote B" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

integration-tests/gatsby-pipeline/src/pages/fetch-remote-c.js

@@ -0,0 +1,31 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    <Layout>
+      <SEO title="Fetch Remote C" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

integration-tests/gatsby-pipeline/src/pages/fetch-remote-d.js

@@ -0,0 +1,31 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    <Layout>
+      <SEO title="Fetch Remote B" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

integration-tests/gatsby-source-wordpress/jest.config.js

@@ -1,4 +1,7 @@
 module.exports = {
   testPathIgnorePatterns: [`/node_modules/`, `__tests__/fixtures`, `.cache`],
   bail: true,
+  moduleNameMapper: {
+    "^gatsby-core-utils/(.*)$": `gatsby-core-utils/dist/$1`, // Workaround for https://github.com/facebook/jest/issues/9771
+  },
 }