fix(gatsby-source-contentful): fixed contentful asset download stalling with high number of assets (#27563)

* fix(gatsby-source-contentful): fixed contentful asset download stalling with high number of assets

* Add new option to README

* Fix test

* Switch to pop and fix a typo made when cleaning up code

Author: me4502

packages/gatsby-source-contentful/README.md

@@ -177,6 +177,10 @@ If you are confident your Content Types will have natural-language IDs (e.g. `bl
 
 Number of entries to retrieve from Contentful at a time. Due to some technical limitations, the response payload should not be greater than 7MB when pulling content from Contentful. If you encounter this issue you can set this param to a lower number than 100, e.g `50`.
 
+**`assetDownloadWorkers`** [number][optional] [default: `50`]
+
+Number of workers to use when downloading contentful assets. Due to technical limitations, opening too many concurrent requests can cause stalled downloads. If you encounter this issue you can set this param to a lower number than 50, e.g 25.
+
 **`richText.resolveFieldLocales`** [boolean][optional] [default: `false`]
 
 If you want to resolve the locales in fields of assets and entries that are referenced by rich text (e.g., via embedded entries or entry hyperlinks), set this to `true`. Otherwise, fields of referenced assets or entries will be objects keyed by locale.

packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js

@@ -65,6 +65,7 @@ describe.only(`downloadContentfulAssets`, () => {
       actions: { touchNode: jest.fn() },
       getNodesByType: () => fixtures,
       cache,
+      assetDownloadWorkers: 50,
     })
 
     fixtures.forEach(n => {

packages/gatsby-source-contentful/src/download-contentful-assets.js

@@ -9,7 +9,23 @@ const bar = new ProgressBar(
   }
 )
 
-let totalJobs = 0
+/**
+ * @name distributeWorkload
+ * @param workers A list of async functions to complete
+ * @param {number} count The number of task runners to use
+ */
+
+async function distributeWorkload(workers, count) {
+  const methods = workers.slice()
+
+  async function task() {
+    while (methods.length > 0) {
+      await methods.pop()()
+    }
+  }
+
+  await Promise.all(new Array(count).fill(undefined).map(() => task()))
+}
 
 /**
  * @name downloadContentfulAssets
@@ -27,16 +43,17 @@ const downloadContentfulAssets = async gatsbyFunctions => {
     getCache,
     getNodesByType,
     reporter,
+    assetDownloadWorkers,
   } = gatsbyFunctions
 
   // Any ContentfulAsset nodes will be downloaded, cached and copied to public/static
   // regardless of if you use `localFile` to link an asset or not.
 
-  await Promise.all(
-    getNodesByType(`ContentfulAsset`).map(async node => {
-      totalJobs += 1
-      bar.total = totalJobs
+  const assetNodes = getNodesByType(`ContentfulAsset`)
+  bar.total = assetNodes.length
 
+  await distributeWorkload(
+    assetNodes.map(node => async () => {
       let fileNodeID
       const { contentful_id: id, node_locale: locale } = node
       const remoteDataCacheKey = `contentful-asset-${id}-${locale}`
@@ -90,7 +107,8 @@ const downloadContentfulAssets = async gatsbyFunctions => {
       }
 
       return node
-    })
+    }),
+    assetDownloadWorkers
   )
 }
 exports.downloadContentfulAssets = downloadContentfulAssets

packages/gatsby-source-contentful/src/gatsby-node.js

@@ -122,6 +122,12 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo
           `Number of entries to retrieve from Contentful at a time. Due to some technical limitations, the response payload should not be greater than 7MB when pulling content from Contentful. If you encounter this issue you can set this param to a lower number than 100, e.g 50.`
         )
         .default(100),
+      assetDownloadWorkers: Joi.number()
+        .integer()
+        .description(
+          `Number of workers to use when downloading contentful assets. Due to technical limitations, opening too many concurrent requests can cause stalled downloads. If you encounter this issue you can set this param to a lower number than 50, e.g 25.`
+        )
+        .default(50),
       proxy: Joi.object()
         .keys({
           host: Joi.string().required(),
@@ -562,6 +568,7 @@ exports.sourceNodes = async (
       getCache,
       getNodesByType,
       reporter,
+      assetDownloadWorkers: pluginConfig.get(`assetDownloadWorkers`),
     })
   }