chore(gatsby-transformer-sqip): improve sqip performance and debug logging (#13349)

Author: axe312ger

packages/gatsby-transformer-sqip/package.json

@@ -8,8 +8,10 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.0.0",
+    "bluebird": "^3.5.4",
     "fs-extra": "^4.0.2",
     "gatsby-plugin-sharp": "^2.1.5",
+    "md5-file": "^4.0.0",
     "mini-svg-data-uri": "^1.0.0",
     "p-queue": "^2.3.0",
     "sqip": "^0.3.3"

packages/gatsby-transformer-sqip/src/extend-node-type.js

@@ -1,4 +1,5 @@
 const { resolve } = require(`path`)
+const md5File = require(`bluebird`).promisify(require(`md5-file`))
 
 const {
   DuotoneGradientType,
@@ -43,7 +44,9 @@ module.exports = async args => {
 
 async function sqipSharp({ type, cache, getNodeAndSavePathDependency, store }) {
   const program = store.getState().program
-  const cacheDir = resolve(`${program.directory}/.cache/sqip/`)
+  const cacheDir = resolve(
+    `${program.directory}/node_modules/.cache/gatsby-transformer-sqip/`
+  )
 
   await ensureDir(cacheDir)
 
@@ -107,6 +110,7 @@ async function sqipSharp({ type, cache, getNodeAndSavePathDependency, store }) {
         }
 
         const file = getNodeAndSavePathDependency(image.parent, context.path)
+        const { contentDigest } = image.internal
 
         const job = await queueImageResizing({ file, args: sharpArgs })
 
@@ -120,6 +124,7 @@ async function sqipSharp({ type, cache, getNodeAndSavePathDependency, store }) {
         return generateSqip({
           cache,
           cacheDir,
+          contentDigest,
           absolutePath,
           numberOfPrimitives,
           blur,
@@ -219,9 +224,12 @@ async function sqipContentful({ type, cache, store }) {
 
         const absolutePath = await cacheImage(store, asset, options)
 
+        const contentDigest = await md5File(absolutePath)
+
         return generateSqip({
           cache,
           cacheDir,
+          contentDigest,
           absolutePath,
           numberOfPrimitives,
           blur,

packages/gatsby-transformer-sqip/src/generate-sqip.js

@@ -18,6 +18,7 @@ module.exports = async function generateSqip(options) {
     blur,
     mode,
     cacheDir,
+    contentDigest,
   } = options
 
   debug({ options })
@@ -35,17 +36,59 @@ module.exports = async function generateSqip(options) {
     .update(JSON.stringify(sqipOptions))
     .digest(`hex`)
 
-  const cacheKey = `sqip-${name}-${optionsHash}`
-  const cachePath = resolve(cacheDir, `${name}-${optionsHash}.svg`)
-  let primitiveData = await cache.get(cacheKey)
+  const cacheKey = `${contentDigest}-${optionsHash}`
+  const cachePath = resolve(cacheDir, `${contentDigest}-${optionsHash}.svg`)
 
-  debug({ primitiveData })
+  debug(
+    `Request preview generation for ${name} (${contentDigest}-${optionsHash})`
+  )
 
-  if (!primitiveData) {
+  return queue.add(async () => {
+    let primitiveData = await cache.get(cacheKey)
     let svg
-    if (await exists(cachePath)) {
-      const svgBuffer = await readFile(cachePath)
-      svg = svgBuffer.toString()
+
+    debug(
+      `Executing preview generation request for ${name} (${contentDigest}-${optionsHash})`
+    )
+
+    if (!primitiveData) {
+      if (await exists(cachePath)) {
+        debug(
+          `Primitive result file already exists for ${name} (${contentDigest}-${optionsHash})`
+        )
+        const svgBuffer = await readFile(cachePath)
+        svg = svgBuffer.toString()
+      } else {
+        debug(
+          `Generate primitive result file of ${name} (${contentDigest}-${optionsHash})`
+        )
+
+        const result = await new Promise((resolve, reject) => {
+          try {
+            const result = sqip({
+              filename: absolutePath,
+              ...sqipOptions,
+            })
+            resolve(result)
+          } catch (error) {
+            reject(error)
+          }
+        })
+
+        svg = result.final_svg
+
+        await writeFile(cachePath, svg)
+        debug(
+          `Wrote primitive result file to disk for ${name} (${contentDigest}-${optionsHash})`
+        )
+      }
+
+      primitiveData = {
+        svg,
+        dataURI: svgToMiniDataURI(svg),
+      }
+
+      await cache.set(cacheKey, primitiveData)
     } else {
       debug(`generate sqip for ${name}`)
       const result = await queue.add(
@@ -74,7 +117,7 @@ module.exports = async function generateSqip(options) {
     }
 
     await cache.set(cacheKey, primitiveData)
-  }
 
-  return primitiveData
+    return primitiveData
+  })
 }

yarn.lock

@@ -5101,6 +5101,11 @@ bluebird@^3.5.3:
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.3.tgz#7d01c6f9616c9a51ab0f8c549a79dfe6ec33efa7"
   integrity sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==
 
+bluebird@^3.5.4:
+  version "3.5.5"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.5.tgz#a8d0afd73251effbbd5fe384a77d73003c17a71f"
+  integrity sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==
+
 [email protected]:
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/bmp-js/-/bmp-js-0.0.3.tgz#64113e9c7cf1202b376ed607bf30626ebe57b18a"
@@ -14654,6 +14659,11 @@ md5-file@^3.1.1:
   dependencies:
     buffer-alloc "^1.1.0"
 
+md5-file@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/md5-file/-/md5-file-4.0.0.tgz#f3f7ba1e2dd1144d5bf1de698d0e5f44a4409584"
+  integrity sha512-UC0qFwyAjn4YdPpKaDNw6gNxRf7Mcx7jC1UGCY4boCzgvU2Aoc1mOGzTtrjjLKhM5ivsnhoKpQVxKPp+1j1qwg==
+
 md5.js@^1.3.4:
   version "1.3.4"
   resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.4.tgz#e9bdbde94a20a5ac18b04340fc5764d5b09d901d"