feat(gatsby-plugin-sharp): Use file streams instead of file paths (#33029)

* copied changes

* new changes

* fix(sharp): retain order of operations and images

* use toBuffer and writeFile instead write stream (at least for now)

* revert making queueImageResizing async

* fixing up too much of revert in previous commit

Co-authored-by: Michal Piechowiak <[email protected]>

Author: LekoArts

packages/gatsby-plugin-sharp/src/__tests__/trace-svg.js

@@ -6,6 +6,11 @@ jest.mock(`sharp`, () => {
       png: () => pipeline,
       jpeg: () => pipeline,
       toFile: (_, cb) => cb(),
+      on: () => pipeline,
+      once: () => pipeline,
+      write: () => pipeline,
+      end: () => pipeline,
+      emit: () => pipeline,
     }
     return pipeline
   }
@@ -16,6 +21,18 @@ jest.mock(`sharp`, () => {
   return sharp
 })
 
+jest.mock(`fs-extra`, () => {
+  return {
+    ...jest.requireActual(`fs-extra`),
+    createReadStream: () => {
+      const stream = {
+        pipe: () => stream,
+      }
+      return stream
+    },
+  }
+})
+
 jest.mock(`potrace`, () => {
   const circleSvgString = `<svg height="100" width="100"><circle cx="50" cy="50" r="40" /></svg>`
   return {

packages/gatsby-plugin-sharp/src/image-data.ts

@@ -2,6 +2,7 @@
 import { IGatsbyImageData, ISharpGatsbyImageArgs } from "gatsby-plugin-image"
 import { GatsbyCache, Node } from "gatsby"
 import { Reporter } from "gatsby/reporter"
+import fs from "fs-extra"
 import { rgbToHex, calculateImageSizes, getSrcSet, getSizes } from "./utils"
 import { traceSVG, getImageSizeAsync, base64, batchQueueImageResizing } from "."
 import sharp from "./safe-sharp"
@@ -15,7 +16,7 @@ const DEFAULT_BREAKPOINTS = [750, 1080, 1366, 1920]
 type ImageFormat = "jpg" | "png" | "webp" | "avif" | "" | "auto"
 
 export type FileNode = Node & {
-  absolutePath?: string
+  absolutePath: string
   extension: string
 }
 
@@ -44,7 +45,9 @@ export async function getImageMetadata(
   }
 
   try {
-    const pipeline = sharp(file.absolutePath)
+    const pipeline = sharp()
+
+    fs.createReadStream(file.absolutePath).pipe(pipeline)
 
     const { width, height, density, format } = await pipeline.metadata()
 

packages/gatsby-plugin-sharp/src/index.js

@@ -267,13 +267,12 @@ async function generateBase64({ file, args = {}, reporter }) {
   })
   let pipeline
   try {
-    pipeline = !options.failOnError
-      ? sharp(file.absolutePath, { failOnError: false })
-      : sharp(file.absolutePath)
+    pipeline = !options.failOnError ? sharp({ failOnError: false }) : sharp()
 
     if (!options.rotate) {
       pipeline.rotate()
     }
+    fs.createReadStream(file.absolutePath).pipe(pipeline)
   } catch (err) {
     reportError(`Failed to process image ${file.absolutePath}`, err, reporter)
     return null
@@ -413,7 +412,10 @@ async function getTracedSVG({ file, options, cache, reporter }) {
 async function stats({ file, reporter }) {
   let imgStats
   try {
-    imgStats = await sharp(file.absolutePath).stats()
+    const pipeline = sharp()
+    fs.createReadStream(file.absolutePath).pipe(pipeline)
+
+    imgStats = await pipeline.stats()
   } catch (err) {
     reportError(
       `Failed to get stats for image ${file.absolutePath}`,
@@ -450,7 +452,10 @@ async function fluid({ file, args = {}, reporter, cache }) {
   // images are intended to be displayed at their native resolution.
   let metadata
   try {
-    metadata = await sharp(file.absolutePath).metadata()
+    const pipeline = sharp()
+    fs.createReadStream(file.absolutePath).pipe(pipeline)
+
+    metadata = await pipeline.metadata()
   } catch (err) {
     reportError(
       `Failed to retrieve metadata from image ${file.absolutePath}`,

packages/gatsby-plugin-sharp/src/process-file.js

@@ -56,14 +56,13 @@ sharp.concurrency(1)
 exports.processFile = (file, transforms, options = {}) => {
   let pipeline
   try {
-    pipeline = !options.failOnError
-      ? sharp(file, { failOnError: false })
-      : sharp(file)
+    pipeline = !options.failOnError ? sharp({ failOnError: false }) : sharp()
 
     // Keep Metadata
     if (!options.stripMetadata) {
       pipeline = pipeline.withMetadata()
     }
+    fs.createReadStream(file).pipe(pipeline)
   } catch (err) {
     throw new SharpError(`Failed to load image ${file} into sharp.`, err)
   }
@@ -152,7 +151,8 @@ exports.processFile = (file, transforms, options = {}) => {
       }
 
       try {
-        await clonedPipeline.toFile(outputPath)
+        const buffer = await clonedPipeline.toBuffer()
+        await fs.writeFile(outputPath, buffer)
       } catch (err) {
         throw new Error(
           `Failed to write ${file} into ${outputPath}. (${err.message})`

packages/gatsby-plugin-sharp/src/trace-svg.js

@@ -1,4 +1,5 @@
 const { promisify } = require(`bluebird`)
+const fs = require(`fs-extra`)
 const _ = require(`lodash`)
 const tmpDir = require(`os`).tmpdir()
 const path = require(`path`)
@@ -17,11 +18,12 @@ exports.notMemoizedPrepareTraceSVGInputFile = async ({
 }) => {
   let pipeline
   try {
-    pipeline = sharp(file.absolutePath)
+    pipeline = sharp()
 
     if (!options.rotate) {
       pipeline.rotate()
     }
+    fs.createReadStream(file.absolutePath).pipe(pipeline)
   } catch (err) {
     reportError(`Failed to process image ${file.absolutePath}`, err, reporter)
     return

packages/gatsby-plugin-sharp/src/utils.js

@@ -1,4 +1,5 @@
 import sharp from "./safe-sharp"
+import fs from "fs-extra"
 
 export function rgbToHex(red, green, blue) {
   return `#${(blue | (green << 8) | (red << 16) | (1 << 24))
@@ -39,6 +40,7 @@ export function calculateImageSizes(args) {
     return []
   }
 }
+
 export function fixedImageSizes({
   file,
   imgDimensions,
@@ -288,7 +290,10 @@ export const getDominantColor = async absolutePath => {
     return dominantColor
   }
 
-  const pipeline = sharp(absolutePath)
+  const pipeline = sharp()
+
+  fs.createReadStream(absolutePath).pipe(pipeline)
+
   const { dominant } = await pipeline.stats()
 
   // Fallback in case sharp doesn't support dominant