feat(gatsby): engines activity (#33273)

Author: wardpeet

integration-tests/artifacts/gatsby-config.js

@@ -6,8 +6,5 @@ module.exports = {
     github: `sidharthachatterjee`,
     moreInfo: `Sid is amazing`,
   },
-  flags: {
-    PRESERVE_WEBPACK_CACHE: true,
-  },
   plugins: [`gatsby-plugin-webpack-1`, `gatsby-plugin-webpack-2`],
 }

packages/gatsby/src/commands/build-html.ts

@@ -3,11 +3,12 @@ import fs from "fs-extra"
 import reporter from "gatsby-cli/lib/reporter"
 import { createErrorFromString } from "gatsby-cli/lib/reporter/errors"
 import { chunk, truncate } from "lodash"
-import webpack, { Stats } from "webpack"
+import { build, watch } from "../utils/webpack/bundle"
 import * as path from "path"
 
 import { emitter, store } from "../redux"
 import { IWebpackWatchingPauseResume } from "../utils/start-server"
+import webpack from "webpack"
 import webpackConfig from "../utils/webpack.config"
 import { structureWebpackErrors } from "../utils/webpack-error-utils"
 import * as buildUtils from "./build-utils"
@@ -17,10 +18,10 @@ import { Span } from "opentracing"
 import { IProgram, Stage } from "./types"
 import { ROUTES_DIRECTORY } from "../constants"
 import { PackageJson } from "../.."
-import type { GatsbyWorkerPool } from "../utils/worker/pool"
 import { IPageDataWithQueryResult } from "../utils/page-data"
 import { processNodeManifests } from "../utils/node-manifest"
 
+import type { GatsbyWorkerPool } from "../utils/worker/pool"
 type IActivity = any // TODO
 
 export interface IBuildArgs extends IProgram {
@@ -37,8 +38,13 @@ export interface IBuildArgs extends IProgram {
   keepPageRenderer: boolean
 }
 
-let devssrWebpackCompiler: webpack.Compiler
-let devssrWebpackWatcher: IWebpackWatchingPauseResume
+interface IBuildRendererResult {
+  rendererPath: string
+  stats: webpack.Stats
+  close: ReturnType<typeof watch>["close"]
+}
+
+let devssrWebpackCompiler: webpack.Watching | undefined
 let needToRecompileSSRBundle = true
 export const getDevSSRWebpack = (): {
   devssrWebpackWatcher: IWebpackWatchingPauseResume
@@ -50,8 +56,8 @@ export const getDevSSRWebpack = (): {
   }
 
   return {
-    devssrWebpackWatcher,
-    devssrWebpackCompiler,
+    devssrWebpackWatcher: devssrWebpackCompiler as webpack.Watching,
+    devssrWebpackCompiler: (devssrWebpackCompiler as webpack.Watching).compiler,
     needToRecompileSSRBundle,
   }
 }
@@ -61,60 +67,33 @@ let newHash = ``
 const runWebpack = (
   compilerConfig,
   stage: Stage,
-  directory,
-  parentSpan?: Span
-): Bluebird<{
-  stats: Stats
-  waitForCompilerClose: Promise<void>
-}> =>
-  new Bluebird((resolve, reject) => {
-    if (!process.env.GATSBY_EXPERIMENTAL_DEV_SSR || stage === `build-html`) {
+  directory: string
+): Promise<{
+  stats: webpack.Stats
+  close: ReturnType<typeof watch>["close"]
+}> => {
+  const isDevSSREnabledAndViable =
+    process.env.GATSBY_EXPERIMENTAL_DEV_SSR && stage === `develop-html`
+
+  return new Promise((resolve, reject) => {
+    if (isDevSSREnabledAndViable) {
       const compiler = webpack(compilerConfig)
-      compiler.run((err, stats) => {
-        let activity
-        if (process.env.GATSBY_EXPERIMENTAL_PRESERVE_WEBPACK_CACHE) {
-          activity = reporter.activityTimer(
-            `Caching HTML renderer compilation`,
-            { parentSpan }
-          )
-          activity.start()
-        }
-
-        const waitForCompilerClose = new Promise<void>((resolve, reject) => {
-          compiler.close(error => {
-            if (activity) {
-              activity.end()
-            }
-
-            if (error) {
-              return reject(error)
-            }
-            return resolve()
-          })
-        })
 
-        if (err) {
-          return reject(err)
-        } else {
-          return resolve({ stats: stats as Stats, waitForCompilerClose })
-        }
-      })
-    } else if (
-      process.env.GATSBY_EXPERIMENTAL_DEV_SSR &&
-      stage === `develop-html`
-    ) {
-      devssrWebpackCompiler = webpack(compilerConfig)
-      devssrWebpackCompiler.hooks.invalid.tap(`ssr file invalidation`, () => {
+      // because of this line we can't use our watch helper
+      // These things should use emitter
+      compiler.hooks.invalid.tap(`ssr file invalidation`, () => {
         needToRecompileSSRBundle = true
       })
-      devssrWebpackWatcher = devssrWebpackCompiler.watch(
+
+      const watcher = compiler.watch(
         {
           ignored: /node_modules/,
         },
         (err, stats) => {
+          // this runs multiple times
           needToRecompileSSRBundle = false
           emitter.emit(`DEV_SSR_COMPILATION_DONE`)
-          devssrWebpackWatcher.suspend()
+          watcher.suspend()
 
           if (err) {
             return reject(err)
@@ -132,59 +111,52 @@ const runWebpack = (
             oldHash = newHash
 
             return resolve({
-              stats: stats as Stats,
-              waitForCompilerClose: Promise.resolve(),
+              stats: stats as webpack.Stats,
+              close: () =>
+                new Promise((resolve, reject): void =>
+                  watcher.close(err => (err ? reject(err) : resolve()))
+                ),
             })
           }
         }
-      ) as IWebpackWatchingPauseResume
+      )
+      devssrWebpackCompiler = watcher
+    } else {
+      build(compilerConfig).then(({ stats, close }) => {
+        resolve({ stats, close })
+      })
     }
   })
+}
 
 const doBuildRenderer = async (
-  { directory }: IProgram,
+  directory: string,
   webpackConfig: webpack.Configuration,
-  stage: Stage,
-  parentSpan?: Span
-): Promise<{ rendererPath: string; waitForCompilerClose }> => {
-  const { stats, waitForCompilerClose } = await runWebpack(
-    webpackConfig,
-    stage,
-    directory,
-    parentSpan
-  )
-  if (stats.hasErrors()) {
+  stage: Stage
+): Promise<IBuildRendererResult> => {
+  const { stats, close } = await runWebpack(webpackConfig, stage, directory)
+  if (stats?.hasErrors()) {
     reporter.panic(structureWebpackErrors(stage, stats.compilation.errors))
   }
 
-  if (
-    stage === `build-html` &&
-    store.getState().html.ssrCompilationHash !== stats.hash
-  ) {
-    store.dispatch({
-      type: `SET_SSR_WEBPACK_COMPILATION_HASH`,
-      payload: stats.hash as string,
-    })
-  }
-
   // render-page.js is hard coded in webpack.config
   return {
     rendererPath: `${directory}/${ROUTES_DIRECTORY}render-page.js`,
-    waitForCompilerClose,
+    stats,
+    close,
   }
 }
 
 export const buildRenderer = async (
   program: IProgram,
   stage: Stage,
   parentSpan?: IActivity
-): Promise<{ rendererPath: string; waitForCompilerClose }> => {
-  const { directory } = program
-  const config = await webpackConfig(program, directory, stage, null, {
+): Promise<IBuildRendererResult> => {
+  const config = await webpackConfig(program, program.directory, stage, null, {
     parentSpan,
   })
 
-  return doBuildRenderer(program, config, stage, parentSpan)
+  return doBuildRenderer(program.directory, config, stage)
 }
 
 // TODO remove after v4 release and update cloud internals

packages/gatsby/src/commands/build-javascript.ts

@@ -1,20 +1,12 @@
 import { Span } from "opentracing"
-import webpack, { WebpackError } from "webpack"
 import webpackConfig from "../utils/webpack.config"
-import { IProgram } from "./types"
-import reporter from "gatsby-cli/lib/reporter"
-import {
-  showExperimentNoticeAfterTimeout,
-  CancelExperimentNoticeCallbackOrUndefined,
-} from "../utils/show-experiment-notice"
+import { build } from "../utils/webpack/bundle"
+import type { IProgram } from "./types"
 
 export const buildProductionBundle = async (
   program: IProgram,
   parentSpan: Span
-): Promise<{
-  stats: webpack.Stats | Error | Array<WebpackError>
-  waitForCompilerClose: Promise<void>
-}> => {
+): Promise<ReturnType<typeof build>> => {
   const { directory } = program
 
   const compilerConfig = await webpackConfig(
@@ -25,67 +17,5 @@ export const buildProductionBundle = async (
     { parentSpan }
   )
 
-  return new Promise((resolve, reject) => {
-    const compiler = webpack(compilerConfig)
-
-    const THIRTY_SECONDS = 30 * 1000
-    let cancelCacheNotice: CancelExperimentNoticeCallbackOrUndefined
-    if (
-      process.env.gatsby_executing_command === `build` &&
-      !process.env.GATSBY_EXPERIMENTAL_PRESERVE_WEBPACK_CACHE
-    ) {
-      cancelCacheNotice = showExperimentNoticeAfterTimeout(
-        `Persistent webpack caching`,
-        `https://github.com/gatsbyjs/gatsby/discussions/28331`,
-        `which enables webpack's persist caching and changes Gatsby's cache clearing behavior to not clear webpack's
-cache unless you run "gatsby clean" or delete the .cache folder manually.
-Here's how to try it:
-
-module.exports = {
-  flags: { PRESERVE_WEBPACK_CACHE: true },
-  plugins: [...]
-}`,
-        THIRTY_SECONDS
-      )
-    }
-
-    compiler.run((err, stats) => {
-      if (cancelCacheNotice) {
-        cancelCacheNotice()
-      }
-
-      // stats can only be empty when an error occurs. Adding it to the if makes typescript happy.
-      if (err || !stats) {
-        return reject(err)
-      }
-
-      if (stats.hasErrors()) {
-        return reject(stats.compilation.errors)
-      }
-
-      let activity
-      if (process.env.GATSBY_EXPERIMENTAL_PRESERVE_WEBPACK_CACHE) {
-        activity = reporter.activityTimer(
-          `Caching JavaScript and CSS webpack compilation`,
-          { parentSpan }
-        )
-        activity.start()
-      }
-
-      const waitForCompilerClose = new Promise<void>((resolve, reject) => {
-        compiler.close(error => {
-          if (activity) {
-            activity.end()
-          }
-
-          if (error) {
-            return reject(error)
-          }
-          return resolve()
-        })
-      })
-
-      return resolve({ stats, waitForCompilerClose })
-    })
-  })
+  return build(compilerConfig)
 }