Skip to content

Commit d2ffc2a

Browse files
piehkathmbeck
andauthored
fix(gatsby): fix webpack compilation when pnpm is used (#38757)
* fix: use different method of figuring out wether to enable .cache resolver plugin * use multiple conditions when deciding wether to enable .cache folder resolver plugin * feat(gatsby-dev-cli): allow using pnpm for installing deps * test: add pnpm test * test: test api functions for pnp and pnpm * fix: api functions * Update packages/gatsby/src/utils/webpack/plugins/cache-folder-resolver.ts Co-authored-by: Katherine Beck <[email protected]> * chore: separate test loops for trying to resolve from cache folder and gatsby package, break on first unresolvable module --------- Co-authored-by: Katherine Beck <[email protected]>
1 parent ce6b3bf commit d2ffc2a

File tree

9 files changed

+188
-24
lines changed

9 files changed

+188
-24
lines changed

.circleci/config.yml

+44
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,9 @@ jobs:
341341
- run:
342342
command: cp -r ./starters/default /tmp/e2e-tests/gatsby-pnp
343343
working_directory: ~/project
344+
- run: # default doesn't have API functions so let's get some of those
345+
command: cp -r ./e2e-tests/adapters/src/api /tmp/e2e-tests/gatsby-pnp/src/api
346+
working_directory: ~/project
344347
- run:
345348
command: touch yarn.lock
346349
working_directory: /tmp/e2e-tests/gatsby-pnp
@@ -379,6 +382,45 @@ jobs:
379382
command: 'DEBUG=start-server-and-test yarn start-server-and-test "yarn develop 2>&1 | tee log.txt" :8000 "! cat log.txt | grep -E ''ERROR #|Require stack:''"'
380383
working_directory: /tmp/e2e-tests/gatsby-pnp
381384

385+
e2e_tests_pnpm:
386+
executor:
387+
name: node
388+
image: "18.12.0"
389+
steps:
390+
- checkout
391+
- run: ./scripts/assert-changed-files.sh "packages/*|.circleci/*"
392+
- <<: *attach_to_bootstrap
393+
- run:
394+
command: mkdir -p /tmp/e2e-tests/
395+
working_directory: ~/project
396+
- run:
397+
command: cp -r ./starters/default /tmp/e2e-tests/gatsby-pnpm
398+
working_directory: ~/project
399+
- run: # default doesn't have API functions so let's get some of those
400+
command: cp -r ./e2e-tests/adapters/src/api /tmp/e2e-tests/gatsby-pnpm/src/api
401+
working_directory: ~/project
402+
- run:
403+
command: rm package-lock.json
404+
working_directory: /tmp/e2e-tests/gatsby-pnpm
405+
- run: # Install pnpm
406+
command: npm install -g pnpm
407+
working_directory: /tmp/e2e-tests/gatsby-pnpm
408+
- run: # Install start-server-and-test
409+
command: npm install -g start-server-and-test@^1.11.0
410+
working_directory: /tmp/e2e-tests/gatsby-pnpm
411+
- run: # Set project dir
412+
command: node ~/project/packages/gatsby-dev-cli/dist/index.js --set-path-to-repo ~/project
413+
working_directory: /tmp/e2e-tests/gatsby-pnpm
414+
- run: # Copy over packages
415+
command: node ~/project/packages/gatsby-dev-cli/dist/index.js --force-install --scan-once --package-manager pnpm
416+
working_directory: /tmp/e2e-tests/gatsby-pnpm
417+
- run:
418+
command: pnpm build
419+
working_directory: /tmp/e2e-tests/gatsby-pnpm
420+
- run:
421+
command: 'DEBUG=start-server-and-test pnpm start-server-and-test "pnpm develop 2>&1 | tee log.txt" :8000 "! cat log.txt | grep -E ''ERROR #|Require stack:''"'
422+
working_directory: /tmp/e2e-tests/gatsby-pnpm
423+
382424
e2e_tests_development_runtime_with_react_18:
383425
<<: *e2e-executor
384426
steps:
@@ -606,6 +648,8 @@ workflows:
606648
- bootstrap
607649
- e2e_tests_pnp:
608650
<<: *e2e-test-workflow
651+
- e2e_tests_pnpm:
652+
<<: *e2e-test-workflow
609653
- e2e_tests_path-prefix:
610654
<<: *e2e-test-workflow
611655
- e2e_tests_visual-regression:

packages/gatsby-dev-cli/src/index.js

+8-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,13 @@ You typically only need to configure this once.`
4444
.alias(`h`, `help`)
4545
.nargs(`v`, 0)
4646
.alias(`v`, `version`)
47-
.describe(`v`, `Print the currently installed version of Gatsby Dev CLI`).argv
47+
.describe(`v`, `Print the currently installed version of Gatsby Dev CLI`)
48+
.choices(`package-manager`, [`yarn`, `pnpm`])
49+
.default(`package-manager`, `yarn`)
50+
.describe(
51+
`package-manager`,
52+
`Package manager to use for installing dependencies.`
53+
).argv
4854

4955
if (argv.version) {
5056
console.log(getVersionInfo())
@@ -154,4 +160,5 @@ watch(gatsbyLocation, argv.packages, {
154160
monoRepoPackages,
155161
packageNameToPath,
156162
externalRegistry: argv.externalRegistry,
163+
packageManager: argv.packageManager,
157164
})

packages/gatsby-dev-cli/src/local-npm-registry/index.js

+2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ exports.publishPackagesLocallyAndInstall = async ({
5050
yarnWorkspaceRoot,
5151
externalRegistry,
5252
root,
53+
packageManager,
5354
}) => {
5455
await startServer()
5556

@@ -75,5 +76,6 @@ exports.publishPackagesLocallyAndInstall = async ({
7576
yarnWorkspaceRoot,
7677
newlyPublishedPackageVersions,
7778
externalRegistry,
79+
packageManager,
7880
})
7981
}

packages/gatsby-dev-cli/src/local-npm-registry/install-packages.js

+25-12
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const installPackages = async ({
99
yarnWorkspaceRoot,
1010
newlyPublishedPackageVersions,
1111
externalRegistry,
12+
packageManager,
1213
}) => {
1314
console.log(
1415
`Installing packages from local registry:\n${packagesToInstall
@@ -127,20 +128,32 @@ const installPackages = async ({
127128

128129
installCmd = [`yarn`, yarnCommands]
129130
} else {
130-
const yarnCommands = [
131-
`add`,
132-
...packagesToInstall.map(packageName => {
133-
const packageVersion = newlyPublishedPackageVersions[packageName]
134-
return `${packageName}@${packageVersion}`
135-
}),
136-
`--exact`,
137-
]
131+
const packageAndVersionsToInstall = packagesToInstall.map(packageName => {
132+
const packageVersion = newlyPublishedPackageVersions[packageName]
133+
return `${packageName}@${packageVersion}`
134+
})
138135

139-
if (!externalRegistry) {
140-
yarnCommands.push(`--registry=${registryUrl}`)
141-
}
136+
if (packageManager === `pnpm`) {
137+
const pnpmCommands = [
138+
`add`,
139+
...packageAndVersionsToInstall,
140+
`--save-exact`,
141+
]
142142

143-
installCmd = [`yarn`, yarnCommands]
143+
if (!externalRegistry) {
144+
pnpmCommands.push(`--registry=${registryUrl}`)
145+
}
146+
147+
installCmd = [`pnpm`, pnpmCommands]
148+
} else {
149+
const yarnCommands = [`add`, ...packageAndVersionsToInstall, `--exact`]
150+
151+
if (!externalRegistry) {
152+
yarnCommands.push(`--registry=${registryUrl}`)
153+
}
154+
155+
installCmd = [`yarn`, yarnCommands]
156+
}
144157
}
145158

146159
try {

packages/gatsby-dev-cli/src/local-npm-registry/verdaccio-config.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ const verdaccioConfig = {
1010
title: `gatsby-dev`,
1111
},
1212
self_path: `./`,
13-
logs: [{ type: `stdout`, format: `pretty-timestamped`, level: `warn` }],
13+
logs: { type: `stdout`, format: `pretty-timestamped`, level: `warn` },
1414
packages: {
1515
"**": {
1616
access: `$all`,

packages/gatsby-dev-cli/src/watch.js

+3
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ async function watch(
3838
localPackages,
3939
packageNameToPath,
4040
externalRegistry,
41+
packageManager,
4142
}
4243
) {
4344
setDefaultSpawnStdio(quiet ? `ignore` : `inherit`)
@@ -158,6 +159,7 @@ async function watch(
158159
yarnWorkspaceRoot,
159160
externalRegistry,
160161
root,
162+
packageManager,
161163
})
162164
} else {
163165
// run `yarn`
@@ -344,6 +346,7 @@ async function watch(
344346
ignorePackageJSONChanges,
345347
externalRegistry,
346348
root,
349+
packageManager,
347350
})
348351
packagesToPublish.clear()
349352
isPublishing = false

packages/gatsby/src/internal-plugins/functions/api-function-webpack-loader.ts

+8-4
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,14 @@ const APIFunctionLoader: LoaderDefinition = async function () {
1313
const functionModule = require('${modulePath}');
1414
const functionToExecute = preferDefault(functionModule);
1515
const matchPath = '${matchPath}';
16-
const { match: reachMatch } = require('@gatsbyjs/reach-router');
17-
const { urlencoded, text, json, raw } = require('body-parser')
18-
const multer = require('multer')
19-
const { createConfig } = require('gatsby/dist/internal-plugins/functions/config')
16+
const { match: reachMatch } = require('${require.resolve(
17+
`@gatsbyjs/reach-router`
18+
)}');
19+
const { urlencoded, text, json, raw } = require('${require.resolve(
20+
`body-parser`
21+
)}')
22+
const multer = require('${require.resolve(`multer`)}')
23+
const { createConfig } = require('${require.resolve(`./config`)}')
2024
2125
function functionWrapper(req, res) {
2226
if (matchPath) {

packages/gatsby/src/internal-plugins/functions/gatsby-node.ts

+13-4
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { reportWebpackWarnings } from "../../utils/webpack-error-utils"
1212
import { internalActions } from "../../redux/actions"
1313
import { IGatsbyFunction } from "../../redux/types"
1414
import { functionMiddlewares } from "./middleware"
15+
import mod from "module"
1516

1617
const isProductionEnv = process.env.gatsby_executing_command !== `develop`
1718

@@ -305,6 +306,10 @@ const createWebpackConfig = async ({
305306
? `functions-production`
306307
: `functions-development`
307308

309+
const gatsbyPluginTSRequire = mod.createRequire(
310+
require.resolve(`gatsby-plugin-typescript`)
311+
)
312+
308313
return {
309314
entry: entries,
310315
output: {
@@ -373,19 +378,23 @@ const createWebpackConfig = async ({
373378
},
374379
},
375380
use: {
376-
loader: `babel-loader`,
381+
loader: require.resolve(`babel-loader`),
377382
options: {
378-
presets: [`@babel/typescript`],
383+
presets: [
384+
gatsbyPluginTSRequire.resolve(`@babel/preset-typescript`),
385+
],
379386
},
380387
},
381388
},
382389
{
383390
test: [/.js$/, /.ts$/],
384391
exclude: /node_modules/,
385392
use: {
386-
loader: `babel-loader`,
393+
loader: require.resolve(`babel-loader`),
387394
options: {
388-
presets: [`@babel/typescript`],
395+
presets: [
396+
gatsbyPluginTSRequire.resolve(`@babel/preset-typescript`),
397+
],
389398
},
390399
},
391400
},

packages/gatsby/src/utils/webpack/plugins/cache-folder-resolver.ts

+84-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import Resolver from "enhanced-resolve/lib/Resolver"
2+
import mod from "module"
23

34
interface IRequest {
45
request?: string
@@ -8,17 +9,98 @@ interface IRequest {
89
type ProcessWithPNP = NodeJS.ProcessVersions & { pnp?: string }
910

1011
/**
11-
* To support PNP we have to make sure dependencies resolved from the .cache folder should be resolved from the gatsby package directory
12+
* To support Yarn PNP and pnpm we have to make sure dependencies resolved from
13+
* the .cache folder should be resolved from the gatsby package directory
14+
* If you see error like
15+
*
16+
* ModuleNotFoundError: Module not found: Error: Can't resolve 'prop-types'
17+
* in '<site-directory>/.cache'
18+
*
19+
* it probably means this plugin is not enabled when it should be and there
20+
* might be need to adjust conditions for setting `this.isEnabled` in the
21+
* constructor.
22+
*
23+
* It's not enabled always because of legacy behavior and to limit potential
24+
* regressions. Might be good idea to enable it always in the future
25+
* OR remove the need for the plugin completely by not copying `cache-dir`
26+
* contents to `.cache` folder and instead adjust setup to use those browser/node
27+
* html renderer runtime files directly from gatsby package
1228
*/
1329
export class CacheFolderResolver {
1430
private requestsFolder: string
31+
private isEnabled = false
1532

1633
constructor(requestsFolder: string) {
1734
this.requestsFolder = requestsFolder
35+
36+
if ((process.versions as ProcessWithPNP).pnp) {
37+
// Yarn PnP
38+
this.isEnabled = true
39+
} else if (/node_modules[/\\]\.pnpm/.test(process.env.NODE_PATH ?? ``)) {
40+
// pnpm when executing through `pnpm` CLI
41+
this.isEnabled = true
42+
} else {
43+
// pnpm when executing through regular `gatsby` CLI / `./node_modules/.bin/gatsby`
44+
// would not set NODE_PATH, but node_modules structure would not allow to resolve
45+
// gatsby deps from the cache folder (unless user would install same deps too)
46+
// so we are checking if we can resolve deps from the cache folder
47+
// this check is not limited to pnpm and other package managers could hit this path too
48+
49+
// Hardcoded list of gatsby deps used in gatsby browser and ssr runtimes
50+
// instead of checking if we use Yarn PnP (via `process.versions.pnp`),
51+
// we check if we can resolve the external deps from the cache-dir folder
52+
// to know if we need to enable this plugin so we also cover pnpm
53+
// It might be good idea to always enable it overall, but to limit potential
54+
// regressions we only enable it if we are sure we need it.
55+
const modulesToCheck = [
56+
`prop-types`,
57+
`lodash/isEqual`,
58+
`mitt`,
59+
`shallow-compare`,
60+
`@gatsbyjs/reach-router`,
61+
`gatsby-react-router-scroll`,
62+
`react-server-dom-webpack`,
63+
`gatsby-link`,
64+
]
65+
66+
// test if we can resolve deps from the cache folder
67+
let isEverythingResolvableFromCacheDir = true
68+
const cacheDirReq = mod.createRequire(requestsFolder)
69+
for (const cacheDirDep of modulesToCheck) {
70+
try {
71+
cacheDirReq.resolve(cacheDirDep)
72+
} catch {
73+
// something is not resolvable from the cache folder, so we should not enable this plugin
74+
isEverythingResolvableFromCacheDir = false
75+
break
76+
}
77+
}
78+
79+
// test if we can resolve deps from the gatsby package
80+
let isEverythingResolvableFromGatsbyPackage = true
81+
for (const cacheDirDep of modulesToCheck) {
82+
try {
83+
require.resolve(cacheDirDep)
84+
} catch {
85+
// something is not resolvable from the gatsby package
86+
isEverythingResolvableFromGatsbyPackage = false
87+
break
88+
}
89+
}
90+
91+
// we only enable this plugin if we are unable to resolve cache-dir deps from .cache folder
92+
// and we can resolve them from gatsby package
93+
if (
94+
!isEverythingResolvableFromCacheDir &&
95+
isEverythingResolvableFromGatsbyPackage
96+
) {
97+
this.isEnabled = true
98+
}
99+
}
18100
}
19101

20102
apply(resolver: Resolver): void {
21-
if (!(process.versions as ProcessWithPNP).pnp) {
103+
if (!this.isEnabled) {
22104
return
23105
}
24106

0 commit comments

Comments
 (0)