Skip to content

Commit 54d29c4

Browse files
piehpieh
authored
chore(gatsby): upgrade from lmdb-store to lmdb (#34576)
* optimistic bump * .clear -> clearSync * tmp: patch lmdb so we can bundle it for engines * adjust cache-resilience to import from lmdb * bump lmdb to latest * use clearKeptObjects() * fix mocking os.platform() in tests causing loading of wrong lmdb binary * ensure single instance of lmdb in a process - for gatsby serve case mostly * add comment about lmdb webpack loader patch
1 parent 3df8583 commit 54d29c4

File tree

12 files changed

+143
-42
lines changed

12 files changed

+143
-42
lines changed

‎integration-tests/cache-resilience/gatsby-node.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ const v8 = require(`v8`)
33
const glob = require(`glob`)
44
const path = require(`path`)
55
const _ = require(`lodash`)
6-
const { open } = require(`lmdb-store`)
6+
const { open } = require(`lmdb`)
77

88
const { saveState } = require(`gatsby/dist/redux/save-state`)
99

‎packages/gatsby/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@
103103
"joi": "^17.4.2",
104104
"json-loader": "^0.5.7",
105105
"latest-version": "5.1.0",
106-
"lmdb-store": "^1.6.11",
106+
"lmdb": "2.2.1",
107107
"lodash": "^4.17.21",
108108
"md5-file": "^5.0.0",
109109
"meant": "^1.0.3",

‎packages/gatsby/src/commands/__tests__/build-utils.ts

+4-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import {
55
IStaticQueryResultState,
66
} from "../../redux/types"
77

8-
const platformSpy = jest.spyOn(require(`os`), `platform`)
98
interface IMinimalStateSliceForTest {
109
html: IGatsbyState["html"]
1110
pages: IGatsbyState["pages"]
@@ -245,16 +244,13 @@ describe(`calcDirtyHtmlFiles`, () => {
245244
})
246245

247246
describe(`onCreatePage + deletePage + createPage that change path casing of a page`, () => {
248-
afterAll(() => {
249-
platformSpy.mockRestore()
250-
})
251-
252247
it(`linux (case sensitive file system)`, () => {
253248
let isolatedCalcDirtyHtmlFiles
254249
jest.isolateModules(() => {
255-
platformSpy.mockImplementation(() => `linux`)
250+
process.env.TEST_FORCE_CASE_FS = `SENSITIVE`
256251
isolatedCalcDirtyHtmlFiles =
257252
require(`../build-utils`).calcDirtyHtmlFiles
253+
delete process.env.TEST_FORCE_CASE_FS
258254
})
259255

260256
const state = generateStateToTestHelper({
@@ -280,9 +276,10 @@ describe(`calcDirtyHtmlFiles`, () => {
280276
it(`windows / mac (case insensitive file system)`, () => {
281277
let isolatedCalcDirtyHtmlFiles
282278
jest.isolateModules(() => {
283-
platformSpy.mockImplementation(() => `win32`)
279+
process.env.TEST_FORCE_CASE_FS = `INSENSITIVE`
284280
isolatedCalcDirtyHtmlFiles =
285281
require(`../build-utils`).calcDirtyHtmlFiles
282+
delete process.env.TEST_FORCE_CASE_FS
286283
})
287284

288285
const state = generateStateToTestHelper({

‎packages/gatsby/src/commands/build-utils.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,9 @@ export const removePageFiles = async (
6565
})
6666
}
6767

68-
const FSisCaseInsensitive = platform() === `win32` || platform() === `darwin`
68+
const FSisCaseInsensitive = process.env.TEST_FORCE_CASE_FS
69+
? process.env.TEST_FORCE_CASE_FS === `INSENSITIVE`
70+
: platform() === `win32` || platform() === `darwin`
6971
function normalizePagePath(path: string): string {
7072
if (path === `/`) {
7173
return `/`

‎packages/gatsby/src/datastore/common/iterable.ts

+20-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
declare module "lmdb" {
2+
// currently lmdb doesn't have typings for this export
3+
export function clearKeptObjects(): void
4+
}
5+
6+
import { clearKeptObjects } from "lmdb"
17
/**
28
* Wrapper for any iterable providing chainable interface and convenience methods
39
* similar to array.
@@ -10,10 +16,22 @@
1016
export class GatsbyIterable<T> {
1117
constructor(private source: Iterable<T> | (() => Iterable<T>)) {}
1218

13-
[Symbol.iterator](): Iterator<T> {
19+
*[Symbol.iterator](): Iterator<T> {
1420
const source =
1521
typeof this.source === `function` ? this.source() : this.source
16-
return source[Symbol.iterator]()
22+
23+
let i = 0
24+
for (const val of source) {
25+
yield val
26+
27+
// clearKeptObjects just make it possible for WeakRefs used in any way during current
28+
// sync execution tick to be garbage collected. It doesn't force GC, just remove
29+
// internal strong references in V8.
30+
// see https://github.com/kriszyp/weak-lru-cache/issues/4
31+
if (++i % 100 === 0) {
32+
clearKeptObjects()
33+
}
34+
}
1735
}
1836

1937
concat<U>(other: Iterable<U>): GatsbyIterable<T | U> {

‎packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts

+31-7
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { RootDatabase, open, ArrayLikeIterable } from "lmdb-store"
1+
import { RootDatabase, open, ArrayLikeIterable } from "lmdb"
22
// import { performance } from "perf_hooks"
33
import { ActionsUnion, IGatsbyNode } from "../../redux/types"
44
import { updateNodes } from "./updates/nodes"
@@ -61,8 +61,31 @@ function getRootDb(): RootDatabase {
6161
return rootDb
6262
}
6363

64+
/* eslint-disable @typescript-eslint/no-namespace */
65+
declare global {
66+
namespace NodeJS {
67+
// eslint-disable-next-line @typescript-eslint/naming-convention
68+
interface Global {
69+
__GATSBY_OPEN_LMDBS?: Map<string, ILmdbDatabases>
70+
}
71+
}
72+
}
73+
6474
function getDatabases(): ILmdbDatabases {
6575
if (!databases) {
76+
// __GATSBY_OPEN_LMDBS tracks if we already opened given db in this process
77+
// In `gatsby serve` case we might try to open it twice - once for engines
78+
// and second to get access to `SitePage` nodes (to power trailing slashes
79+
// redirect middleware). This ensure there is single instance within a process.
80+
// Using more instances seems to cause weird random errors.
81+
if (!globalThis.__GATSBY_OPEN_LMDBS) {
82+
globalThis.__GATSBY_OPEN_LMDBS = new Map()
83+
}
84+
databases = globalThis.__GATSBY_OPEN_LMDBS.get(fullDbPath)
85+
if (databases) {
86+
return databases
87+
}
88+
6689
const rootDb = getRootDb()
6790
databases = {
6891
nodes: rootDb.openDB({
@@ -86,6 +109,7 @@ function getDatabases(): ILmdbDatabases {
86109
// dupSort: true
87110
}),
88111
}
112+
globalThis.__GATSBY_OPEN_LMDBS.set(fullDbPath, databases)
89113
}
90114
return databases
91115
}
@@ -184,10 +208,10 @@ function updateDataStore(action: ActionsUnion): void {
184208
const dbs = getDatabases()
185209
// Force sync commit
186210
dbs.nodes.transactionSync(() => {
187-
dbs.nodes.clear()
188-
dbs.nodesByType.clear()
189-
dbs.metadata.clear()
190-
dbs.indexes.clear()
211+
dbs.nodes.clearSync()
212+
dbs.nodesByType.clearSync()
213+
dbs.metadata.clearSync()
214+
dbs.indexes.clearSync()
191215
})
192216
break
193217
}
@@ -229,8 +253,8 @@ function updateDataStore(action: ActionsUnion): void {
229253
function clearIndexes(): void {
230254
const dbs = getDatabases()
231255
dbs.nodes.transactionSync(() => {
232-
dbs.metadata.clear()
233-
dbs.indexes.clear()
256+
dbs.metadata.clearSync()
257+
dbs.indexes.clearSync()
234258
})
235259
}
236260

‎packages/gatsby/src/datastore/lmdb/updates/nodes.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { ActionsUnion, IGatsbyNode } from "../../../redux/types"
2-
import { Database } from "lmdb-store"
2+
import type { Database } from "lmdb"
33

44
type NodeId = string
55

‎packages/gatsby/src/datastore/types.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Database } from "lmdb-store"
1+
import { Database } from "lmdb"
22
import { IGatsbyNode } from "../redux/types"
33
import { GatsbyGraphQLType } from "../../index"
44
import { IInputQuery } from "./common/query"

‎packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts

+15
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,21 @@ export async function createGraphqlEngineBundle(
7070
],
7171
module: {
7272
rules: [
73+
{
74+
test: require.resolve(`lmdb`),
75+
parser: { amd: false },
76+
use: [
77+
{
78+
loader: require.resolve(`@vercel/webpack-asset-relocator-loader`),
79+
options: {
80+
outputAssetBase: `assets`,
81+
},
82+
},
83+
{
84+
loader: require.resolve(`./lmdb-bundling-patch`),
85+
},
86+
],
87+
},
7388
{
7489
test: /\.m?js$/,
7590
type: `javascript/auto`,

‎packages/gatsby/src/schema/graphql-engine/lmdb-bundling-patch.ts

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { createRequireFromPath } from "gatsby-core-utils"
2+
3+
// This is hacky webpack loader that does string replacements to
4+
// allow lmdb@2 to be bundled by webpack for engines.
5+
// Currently `@vercel/webpack-asset-relocator-loader doesn't handle
6+
// the way lmdb is loading binaries and dictionary file
7+
// (can't statically analyze it). So we perform few localized changes
8+
// and we replace dynamic values with hardcoded ones to allow
9+
// asset-relocator to pick those assets up and handle them.
10+
//
11+
// Because lmdb code can diverge, we also pin version in gatsby
12+
// dependencies and will have manually bump it (with renovate most likely).
13+
//
14+
// To solve this upstream few things would need to change:
15+
// - https://github.com/DoctorEvidence/lmdb-js/blob/544b3fda402f24a70a0e946921e4c9134c5adf85/node-index.js#L14-L16
16+
// - https://github.com/DoctorEvidence/lmdb-js/blob/544b3fda402f24a70a0e946921e4c9134c5adf85/open.js#L77
17+
// Reliance on `import.meta.url` + usage of `.replace` is what seems to cause problems currently.
18+
19+
export default function (source: string): string {
20+
let lmdbBinaryLocation
21+
try {
22+
const lmdbRequire = createRequireFromPath(require.resolve(`lmdb`))
23+
const nodeGypBuild = lmdbRequire(`node-gyp-build`)
24+
const path = require(`path`)
25+
26+
lmdbBinaryLocation = nodeGypBuild.path(
27+
path.dirname(require.resolve(`lmdb`)).replace(`/dist`, ``)
28+
)
29+
} catch (e) {
30+
return source
31+
}
32+
33+
return source
34+
.replace(
35+
`require$1('node-gyp-build')(dirName)`,
36+
`require(${JSON.stringify(lmdbBinaryLocation)})`
37+
)
38+
.replace(
39+
`require$2.resolve('./dict/dict.txt')`,
40+
`require.resolve('../dict/dict.txt')`
41+
)
42+
.replace(
43+
/fs\.readFileSync\(new URL\('\.\/dict\/dict\.txt',\s*\(typeof\s*document\s*===\s*'undefined'\s*\?\s*new\s*\(require\('u'\s*\+\s*'rl'\)\.URL\)\s*\('file:'\s*\+\s*__filename\).href\s*:\s*\(document\.currentScript\s*&&\s*document\.currentScript\.src\s*\|\|\s*new URL\('index\.cjs',\s*document\.baseURI\)\.href\)\)\.replace\(\/dist\[\\\\\\\/\]index\.cjs\$\/,\s*''\)\)\)/g,
44+
`fs.readFileSync(require.resolve('../dict/dict.txt'))`
45+
)
46+
}

‎packages/gatsby/src/utils/cache-lmdb.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { open, RootDatabase, Database, DatabaseOptions } from "lmdb-store"
1+
import { open, RootDatabase, Database, DatabaseOptions } from "lmdb"
22
import fs from "fs-extra"
33
import path from "path"
44

‎yarn.lock

+19-20
Original file line numberDiff line numberDiff line change
@@ -14583,17 +14583,16 @@ livereload-js@^2.3.0:
1458314583
version "2.3.0"
1458414584
resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.3.0.tgz#c3ab22e8aaf5bf3505d80d098cbad67726548c9a"
1458514585

14586-
lmdb-store@^1.6.11:
14587-
version "1.6.11"
14588-
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d"
14589-
integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q==
14586+
lmdb@2.2.1:
14587+
version "2.2.1"
14588+
resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.2.1.tgz#b7fd22ed2268ab74aa71108b793678314a7b94bb"
14589+
integrity sha512-tUlIjyJvbd4mqdotI9Xe+3PZt/jqPx70VKFDrKMYu09MtBWOT3y2PbuTajX+bJFDjbgLkQC0cTx2n6dithp/zQ==
1459014590
dependencies:
14591+
msgpackr "^1.5.4"
1459114592
nan "^2.14.2"
1459214593
node-gyp-build "^4.2.3"
14593-
ordered-binary "^1.0.0"
14594-
weak-lru-cache "^1.0.0"
14595-
optionalDependencies:
14596-
msgpackr "^1.4.7"
14594+
ordered-binary "^1.2.4"
14595+
weak-lru-cache "^1.2.2"
1459714596

1459814597
load-bmfont@^1.3.1, load-bmfont@^1.4.0:
1459914598
version "1.4.0"
@@ -16222,10 +16221,10 @@ msgpackr-extract@^1.0.14:
1622216221
nan "^2.14.2"
1622316222
node-gyp-build "^4.2.3"
1622416223

16225-
msgpackr@^1.4.7:
16226-
version "1.4.7"
16227-
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.4.7.tgz#d802ade841e7d2e873000b491cdda6574a3d5748"
16228-
integrity sha512-bhC8Ed1au3L3oHaR/fe4lk4w7PLGFcWQ5XY/Tk9N6tzDRz8YndjCG68TD8zcvYZoxNtw767eF/7VpaTpU9kf9w==
16224+
msgpackr@^1.5.4:
16225+
version "1.5.4"
16226+
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.5.4.tgz#2b6ea6cb7d79c0ad98fc76c68163c48eda50cf0d"
16227+
integrity sha512-Z7w5Jg+2Q9z9gJxeM68d7tSuWZZGnFIRhZnyqcZCa/1dKkhOCNvR1TUV3zzJ3+vj78vlwKRzUgVDlW4jiSOeDA==
1622916228
optionalDependencies:
1623016229
msgpackr-extract "^1.0.14"
1623116230

@@ -17121,10 +17120,10 @@ ora@^5.4.1:
1712117120
strip-ansi "^6.0.0"
1712217121
wcwidth "^1.0.1"
1712317122

17124-
ordered-binary@^1.0.0:
17125-
version "1.1.3"
17126-
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.1.3.tgz#11dbc0a4cb7f8248183b9845e031b443be82571e"
17127-
integrity sha512-tDTls+KllrZKJrqRXUYJtIcWIyoQycP7cVN7kzNNnhHKF2bMKHflcAQK+pF2Eb1iVaQodHxqZQr0yv4HWLGBhQ==
17123+
ordered-binary@^1.2.4:
17124+
version "1.2.4"
17125+
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.2.4.tgz#51d3a03af078a0bdba6c7bc8f4fedd1f5d45d83e"
17126+
integrity sha512-A/csN0d3n+igxBPfUrjbV5GC69LWj2pjZzAAeeHXLukQ4+fytfP4T1Lg0ju7MSPSwq7KtHkGaiwO8URZN5IpLg==
1712817127

1712917128
ordered-read-streams@^1.0.0:
1713017129
version "1.0.1"
@@ -24527,10 +24526,10 @@ wcwidth@^1.0.0, wcwidth@^1.0.1:
2452724526
dependencies:
2452824527
defaults "^1.0.3"
2452924528

24530-
weak-lru-cache@^1.0.0:
24531-
version "1.1.2"
24532-
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.1.2.tgz#a909a97372aabdfbfe3eb33580af255b3b198834"
24533-
integrity sha512-Bi5ae8Bev3YulgtLTafpmHmvl3vGbanRkv+qqA2AX8c3qj/MUdvSuaHq7ukDYBcMDINIaRPTPEkXSNCqqWivuA==
24529+
weak-lru-cache@^1.2.2:
24530+
version "1.2.2"
24531+
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
24532+
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==
2453424533

2453524534
web-namespaces@^1.0.0:
2453624535
version "1.1.2"

0 commit comments

Comments
 (0)