fix(gatsby): use lmdb for resultHash cache so shared across workers (#…

…34925) * use lmdb for resultHash cache so shared across workers * Clear lmdb before workers start processes queries * Pull db name into const * Undo reset changes, as it's not actually needed * Undo accidental e2e change * Add to pagepaths if component path/id was modified * Update snapshots * Move lmdb creation into function * require -> import Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com> Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>
gatsbyjs · Mar 11, 2022 · a5cd72a · a5cd72a
1 parent c0f394d
commit a5cd72a
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 4 deletions.
diff --git a/packages/gatsby/src/query/__tests__/data-tracking.js b/packages/gatsby/src/query/__tests__/data-tracking.js
@@ -51,6 +51,7 @@ jest.mock(`../../utils/cache-lmdb`, () => {
       get = jest.fn(() => Promise.resolve())
       set = jest.fn(() => Promise.resolve())
     },
+    __esModule: true,
   }
 })
 

diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts
@@ -14,8 +14,18 @@ import errorParser from "./error-parser"
 import { GraphQLRunner } from "./graphql-runner"
 import { IExecutionResult, PageContext } from "./types"
 import { pageDataExists, savePageQueryResult } from "../utils/page-data"
-
-const resultHashes = new Map()
+import GatsbyCacheLmdb from "../utils/cache-lmdb"
+
+let resultHashCache: GatsbyCacheLmdb | undefined
+function getResultHashCache(): GatsbyCacheLmdb {
+  if (!resultHashCache) {
+    resultHashCache = new GatsbyCacheLmdb({
+      name: `query-result-hashes`,
+      encoding: `string`,
+    }).init()
+  }
+  return resultHashCache
+}
 
 export interface IQueryJob {
   id: string
@@ -171,12 +181,13 @@ export async function queryRunner(
     .update(resultJSON)
     .digest(`base64`)
 
+  const resultHashCache = getResultHashCache()
   if (
-    resultHash !== resultHashes.get(queryJob.id) ||
+    resultHash !== (await resultHashCache.get(queryJob.id)) ||
     (queryJob.isPage &&
       !pageDataExists(path.join(program.directory, `public`), queryJob.id))
   ) {
-    resultHashes.set(queryJob.id, resultHash)
+    await resultHashCache.set(queryJob.id, resultHash)
 
     if (queryJob.isPage) {
       // We need to save this temporarily in cache because

diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap
@@ -80,6 +80,7 @@ Map {
 
 exports[`Add pages allows you to add pages 1`] = `
 Object {
+  "componentModified": false,
   "contextModified": false,
   "payload": Object {
     "component": "/whatever/index.js",
@@ -124,6 +125,7 @@ Map {
 
 exports[`Add pages allows you to add pages with context 1`] = `
 Object {
+  "componentModified": false,
   "contextModified": false,
   "payload": Object {
     "component": "/whatever/index.js",
@@ -172,6 +174,7 @@ Map {
 
 exports[`Add pages allows you to add pages with matchPath 1`] = `
 Object {
+  "componentModified": false,
   "contextModified": false,
   "payload": Object {
     "component": "/whatever/index.js",

diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js
@@ -429,6 +429,8 @@ ${reservedFields.map(f => `  * "${f}"`).join(`\n`)}
   const oldPage: Page = store.getState().pages.get(internalPage.path)
   const contextModified =
     !!oldPage && !_.isEqual(oldPage.context, internalPage.context)
+  const componentModified =
+    !!oldPage && !_.isEqual(oldPage.component, internalPage.component)
 
   const alternateSlashPath = page.path.endsWith(`/`)
     ? page.path.slice(0, -1)
@@ -496,6 +498,7 @@ ${reservedFields.map(f => `  * "${f}"`).join(`\n`)}
       ...actionOptions,
       type: `CREATE_PAGE`,
       contextModified,
+      componentModified,
       plugin,
       payload: internalPage,
     },

diff --git a/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts b/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts
@@ -7,6 +7,12 @@ export const pendingPageDataWritesReducer = (
   action: ActionsUnion
 ): IGatsbyState["pendingPageDataWrites"] => {
   switch (action.type) {
+    case `CREATE_PAGE`:
+      if (action.componentModified) {
+        state.pagePaths.add(action.payload.path)
+      }
+
+      return state
     case `ADD_PENDING_PAGE_DATA_WRITE`:
       state.pagePaths.add(action.payload.path)
       return state
@@ -15,6 +21,7 @@ export const pendingPageDataWritesReducer = (
       for (const page of action.payload.pages) {
         state.pagePaths.add(page)
       }
+
       return state
     }
 

diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts
@@ -707,6 +707,7 @@ export interface ICreatePageAction {
   payload: IGatsbyPage
   plugin?: IGatsbyPlugin
   contextModified?: boolean
+  componentModified?: boolean
 }
 
 export interface ICreateRedirectAction {