feat(gatsby): use webpack fs caching in prod behind feature flag (#30857

)

* tmp

* add infrastructure logging for webpack caching

* add hooks

* no weird import

* wait for webpack fs caching to finish

* handle build-html

* fix develop (?)

* adjust artifcats ssr to match structure of browser so things render properly in browser

* clean webpack config

* add test cases for changing webpack plugins and cache invalidation

* add parentspans to activities

* enable fs caching behind feature flag

* show notice about persistent webpack caching when building js bundle takes longer than 30s

* bump minimum webpack to contain fix for resolving paths containing #

integration-tests/artifacts/__tests__/index.js

@@ -170,7 +170,7 @@ function assertHTMLCorrectness(runNumber) {
         : `component---src-templates-deps-page-query-alternative-js`
 
     const expectedBackground =
-      runNumber < 6 ? `white` : runNumber === 6 ? `yellow` : `green`
+      runNumber < 8 ? `white` : runNumber === 8 ? `yellow` : `green`
 
     let pageDataContent
     let htmlContent
@@ -264,6 +264,50 @@ function assertHTMLCorrectness(runNumber) {
       })
     })
   })
+
+  describe(`/webpack/local-plugin-1/`, () => {
+    let htmlContent
+    beforeAll(() => {
+      htmlContent = fs.readFileSync(
+        path.join(
+          process.cwd(),
+          `public`,
+          `webpack`,
+          `local-plugin-1`,
+          `index.html`
+        ),
+        `utf-8`
+      )
+    })
+
+    it(`webpack plugin is used correctly (webpack cache gets invalidate on plugin change)`, () => {
+      expect(htmlContent).toContain(
+        runNumber < 6 ? `localWebpackPlugin1_1` : `localWebpackPlugin1_2`
+      )
+    })
+  })
+
+  describe(`/webpack/local-plugin-2/`, () => {
+    let htmlContent
+    beforeAll(() => {
+      htmlContent = fs.readFileSync(
+        path.join(
+          process.cwd(),
+          `public`,
+          `webpack`,
+          `local-plugin-2`,
+          `index.html`
+        ),
+        `utf-8`
+      )
+    })
+
+    it(`webpack plugin is used correctly (webpack cache gets invalidate on plugin change)`, () => {
+      expect(htmlContent).toContain(
+        runNumber < 7 ? `localWebpackPlugin2_1` : `localWebpackPlugin2_2`
+      )
+    })
+  })
 }
 
 function assertNodeCorrectness(runNumber) {
@@ -860,7 +904,7 @@ describe(`Fifth run (.cache is deleted but public isn't)`, () => {
   assertNodeCorrectness(runNumber)
 })
 
-describe(`Sixth run (ssr-only change - only ssr compilation hash changes)`, () => {
+describe(`Sixth run (change webpack plugin variant 1 - invalidate webpack cache)`, () => {
   const runNumber = 6
 
   const expectedPages = [
@@ -877,6 +921,203 @@ describe(`Sixth run (ssr-only change - only ssr compilation hash changes)`, () =
     `/page-query-dynamic-5/`,
   ]
 
+  let changedFileOriginalContent
+  const changedFileAbspath = path.join(
+    process.cwd(),
+    `plugins`,
+    `gatsby-plugin-webpack-1`,
+    `local-webpack-plugin.js`
+  )
+
+  beforeAll(async () => {
+    // make change to used webpack plugin
+    changedFileOriginalContent = fs.readFileSync(changedFileAbspath, `utf-8`)
+    filesToRevert[changedFileAbspath] = changedFileOriginalContent
+
+    const newContent = changedFileOriginalContent.replace(
+      `localWebpackPlugin1_1`,
+      `localWebpackPlugin1_2`
+    )
+
+    if (newContent === changedFileOriginalContent) {
+      throw new Error(`Test setup failed`)
+    }
+
+    fs.writeFileSync(changedFileAbspath, newContent)
+    await runGatsbyWithRunTestSetup(runNumber)()
+  })
+
+  describe(`html files`, () => {
+    const type = `html`
+
+    describe(`should have expected html files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: expectedPages,
+        type,
+        shouldExist: true,
+      })
+    })
+
+    describe(`shouldn't have unexpected html files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: unexpectedPages,
+        type,
+        shouldExist: false,
+      })
+    })
+
+    it(`should recreate all html files`, () => {
+      expect(manifest[runNumber].generated.sort()).toEqual(
+        manifest[runNumber].allPages.sort()
+      )
+    })
+  })
+
+  describe(`page-data files`, () => {
+    const type = `page-data`
+
+    describe(`should have expected page-data files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: expectedPages,
+        type,
+        shouldExist: true,
+      })
+    })
+
+    describe(`shouldn't have unexpected page-data files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: unexpectedPages,
+        type,
+        shouldExist: false,
+      })
+    })
+  })
+
+  // Sixth run - webpack plugin changed - both ssr and browser bundles should be invalidated
+  assertWebpackBundleChanges({ browser: true, ssr: true, runNumber })
+
+  assertHTMLCorrectness(runNumber)
+
+  assertNodeCorrectness(runNumber)
+})
+
+describe(`Seventh run (change webpack plugin variant 2 - invalidate webpack cache)`, () => {
+  const runNumber = 7
+
+  const expectedPages = [
+    `/stale-pages/only-not-in-first`,
+    `/page-query-dynamic-7/`,
+  ]
+
+  const unexpectedPages = [
+    `/stale-pages/only-in-first/`,
+    `/page-query-dynamic-1/`,
+    `/page-query-dynamic-2/`,
+    `/page-query-dynamic-3/`,
+    `/page-query-dynamic-4/`,
+    `/page-query-dynamic-5/`,
+    `/page-query-dynamic-6/`,
+  ]
+
+  let changedFileOriginalContent
+  const changedFileAbspath = path.join(
+    process.cwd(),
+    `plugins`,
+    `gatsby-plugin-webpack-2`,
+    `on-create-webpack-config.js`
+  )
+
+  beforeAll(async () => {
+    // make change to used webpack plugin
+    changedFileOriginalContent = fs.readFileSync(changedFileAbspath, `utf-8`)
+    filesToRevert[changedFileAbspath] = changedFileOriginalContent
+
+    const newContent = changedFileOriginalContent.replace(
+      `localWebpackPlugin2_1`,
+      `localWebpackPlugin2_2`
+    )
+
+    if (newContent === changedFileOriginalContent) {
+      throw new Error(`Test setup failed`)
+    }
+
+    fs.writeFileSync(changedFileAbspath, newContent)
+    await runGatsbyWithRunTestSetup(runNumber)()
+  })
+
+  describe(`html files`, () => {
+    const type = `html`
+
+    describe(`should have expected html files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: expectedPages,
+        type,
+        shouldExist: true,
+      })
+    })
+
+    describe(`shouldn't have unexpected html files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: unexpectedPages,
+        type,
+        shouldExist: false,
+      })
+    })
+
+    it(`should recreate all html files`, () => {
+      expect(manifest[runNumber].generated.sort()).toEqual(
+        manifest[runNumber].allPages.sort()
+      )
+    })
+  })
+
+  describe(`page-data files`, () => {
+    const type = `page-data`
+
+    describe(`should have expected page-data files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: expectedPages,
+        type,
+        shouldExist: true,
+      })
+    })
+
+    describe(`shouldn't have unexpected page-data files`, () => {
+      assertFileExistenceForPagePaths({
+        pagePaths: unexpectedPages,
+        type,
+        shouldExist: false,
+      })
+    })
+  })
+
+  // Seventh run - webpack plugin changed - both ssr and browser bundles should be invalidated
+  assertWebpackBundleChanges({ browser: true, ssr: true, runNumber })
+
+  assertHTMLCorrectness(runNumber)
+
+  assertNodeCorrectness(runNumber)
+})
+
+describe(`Eight run (ssr-only change - only ssr compilation hash changes)`, () => {
+  const runNumber = 8
+
+  const expectedPages = [
+    `/stale-pages/only-not-in-first`,
+    `/page-query-dynamic-8/`,
+  ]
+
+  const unexpectedPages = [
+    `/stale-pages/only-in-first/`,
+    `/page-query-dynamic-1/`,
+    `/page-query-dynamic-2/`,
+    `/page-query-dynamic-3/`,
+    `/page-query-dynamic-4/`,
+    `/page-query-dynamic-5/`,
+    `/page-query-dynamic-6/`,
+    `/page-query-dynamic-7/`,
+  ]
+
   let changedFileOriginalContent
   const changedFileAbspath = path.join(process.cwd(), `gatsby-ssr.js`)
 
@@ -944,20 +1185,20 @@ describe(`Sixth run (ssr-only change - only ssr compilation hash changes)`, () =
     })
   })
 
-  // Sixth run - only ssr bundle should change as only file used by ssr was changed
+  // Eight run - only ssr bundle should change as only file used by ssr was changed
   assertWebpackBundleChanges({ browser: false, ssr: true, runNumber })
 
   assertHTMLCorrectness(runNumber)
 
   assertNodeCorrectness(runNumber)
 })
 
-describe(`Seventh run (no change in any file that is bundled, we change untracked file, but previous build used unsafe method so all should rebuild)`, () => {
-  const runNumber = 7
+describe(`Ninth run (no change in any file that is bundled, we change untracked file, but previous build used unsafe method so all should rebuild)`, () => {
+  const runNumber = 9
 
   const expectedPages = [
     `/stale-pages/only-not-in-first`,
-    `/page-query-dynamic-7/`,
+    `/page-query-dynamic-9/`,
   ]
 
   const unexpectedPages = [
@@ -968,6 +1209,8 @@ describe(`Seventh run (no change in any file that is bundled, we change untracke
     `/page-query-dynamic-4/`,
     `/page-query-dynamic-5/`,
     `/page-query-dynamic-6/`,
+    `/page-query-dynamic-7/`,
+    `/page-query-dynamic-8/`,
   ]
 
   let changedFileOriginalContent
@@ -1034,7 +1277,7 @@ describe(`Seventh run (no change in any file that is bundled, we change untracke
     })
   })
 
-  // Seventh run - no bundle should change as we don't change anything that IS bundled
+  // Ninth run - no bundle should change as we don't change anything that IS bundled
   assertWebpackBundleChanges({ browser: false, ssr: false, runNumber })
 
   assertHTMLCorrectness(runNumber)

integration-tests/artifacts/gatsby-config.js

@@ -6,5 +6,5 @@ module.exports = {
     github: `sidharthachatterjee`,
     moreInfo: `Sid is amazing`,
   },
-  plugins: [],
+  plugins: [`gatsby-plugin-webpack-1`, `gatsby-plugin-webpack-2`],
 }

integration-tests/artifacts/gatsby-ssr.js

@@ -10,3 +10,23 @@ export function onRenderBody({ setHeadComponents }) {
     />
   )
 }
+
+export function wrapPageElement({ element }) {
+  return (
+    <>
+      <h1>(to match gatsby-browser structure without using same imports)</h1>
+      {element}
+    </>
+  )
+}
+
+export function wrapRootElement({ element }) {
+  return (
+    <>
+      <div>
+        (to match src/components/github.js structure without using same imports)
+      </div>
+      {element}
+    </>
+  )
+}

integration-tests/artifacts/plugins/gatsby-plugin-webpack-1/gatsby-node.js

@@ -0,0 +1,7 @@
+const MyLocalPlugin = require(`./local-webpack-plugin`)
+
+exports.onCreateWebpackConfig = ({ actions }) => {
+  actions.setWebpackConfig({
+    plugins: [new MyLocalPlugin()],
+  })
+}

integration-tests/artifacts/plugins/gatsby-plugin-webpack-1/local-webpack-plugin.js

@@ -0,0 +1,13 @@
+const { DefinePlugin } = require(`webpack`)
+
+module.exports = class LocalWebpackPlugin1 {
+  constructor() {
+    this.plugin = new DefinePlugin({
+      REPLACE_ME_1: JSON.stringify(`localWebpackPlugin1_1`),
+    })
+  }
+
+  apply(compiler) {
+    this.plugin.apply.call(this.plugin, compiler)
+  }
+}

integration-tests/artifacts/plugins/gatsby-plugin-webpack-1/package.json

@@ -0,0 +1 @@
+{}

integration-tests/artifacts/plugins/gatsby-plugin-webpack-2/gatsby-node.js

@@ -0,0 +1 @@
+exports.onCreateWebpackConfig = require(`./on-create-webpack-config`)

integration-tests/artifacts/plugins/gatsby-plugin-webpack-2/on-create-webpack-config.js

@@ -0,0 +1,19 @@
+const { DefinePlugin } = require(`webpack`)
+
+class LocalWebpackPlugin2 {
+  constructor() {
+    this.plugin = new DefinePlugin({
+      REPLACE_ME_2: JSON.stringify(`localWebpackPlugin2_1`),
+    })
+  }
+
+  apply(compiler) {
+    this.plugin.apply.call(this.plugin, compiler)
+  }
+}
+
+module.exports = ({ actions }) => {
+  actions.setWebpackConfig({
+    plugins: [new LocalWebpackPlugin2()],
+  })
+}

integration-tests/artifacts/plugins/gatsby-plugin-webpack-2/package.json

@@ -0,0 +1 @@
+{}