fix(gatsby): resend pageData when socket disconnect (#26868)

Author: wardpeet

packages/gatsby/cache-dir/socketIo.js

@@ -1,23 +1,30 @@
+import io from "socket.io-client"
 import { reportError, clearError } from "./error-overlay-handler"
 import normalizePagePath from "./normalize-page-path"
 
 let socket = null
 
+const inFlightGetPageDataPromiseCache = {}
 let staticQueryData = {}
 let pageQueryData = {}
-let isInitialized = false
 
 export const getStaticQueryData = () => staticQueryData
 export const getPageQueryData = () => pageQueryData
-export const getIsInitialized = () => isInitialized
 
 export default function socketIo() {
   if (process.env.NODE_ENV !== `production`) {
     if (!socket) {
       // Try to initialize web socket if we didn't do it already
       try {
-        // eslint-disable-next-line no-undef
-        socket = io()
+        // force websocket as transport
+        socket = io({
+          transports: [`websocket`],
+        })
+
+        // when websocket fails, we'll try polling
+        socket.on(`reconnect_attempt`, () => {
+          socket.io.opts.transports = [`polling`, `websocket`]
+        })
 
         const didDataChange = (msg, queryData) => {
           const id =
@@ -30,6 +37,14 @@ export default function socketIo() {
           )
         }
 
+        socket.on(`connect`, () => {
+          // we might have disconnected so we loop over the page-data requests in flight
+          // so we can get the data again
+          Object.keys(inFlightGetPageDataPromiseCache).forEach(pathname => {
+            socket.emit(`getDataForPath`, pathname)
+          })
+        })
+
         socket.on(`message`, msg => {
           if (msg.type === `staticQueryResult`) {
             if (didDataChange(msg, staticQueryData)) {
@@ -74,7 +89,6 @@ export default function socketIo() {
   }
 }
 
-const inFlightGetPageDataPromiseCache = {}
 function getPageData(pathname) {
   pathname = normalizePagePath(pathname)
   if (inFlightGetPageDataPromiseCache[pathname]) {

packages/gatsby/src/utils/webpack.config.js

@@ -690,6 +690,12 @@ module.exports = async (
     ]
   }
 
+  if (stage === `develop`) {
+    config.externals = {
+      "socket.io-client": `io`,
+    }
+  }
+
   store.dispatch(actions.replaceWebpackConfig(config))
   const getConfig = () => store.getState().webpack
 

packages/gatsby/src/utils/websocket-manager.ts

@@ -3,7 +3,7 @@ import path from "path"
 import { store } from "../redux"
 import { Server as HTTPSServer } from "https"
 import { Server as HTTPServer } from "http"
-import fs from "fs"
+import fs from "fs-extra"
 import { readPageData, IPageDataWithQueryResult } from "../utils/page-data"
 import telemetry from "gatsby-telemetry"
 import url from "url"
@@ -25,87 +25,72 @@ type PageResultsMap = Map<string, IPageQueryResult>
 type QueryResultsMap = Map<string, IStaticQueryResult>
 
 /**
- * Get cached page query result for given page path.
+ * Get page query result for given page path.
  * @param {string} pagePath Path to a page.
  */
-const getCachedPageData = async (
-  pagePath: string
-): Promise<IPageQueryResult> => {
+async function getPageData(pagePath: string): Promise<IPageQueryResult> {
   const { program, pages } = store.getState()
   const publicDir = path.join(program.directory, `public`)
+
+  const result: IPageQueryResult = {
+    id: pagePath,
+    result: undefined,
+  }
   if (pages.has(denormalizePagePath(pagePath)) || pages.has(pagePath)) {
     try {
       const pageData: IPageDataWithQueryResult = await readPageData(
         publicDir,
         pagePath
       )
 
-      return {
-        result: pageData,
-        id: pagePath,
-      }
+      result.result = pageData
     } catch (err) {
       throw new Error(
         `Error loading a result for the page query in "${pagePath}". Query was not run and no cached result was found.`
       )
     }
   }
 
-  return {
-    id: pagePath,
-    result: undefined,
-  }
+  return result
 }
 
-const hashPaths = (
-  paths?: Array<string>
-): undefined | Array<string | undefined> => {
-  if (!paths) {
-    return undefined
+/**
+ * Get page query result for given page path.
+ * @param {string} pagePath Path to a page.
+ */
+async function getStaticQueryData(
+  staticQueryId: string
+): Promise<IStaticQueryResult> {
+  const { program } = store.getState()
+  const publicDir = path.join(program.directory, `public`)
+
+  const filePath = path.join(
+    publicDir,
+    `page-data`,
+    `sq`,
+    `d`,
+    `${staticQueryId}.json`
+  )
+
+  const result: IStaticQueryResult = {
+    id: staticQueryId,
+    result: undefined,
   }
-  return paths.map(path => {
-    if (!path) {
-      return undefined
+  if (await fs.pathExists(filePath)) {
+    try {
+      const fileResult = await fs.readJson(filePath)
+
+      result.result = fileResult
+    } catch (err) {
+      // ignore errors
     }
-    return createHash(`sha256`).update(path).digest(`hex`)
-  })
+  }
+
+  return result
 }
 
-/**
- * Get cached StaticQuery results for components that Gatsby didn't run query yet.
- * @param {QueryResultsMap} resultsMap Already stored results for queries that don't need to be read from files.
- * @param {string} directory Root directory of current project.
- */
-const getCachedStaticQueryResults = (
-  resultsMap: QueryResultsMap,
-  directory: string
-): QueryResultsMap => {
-  const cachedStaticQueryResults: QueryResultsMap = new Map()
-  const { staticQueryComponents } = store.getState()
-  staticQueryComponents.forEach(staticQueryComponent => {
-    // Don't read from file if results were already passed from query runner
-    if (resultsMap.has(staticQueryComponent.hash)) return
-    const filePath = path.join(
-      directory,
-      `public`,
-      `page-data`,
-      `sq`,
-      `d`,
-      `${staticQueryComponent.hash}.json`
-    )
-    const fileResult = fs.readFileSync(filePath, `utf-8`)
-    if (fileResult === `undefined`) {
-      console.log(
-        `Error loading a result for the StaticQuery in "${staticQueryComponent.componentPath}". Query was not run and no cached result was found.`
-      )
-      return
-    }
-    cachedStaticQueryResults.set(staticQueryComponent.hash, {
-      result: JSON.parse(fileResult),
-      id: staticQueryComponent.hash,
-    })
-  })
-  return cachedStaticQueryResults
+function hashPaths(paths: Array<string>): Array<string> {
+  return paths.map(path => createHash(`sha256`).update(path).digest(`hex`))
 }
 
 const getRoomNameFromPath = (path: string): string => `path-${path}`
@@ -119,22 +104,17 @@ export class WebsocketManager {
   websocket: socketIO.Server | undefined
 
   init = ({
-    directory,
     server,
   }: {
     directory: string
     server: HTTPSServer | HTTPServer
   }): socketIO.Server => {
-    const cachedStaticQueryResults = getCachedStaticQueryResults(
-      this.staticQueryResults,
-      directory
-    )
-    this.staticQueryResults = new Map([
-      ...this.staticQueryResults,
-      ...cachedStaticQueryResults,
-    ])
-
-    this.websocket = socketIO(server)
+    this.websocket = socketIO(server, {
+      // we see ping-pong timeouts on gatsby-cloud when socket.io is running for a while
+      // increasing it should help
+      // @see https://github.com/socketio/socket.io/issues/3259#issuecomment-448058937
+      pingTimeout: 30000,
+    })
 
     this.websocket.on(`connection`, socket => {
       let activePath: string | null = null
@@ -147,13 +127,6 @@ export class WebsocketManager {
       }
 
       this.connectedClients += 1
-      // Send already existing static query results
-      this.staticQueryResults.forEach(result => {
-        socket.send({
-          type: `staticQueryResult`,
-          payload: result,
-        })
-      })
       this.errors.forEach((message, errorID) => {
         socket.send({
           type: `overlayError`,
@@ -176,22 +149,39 @@ export class WebsocketManager {
       }
 
       const getDataForPath = async (path: string): Promise<void> => {
-        if (!this.pageResults.has(path)) {
+        let pageData = this.pageResults.get(path)
+        if (!pageData) {
           try {
-            const result = await getCachedPageData(path)
+            pageData = await getPageData(path)
 
-            this.pageResults.set(path, result)
+            this.pageResults.set(path, pageData)
           } catch (err) {
             console.log(err.message)
-
             return
           }
         }
 
+        const staticQueryHashes = pageData.result?.staticQueryHashes ?? []
+        await Promise.all(
+          staticQueryHashes.map(async queryId => {
+            let staticQueryResult = this.staticQueryResults.get(queryId)
+
+            if (!staticQueryResult) {
+              staticQueryResult = await getStaticQueryData(queryId)
+              this.staticQueryResults.set(queryId, staticQueryResult)
+            }
+
+            socket.send({
+              type: `staticQueryResult`,
+              payload: staticQueryResult,
+            })
+          })
+        )
+
         socket.send({
           type: `pageQueryResult`,
           why: `getDataForPath`,
-          payload: this.pageResults.get(path),
+          payload: pageData,
         })
 
         if (this.connectedClients > 0) {
@@ -236,6 +226,7 @@ export class WebsocketManager {
 
     if (this.websocket) {
       this.websocket.send({ type: `staticQueryResult`, payload: data })
+
       if (this.connectedClients > 0) {
         telemetry.trackCli(
           `WEBSOCKET_EMIT_STATIC_PAGE_DATA_UPDATE`,