feat(gatsby-core-utils): Add hashing methods from hash-wasm (#37433)

* add dep to core-utils * add md5-file function * add hash re-exports * misc todo comment * add missing export * use new utils * remove dep from gatsby * update tests * remove not needed todo * Restore packages/gatsby-core-utils/src/create-content-digest.ts * update readme
gatsbyjs · Jan 16, 2023 · df27ff4 · df27ff4
1 parent 2b24aec
commit df27ff4
Show file tree

Hide file tree

Showing 15 changed files with 77 additions and 56 deletions.
diff --git a/packages/gatsby-core-utils/README.md b/packages/gatsby-core-utils/README.md
@@ -121,3 +121,17 @@ await fs.writeFile("pathToFile", "my custom content")
 
 await mutex.release()
 ```
+
+### Hashing
+
+Parts of [`hash-wasm`](https://github.com/Daninet/hash-wasm) are re-exported from `gatsby-core-utils` or used in custom functions. When working on hashing where you'd normally use `crypto` from Node.js, you can use these functions instead. They especially show their advantage on large inputs so don't feel obliged to _always_ use them. Refer to `hash-wasm`'s documentation for more details on usage for the re-exported functions.
+
+```js
+const { md5File, md5, createMD5, sha256, sha1 } = require("gatsby-core-utils")
+
+// For md5, createMD5, sha256, sha1 refer to hash-wasm
+await md5(`some-string`)
+
+// md5File gives you the MD5 hex hash for a given filepath
+await md5File(`package.json`)
+```
diff --git a/packages/gatsby-core-utils/package.json b/packages/gatsby-core-utils/package.json
@@ -68,6 +68,7 @@
     "file-type": "^16.5.3",
     "fs-extra": "^11.1.0",
     "got": "^11.8.5",
+    "hash-wasm": "^4.9.0",
     "import-from": "^4.0.0",
     "lmdb": "2.5.3",
     "lock": "^1.1.0",

diff --git a/packages/gatsby-core-utils/src/hash.ts b/packages/gatsby-core-utils/src/hash.ts
@@ -0,0 +1 @@
+export { md5, createMD5, sha256, sha1 } from "hash-wasm"
diff --git a/packages/gatsby-core-utils/src/index.ts b/packages/gatsby-core-utils/src/index.ts
@@ -21,5 +21,7 @@ export { createFilePath } from "./filename-utils"
 export { readConfigFile, getConfigPath } from "./utils"
 export { lock } from "./lock"
 export { murmurhash } from "./murmurhash"
+export * from "./hash"
+export { md5File } from "./md5-file"
 
 export type { IFetchRemoteFileOptions } from "./fetch-remote-file"
diff --git a/packages/gatsby-core-utils/src/md5-file.ts b/packages/gatsby-core-utils/src/md5-file.ts
@@ -0,0 +1,29 @@
+import { createMD5 } from "hash-wasm"
+import * as fs from "fs-extra"
+
+/**
+ * Create a MD5 hash from a given filePath
+ * @param filePath Absolute path to the file
+ * @returns MD5 hash in hex format
+ */
+export const md5File = async (filePath: string): Promise<string> => {
+  const md5hasher = await createMD5()
+
+  return new Promise((resolve, reject) => {
+    md5hasher.init()
+
+    const fileInput = fs.createReadStream(filePath)
+
+    fileInput.on(`error`, err => {
+      reject(err)
+    })
+
+    fileInput.on(`data`, data => {
+      md5hasher.update(data)
+    })
+
+    fileInput.on(`end`, () => {
+      resolve(md5hasher.digest(`hex`))
+    })
+  })
+}
diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
@@ -120,7 +120,6 @@
     "latest-version": "^7.0.0",
     "lmdb": "2.5.3",
     "lodash": "^4.17.21",
-    "md5-file": "^5.0.0",
     "meant": "^1.0.3",
     "memoizee": "^0.4.15",
     "micromatch": "^4.0.5",

diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts
@@ -6,6 +6,7 @@ export function createFileContentHash(
   root: string,
   globPattern: string
 ): string {
+  // TODO: Use hash-wasm
   const hash = crypto.createHash(`md5`)
   const files = glob.sync(`${root}/${globPattern}`, { nodir: true })
 

diff --git a/packages/gatsby/src/bootstrap/redirects-writer.ts b/packages/gatsby/src/bootstrap/redirects-writer.ts
@@ -1,10 +1,9 @@
 import _ from "lodash"
-import crypto from "crypto"
 import fs from "fs-extra"
+import { joinPath, md5 } from "gatsby-core-utils"
+import reporter from "gatsby-cli/lib/reporter"
 import { store, emitter } from "../redux"
 import { IRedirect } from "../redux/types"
-import { joinPath } from "gatsby-core-utils"
-import reporter from "gatsby-cli/lib/reporter"
 
 let lastHash: string | null = null
 let bootstrapFinished = false
@@ -53,10 +52,7 @@ export const writeRedirects = async (): Promise<void> => {
     )
   }
 
-  const newHash = crypto
-    .createHash(`md5`)
-    .update(JSON.stringify(browserRedirects))
-    .digest(`hex`)
+  const newHash = await md5(JSON.stringify(browserRedirects))
 
   if (newHash === lastHash) {
     return

diff --git a/packages/gatsby/src/bootstrap/requires-writer.ts b/packages/gatsby/src/bootstrap/requires-writer.ts
@@ -1,11 +1,9 @@
 import _ from "lodash"
 import path from "path"
 import fs from "fs-extra"
-import crypto from "crypto"
-import { slash } from "gatsby-core-utils"
 import reporter from "gatsby-cli/lib/reporter"
 import { match } from "@gatsbyjs/reach-router"
-import { joinPath } from "gatsby-core-utils"
+import { joinPath, md5, slash } from "gatsby-core-utils"
 import { store, emitter } from "../redux/"
 import { IGatsbyState, IGatsbyPage, IGatsbySlice } from "../redux/types"
 import {
@@ -177,22 +175,6 @@ const getMatchPaths = (
     })
 }
 
-const createHash = (
-  matchPaths: Array<IGatsbyPageMatchPath>,
-  components: Array<IGatsbyPageComponent>,
-  cleanedSSRVisitedPageComponents: Array<IGatsbyPageComponent>
-): string =>
-  crypto
-    .createHash(`md5`)
-    .update(
-      JSON.stringify({
-        matchPaths,
-        components,
-        cleanedSSRVisitedPageComponents,
-      })
-    )
-    .digest(`hex`)
-
 // Write out pages information.
 export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
   const { program, slices } = state
@@ -212,10 +194,12 @@ export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
     )
   }
 
-  const newHash = createHash(
-    matchPaths,
-    components,
-    cleanedSSRVisitedPageComponents
+  const newHash = await md5(
+    JSON.stringify({
+      matchPaths,
+      components,
+      cleanedSSRVisitedPageComponents,
+    })
   )
 
   if (newHash === lastHash) {

diff --git a/packages/gatsby/src/query/file-parser.js b/packages/gatsby/src/query/file-parser.js
@@ -23,7 +23,7 @@ const report = require(`gatsby-cli/lib/reporter`)
 import type { DocumentNode } from "graphql"
 import { babelParseToAst } from "../utils/babel-parse-to-ast"
 import { codeFrameColumns } from "@babel/code-frame"
-import { getPathToLayoutComponent } from "gatsby-core-utils"
+import { getPathToLayoutComponent, md5 } from "gatsby-core-utils"
 
 const apiRunnerNode = require(`../utils/api-runner-node`)
 const { actions } = require(`../redux/actions`)
@@ -524,11 +524,7 @@ export default class FileParser {
       return null
     }
 
-    const hash = crypto
-      .createHash(`md5`)
-      .update(file)
-      .update(text)
-      .digest(`hex`)
+    const hash = await md5(file + text)
 
     try {
       if (!cache[hash]) {

diff --git a/packages/gatsby/src/query/graphql-runner.ts b/packages/gatsby/src/query/graphql-runner.ts
@@ -1,4 +1,3 @@
-import crypto from "crypto"
 import { Span } from "opentracing"
 import {
   parse,
@@ -14,8 +13,9 @@ import {
 } from "graphql"
 import { debounce } from "lodash"
 import reporter from "gatsby-cli/lib/reporter"
-import { createPageDependency } from "../redux/actions/add-page-dependency"
+import { sha1 } from "gatsby-core-utils/hash"
 
+import { createPageDependency } from "../redux/actions/add-page-dependency"
 import withResolverContext from "../schema/context"
 import { LocalNodeModel } from "../schema/node-model"
 import { Store } from "redux"
@@ -210,9 +210,9 @@ export class GraphQLRunner {
     if (this.stats) {
       this.stats.totalQueries++
 
-      this.stats.uniqueQueries.add(
-        crypto.createHash(`sha1`).update(queryText).digest(`hex`)
-      )
+      const hash = await sha1(queryText)
+
+      this.stats.uniqueQueries.add(hash)
     }
 
     const { errors, warnings, document } = this.validate(

diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts
@@ -2,8 +2,8 @@ import { Span } from "opentracing"
 import _ from "lodash"
 import fs from "fs-extra"
 import report from "gatsby-cli/lib/reporter"
-import crypto from "crypto"
 import { ExecutionResult, GraphQLError } from "graphql"
+import { sha1 } from "gatsby-core-utils/hash"
 
 import path from "path"
 import { store } from "../redux"
@@ -172,10 +172,7 @@ export async function queryRunner(
   }
 
   const resultJSON = JSON.stringify(result)
-  const resultHash = crypto
-    .createHash(`sha1`)
-    .update(resultJSON)
-    .digest(`base64`)
+  const resultHash = await sha1(resultJSON)
 
   const resultHashCache = getResultHashCache()
 

diff --git a/packages/gatsby/src/services/initialize.ts b/packages/gatsby/src/services/initialize.ts
@@ -2,8 +2,7 @@ import _ from "lodash"
 import { slash, isCI } from "gatsby-core-utils"
 import * as fs from "fs-extra"
 import { releaseAllMutexes } from "gatsby-core-utils/mutex"
-import md5File from "md5-file"
-import crypto from "crypto"
+import { md5, md5File } from "gatsby-core-utils"
 import path from "path"
 import telemetry from "gatsby-telemetry"
 import glob from "globby"
@@ -338,10 +337,7 @@ export async function initialize({
     )
   )
 
-  const pluginsHash = crypto
-    .createHash(`md5`)
-    .update(JSON.stringify(pluginVersions.concat(hashes)))
-    .digest(`hex`)
+  const pluginsHash = await md5(JSON.stringify(pluginVersions.concat(hashes)))
 
   const oldPluginsHash = state && state.status ? state.status.PLUGINS_HASH : ``
 

diff --git a/packages/gatsby/src/utils/worker/__tests__/queries.ts b/packages/gatsby/src/utils/worker/__tests__/queries.ts
@@ -384,7 +384,7 @@ describe(`worker (queries)`, () => {
           queryType: `static`,
           path: `sq--q1`,
           queryHash: `q1-hash`,
-          resultHash: `Dr5hgCDB+R0S9oRBWeZYj3lB7VI=`,
+          resultHash: `0ebe618020c1f91d12f6844159e6588f7941ed52`,
         },
         type: `PAGE_QUERY_RUN`,
       },
@@ -415,7 +415,7 @@ describe(`worker (queries)`, () => {
           componentPath: `/foo.js`,
           queryType: `page`,
           path: `/foo`,
-          resultHash: `8dW7PoqwZNk/0U8LO6kTj1qBCwU=`,
+          resultHash: `f1d5bb3e8ab064d93fd14f0b3ba9138f5a810b05`,
         },
         type: `PAGE_QUERY_RUN`,
       },
@@ -430,7 +430,7 @@ describe(`worker (queries)`, () => {
           componentPath: `/bar.js`,
           queryType: `page`,
           path: `/bar`,
-          resultHash: `iKmhf9XgbsfK7qJw0tw95pmGwJM=`,
+          resultHash: `88a9a17fd5e06ec7caeea270d2dc3de69986c093`,
         },
         type: `PAGE_QUERY_RUN`,
       },

diff --git a/yarn.lock b/yarn.lock
@@ -12219,6 +12219,11 @@ has@^1.0.0, has@^1.0.1, has@^1.0.3:
   dependencies:
     function-bind "^1.1.1"
 
+hash-wasm@^4.9.0:
+  version "4.9.0"
+  resolved "https://registry.yarnpkg.com/hash-wasm/-/hash-wasm-4.9.0.tgz#7e9dcc9f7d6bd0cc802f2a58f24edce999744206"
+  integrity sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w==
+
 hasha@^5.2.2:
   version "5.2.2"
   resolved "https://registry.yarnpkg.com/hasha/-/hasha-5.2.2.tgz#a48477989b3b327aea3c04f53096d816d97522a1"