Skip to content

Commit c50e8f2

Browse files
gatsbybotpieh
andauthoredJan 25, 2024
fix: add missing fs method rewrites to handle fetchRemoteFile in dsg/ssr engine (#38822) (#38823)
* test(adapters-e2e): enable placeholder tests in ssr remote-file * fix(gatsby-adapter-netlify): bundling file-cdn * fix(gatsby): set pathPrefix in engines * fix(gatsby): add missing fs method rewrites to handle fetchRemoteFile in dsg/ssr engine (cherry picked from commit bbdddd7) Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>
1 parent d328fd8 commit c50e8f2

File tree

6 files changed

+160
-24
lines changed

6 files changed

+160
-24
lines changed
 

‎e2e-tests/adapters/cypress/e2e/remote-file.cy.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ const configs = [
2929
{
3030
title: `remote-file (SSR, Page Query)`,
3131
pagePath: `/routes/ssr/remote-file/`,
32-
placeholders: false,
32+
placeholders: true,
3333
},
3434
]
3535

‎e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx

+14-12
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import React from "react"
44
import { GatsbyImage } from "gatsby-plugin-image"
55
import Layout from "../../../components/layout"
66

7-
const RemoteFile = ({ data }) => {
7+
const RemoteFile = ({ data, serverData }) => {
88
return (
99
<Layout>
1010
{data.allMyRemoteImage.nodes.map(node => {
@@ -54,6 +54,7 @@ const RemoteFile = ({ data }) => {
5454
</div>
5555
)
5656
})}
57+
<pre>{JSON.stringify(serverData, null, 2)}</pre>
5758
</Layout>
5859
)
5960
}
@@ -65,8 +66,7 @@ export const pageQuery = graphql`
6566
id
6667
url
6768
filename
68-
# FILE_CDN is not supported in SSR/DSG yet
69-
# publicUrl
69+
publicUrl
7070
resize(width: 100) {
7171
height
7272
width
@@ -75,23 +75,17 @@ export const pageQuery = graphql`
7575
fixed: gatsbyImage(
7676
layout: FIXED
7777
width: 100
78-
# only NONE placeholder is supported in SSR/DSG
79-
# placeholder: DOMINANT_COLOR
80-
placeholder: NONE
78+
placeholder: DOMINANT_COLOR
8179
)
8280
constrained: gatsbyImage(
8381
layout: CONSTRAINED
8482
width: 300
85-
# only NONE placeholder is supported in SSR/DSG
86-
# placeholder: DOMINANT_COLOR
87-
placeholder: NONE
83+
placeholder: BLURRED
8884
)
8985
constrained_traced: gatsbyImage(
9086
layout: CONSTRAINED
9187
width: 300
92-
# only NONE placeholder is supported in SSR/DSG
93-
# placeholder: DOMINANT_COLOR
94-
placeholder: NONE
88+
placeholder: TRACED_SVG
9589
)
9690
full: gatsbyImage(layout: FULL_WIDTH, width: 500, placeholder: NONE)
9791
}
@@ -109,3 +103,11 @@ export const pageQuery = graphql`
109103
`
110104

111105
export default RemoteFile
106+
107+
export function getServerData() {
108+
return {
109+
props: {
110+
ssr: true,
111+
},
112+
}
113+
}

‎packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import crypto from "crypto"
1+
import { createHash } from "crypto"
22
import { basename } from "path"
33

44
import type { FileCdnUrlGeneratorFn, FileCdnSourceImage } from "gatsby"
@@ -21,8 +21,7 @@ export const generateFileUrl: FileCdnUrlGeneratorFn = function generateFileUrl(
2121
baseURL.searchParams.append(`cd`, source.internal.contentDigest)
2222
} else {
2323
baseURL = new URL(
24-
`${placeholderOrigin}${pathPrefix}/_gatsby/file/${crypto
25-
.createHash(`md5`)
24+
`${placeholderOrigin}${pathPrefix}/_gatsby/file/${createHash(`md5`)
2625
.update(source.url)
2726
.digest(`hex`)}/${basename(source.filename)}`
2827
)

‎packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,11 @@ export async function createGraphqlEngineBundle(
129129
reporter: Reporter,
130130
isVerbose?: boolean
131131
): Promise<webpack.Compilation | undefined> {
132+
const state = store.getState()
133+
const pathPrefix = state.program.prefixPaths
134+
? state.config.pathPrefix ?? ``
135+
: ``
136+
132137
const schemaSnapshotString = await fs.readFile(
133138
path.join(rootDir, `.cache`, `schema.gql`),
134139
`utf-8`
@@ -151,7 +156,7 @@ export async function createGraphqlEngineBundle(
151156

152157
// We force a specific lmdb binary module if we detected a broken lmdb installation or if we detect the presence of an adapter
153158
let forcedLmdbBinaryModule: string | undefined = undefined
154-
if (store.getState().adapter.instance) {
159+
if (state.adapter.instance) {
155160
forcedLmdbBinaryModule = `${lmdbPackage}/node.abi83.glibc.node`
156161
}
157162
// We always force the binary if we've installed an alternative path
@@ -317,6 +322,7 @@ export async function createGraphqlEngineBundle(
317322
"process.env.GATSBY_SKIP_WRITING_SCHEMA_TO_FILE": `true`,
318323
"process.env.NODE_ENV": JSON.stringify(`production`),
319324
SCHEMA_SNAPSHOT: JSON.stringify(schemaSnapshotString),
325+
PATH_PREFIX: JSON.stringify(pathPrefix),
320326
"process.env.GATSBY_LOGGER": JSON.stringify(`yurnalist`),
321327
"process.env.GATSBY_SLICES": JSON.stringify(
322328
!!process.env.GATSBY_SLICES

‎packages/gatsby/src/schema/graphql-engine/entry.ts

+22
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,34 @@ export class GraphQLEngine {
4343
this.getRunner()
4444
}
4545

46+
private setupPathPrefix(pathPrefix: string): void {
47+
if (pathPrefix) {
48+
store.dispatch({
49+
type: `SET_PROGRAM`,
50+
payload: {
51+
prefixPaths: true,
52+
},
53+
})
54+
55+
store.dispatch({
56+
type: `SET_SITE_CONFIG`,
57+
payload: {
58+
...store.getState().config,
59+
pathPrefix,
60+
},
61+
})
62+
}
63+
}
64+
4665
private async _doGetRunner(): Promise<GraphQLRunner> {
4766
await tracerReadyPromise
4867

4968
const wrapActivity = reporter.phantomActivity(`Initializing GraphQL Engine`)
5069
wrapActivity.start()
5170
try {
71+
// @ts-ignore PATH_PREFIX is being "inlined" by bundler
72+
this.setupPathPrefix(PATH_PREFIX)
73+
5274
// @ts-ignore SCHEMA_SNAPSHOT is being "inlined" by bundler
5375
store.dispatch(actions.createTypes(SCHEMA_SNAPSHOT))
5476

‎packages/gatsby/src/utils/page-ssr-module/lambda.ts

+114-7
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { promisify } from "util"
1010

1111
import type { IGatsbyPage } from "../../internal"
1212
import type { ISSRData } from "./entry"
13-
import { link } from "linkfs"
13+
import { link, rewritableMethods as linkRewritableMethods } from "linkfs"
1414

1515
const cdnDatastore = `%CDN_DATASTORE_PATH%`
1616
const PATH_PREFIX = `%PATH_PREFIX%`
@@ -30,7 +30,7 @@ function setupFsWrapper(): string {
3030
global.__GATSBY.root = TEMP_DIR
3131

3232
// TODO: don't hardcode this
33-
const cacheDir = `/var/task/.cache`
33+
const cacheDir = `${process.cwd()}/.cache`
3434

3535
// we need to rewrite fs
3636
const rewrites = [
@@ -47,25 +47,132 @@ function setupFsWrapper(): string {
4747
to: TEMP_CACHE_DIR,
4848
rewrites,
4949
})
50+
51+
// copied from https://github.com/streamich/linkfs/blob/master/src/index.ts#L126-L142
52+
function mapPathUsingRewrites(fsPath: fs.PathLike): string {
53+
let filename = path.resolve(String(fsPath))
54+
for (const [from, to] of rewrites) {
55+
if (filename.indexOf(from) === 0) {
56+
const rootRegex = /(?:^[a-zA-Z]:\\$)|(?:^\/$)/ // C:\ vs /
57+
const isRoot = from.match(rootRegex)
58+
const baseRegex = `^(` + from.replace(/\\/g, `\\\\`) + `)`
59+
60+
if (isRoot) {
61+
const regex = new RegExp(baseRegex)
62+
filename = filename.replace(regex, () => to + path.sep)
63+
} else {
64+
const regex = new RegExp(baseRegex + `(\\\\|/|$)`)
65+
filename = filename.replace(regex, (_match, _p1, p2) => to + p2)
66+
}
67+
}
68+
}
69+
return filename
70+
}
71+
72+
function applyRename<
73+
T = typeof import("fs") | typeof import("fs").promises
74+
>(fsToRewrite: T, lfs: T, method: "rename" | "renameSync"): void {
75+
const original = fsToRewrite[method]
76+
if (original) {
77+
// @ts-ignore - complains about __promisify__ which doesn't actually exist in runtime
78+
lfs[method] = (
79+
...args: Parameters<typeof import("fs")["rename" | "renameSync"]>
80+
): ReturnType<typeof import("fs")["rename" | "renameSync"]> => {
81+
args[0] = mapPathUsingRewrites(args[0])
82+
args[1] = mapPathUsingRewrites(args[1])
83+
// @ts-ignore - can't decide which signature this is, but we just preserve the original
84+
// signature here and mostly care about first 2 arguments being PathLike
85+
return original.apply(fsToRewrite, args)
86+
}
87+
}
88+
}
89+
90+
// linkfs doesn't handle following methods, so we need to add them manually
91+
linkRewritableMethods.push(`createWriteStream`, `createReadStream`, `rm`)
92+
93+
function createLinkedFS<
94+
T = typeof import("fs") | typeof import("fs").promises
95+
>(fsToRewrite: T): T {
96+
const linkedFS = link(fsToRewrite, rewrites) as T
97+
98+
// linkfs doesn't handle `to` argument in `rename` and `renameSync` methods
99+
applyRename(fsToRewrite, linkedFS, `rename`)
100+
applyRename(fsToRewrite, linkedFS, `renameSync`)
101+
102+
return linkedFS
103+
}
104+
50105
// Alias the cache dir paths to the temp dir
51-
const lfs = link(fs, rewrites) as typeof import("fs")
106+
const lfs = createLinkedFS(fs)
52107

53108
// linkfs doesn't pass across the `native` prop, which graceful-fs needs
54109
for (const key in lfs) {
55110
if (Object.hasOwnProperty.call(fs[key], `native`)) {
56111
lfs[key].native = fs[key].native
57112
}
58113
}
59-
60-
const dbPath = path.join(TEMP_CACHE_DIR, `data`, `datastore`)
61-
62114
// 'promises' is not initially linked within the 'linkfs'
63115
// package, and is needed by underlying Gatsby code (the
64116
// @graphql-tools/code-file-loader)
65-
lfs.promises = link(fs.promises, rewrites)
117+
lfs.promises = createLinkedFS(fs.promises)
118+
119+
const originalWritesStream = fs.WriteStream
120+
function LinkedWriteStream(
121+
this: fs.WriteStream,
122+
...args: Parameters<(typeof fs)["createWriteStream"]>
123+
): fs.WriteStream {
124+
args[0] = mapPathUsingRewrites(args[0])
125+
args[1] =
126+
typeof args[1] === `string`
127+
? {
128+
flags: args[1],
129+
// @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
130+
fs: lfs,
131+
}
132+
: {
133+
...(args[1] || {}),
134+
// @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
135+
fs: lfs,
136+
}
137+
138+
// @ts-ignore TS doesn't like extending prototype "classes"
139+
return originalWritesStream.apply(this, args)
140+
}
141+
LinkedWriteStream.prototype = Object.create(originalWritesStream.prototype)
142+
// @ts-ignore TS doesn't like extending prototype "classes"
143+
lfs.WriteStream = LinkedWriteStream
144+
145+
const originalReadStream = fs.ReadStream
146+
function LinkedReadStream(
147+
this: fs.ReadStream,
148+
...args: Parameters<(typeof fs)["createReadStream"]>
149+
): fs.ReadStream {
150+
args[0] = mapPathUsingRewrites(args[0])
151+
args[1] =
152+
typeof args[1] === `string`
153+
? {
154+
flags: args[1],
155+
// @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
156+
fs: lfs,
157+
}
158+
: {
159+
...(args[1] || {}),
160+
// @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
161+
fs: lfs,
162+
}
163+
164+
// @ts-ignore TS doesn't like extending prototype "classes"
165+
return originalReadStream.apply(this, args)
166+
}
167+
LinkedReadStream.prototype = Object.create(originalReadStream.prototype)
168+
// @ts-ignore TS doesn't like extending prototype "classes"
169+
lfs.ReadStream = LinkedReadStream
170+
171+
const dbPath = path.join(TEMP_CACHE_DIR, `data`, `datastore`)
66172

67173
// Gatsby uses this instead of fs if present
68174
// eslint-disable-next-line no-underscore-dangle
175+
// @ts-ignore __promisify__ stuff
69176
global._fsWrapper = lfs
70177

71178
if (!cdnDatastore) {

0 commit comments

Comments
 (0)
Please sign in to comment.