Skip to content

Commit 56ddcce

Browse files
gatsbybotpieh
gatsbybot
and
pieh
authoredSep 20, 2023
fix(gatsby-adapter-netlify): handle cases with large cached _redirects and/or _headers files (#38559) (#38564)
* test(gatsby-adapter-netlify): add unit tests for entries injection * fix(gatsby-adapter-netlify): handle cases with large cached _redirects and/or _headers filestest(gatsby-adapter-netlify): add unit tests for entries injection * Update route-handler.ts (cherry picked from commit db41d13) Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>
·
@gatsbyjs/parcel-namer-relative-to-cwd@2.12.1babel-plugin-remove-graphql-queries@5.12.1
1 parent 68be864 commit 56ddcce

File tree

2 files changed

+252
-28
lines changed

2 files changed

+252
-28
lines changed
 

‎packages/gatsby-adapter-netlify/src/__tests__/route-handler.ts

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import fs from "fs-extra"
2+
import { tmpdir } from "os"
3+
import { join } from "path"
4+
import {
5+
injectEntries,
6+
ADAPTER_MARKER_START,
7+
ADAPTER_MARKER_END,
8+
NETLIFY_PLUGIN_MARKER_START,
9+
NETLIFY_PLUGIN_MARKER_END,
10+
GATSBY_PLUGIN_MARKER_START,
11+
} from "../route-handler"
12+
13+
function generateLotOfContent(placeholderCharacter: string): string {
14+
return (placeholderCharacter.repeat(80) + `\n`).repeat(1_000_000)
15+
}
16+
17+
const newAdapterContent = generateLotOfContent(`a`)
18+
const previousAdapterContent =
19+
ADAPTER_MARKER_START +
20+
`\n` +
21+
generateLotOfContent(`b`) +
22+
ADAPTER_MARKER_END +
23+
`\n`
24+
25+
const gatsbyPluginNetlifyContent =
26+
GATSBY_PLUGIN_MARKER_START + `\n` + generateLotOfContent(`c`)
27+
28+
const netlifyPluginGatsbyContent =
29+
NETLIFY_PLUGIN_MARKER_START +
30+
`\n` +
31+
generateLotOfContent(`c`) +
32+
NETLIFY_PLUGIN_MARKER_END +
33+
`\n`
34+
35+
const customContent1 =
36+
`# customContent1 start` +
37+
`\n` +
38+
generateLotOfContent(`x`) +
39+
`# customContent1 end` +
40+
`\n`
41+
const customContent2 =
42+
`# customContent2 start` +
43+
`\n` +
44+
generateLotOfContent(`y`) +
45+
`# customContent2 end` +
46+
`\n`
47+
const customContent3 =
48+
`# customContent3 start` +
49+
`\n` +
50+
generateLotOfContent(`z`) +
51+
`# customContent3 end` +
52+
`\n`
53+
54+
async function getContent(previousContent?: string): Promise<string> {
55+
const filePath = join(
56+
await fs.mkdtemp(join(tmpdir(), `inject-entries`)),
57+
`out.txt`
58+
)
59+
60+
if (typeof previousContent !== `undefined`) {
61+
await fs.writeFile(filePath, previousContent)
62+
}
63+
64+
await injectEntries(filePath, newAdapterContent)
65+
66+
return fs.readFile(filePath, `utf8`)
67+
}
68+
69+
jest.setTimeout(60_000)
70+
71+
describe(`route-handler`, () => {
72+
describe(`injectEntries`, () => {
73+
it(`no cached file`, async () => {
74+
const content = await getContent()
75+
76+
expect(content.indexOf(newAdapterContent)).not.toBe(-1)
77+
})
78+
79+
describe(`has cached file`, () => {
80+
it(`no previous adapter or plugins or custom entries`, async () => {
81+
const content = await getContent(``)
82+
83+
expect(content.indexOf(newAdapterContent)).not.toBe(-1)
84+
})
85+
86+
it(`has just custom entries`, async () => {
87+
const content = await getContent(customContent1)
88+
89+
expect(content.indexOf(newAdapterContent)).not.toBe(-1)
90+
expect(content.indexOf(customContent1)).not.toBe(-1)
91+
})
92+
93+
it(`has just gatsby-plugin-netlify entries`, async () => {
94+
const content = await getContent(gatsbyPluginNetlifyContent)
95+
96+
expect(content.indexOf(newAdapterContent)).not.toBe(-1)
97+
// it removes gatsby-plugin-netlify entries
98+
expect(content.indexOf(GATSBY_PLUGIN_MARKER_START)).toBe(-1)
99+
expect(content.indexOf(gatsbyPluginNetlifyContent)).toBe(-1)
100+
})
101+
102+
it(`has just netlify-plugin-gatsby entries`, async () => {
103+
const content = await getContent(netlifyPluginGatsbyContent)
104+
105+
expect(content.indexOf(newAdapterContent)).not.toBe(-1)
106+
// it removes netlify-plugin-gatsby entries
107+
expect(content.indexOf(NETLIFY_PLUGIN_MARKER_START)).toBe(-1)
108+
expect(content.indexOf(NETLIFY_PLUGIN_MARKER_END)).toBe(-1)
109+
expect(content.indexOf(netlifyPluginGatsbyContent)).toBe(-1)
110+
})
111+
112+
it(`has gatsby-plugin-netlify, nelify-plugin-gatsby, custom content and previous adapter content`, async () => {
113+
// kitchen-sink
114+
const previousContent =
115+
customContent1 +
116+
previousAdapterContent +
117+
customContent2 +
118+
netlifyPluginGatsbyContent +
119+
customContent3 +
120+
gatsbyPluginNetlifyContent
121+
122+
const content = await getContent(previousContent)
123+
124+
expect(content.indexOf(newAdapterContent)).not.toBe(-1)
125+
126+
// it preserve any custom entries
127+
expect(content.indexOf(customContent1)).not.toBe(-1)
128+
expect(content.indexOf(customContent2)).not.toBe(-1)
129+
expect(content.indexOf(customContent3)).not.toBe(-1)
130+
131+
// it removes previous gatsby-adapter-netlify entries
132+
expect(content.indexOf(previousAdapterContent)).toBe(-1)
133+
134+
// it removes gatsby-plugin-netlify entries
135+
expect(content.indexOf(GATSBY_PLUGIN_MARKER_START)).toBe(-1)
136+
expect(content.indexOf(gatsbyPluginNetlifyContent)).toBe(-1)
137+
138+
// it removes netlify-plugin-gatsby entries
139+
expect(content.indexOf(NETLIFY_PLUGIN_MARKER_START)).toBe(-1)
140+
expect(content.indexOf(NETLIFY_PLUGIN_MARKER_END)).toBe(-1)
141+
expect(content.indexOf(netlifyPluginGatsbyContent)).toBe(-1)
142+
})
143+
})
144+
})
145+
})

‎packages/gatsby-adapter-netlify/src/route-handler.ts

+107-28
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import type { RoutesManifest } from "gatsby"
2-
import { EOL } from "os"
2+
import { tmpdir } from "os"
3+
import { Transform } from "stream"
4+
import { join, basename } from "path"
35
import fs from "fs-extra"
46

57
const NETLIFY_REDIRECT_KEYWORDS_ALLOWLIST = new Set([
@@ -20,35 +22,112 @@ const toNetlifyPath = (fromPath: string, toPath: string): Array<string> => {
2022

2123
return [netlifyFromPath, netlifyToPath]
2224
}
23-
const MARKER_START = `# gatsby-adapter-netlify start`
24-
const MARKER_END = `# gatsby-adapter-netlify end`
25-
26-
async function injectEntries(fileName: string, content: string): Promise<void> {
25+
export const ADAPTER_MARKER_START = `# gatsby-adapter-netlify start`
26+
export const ADAPTER_MARKER_END = `# gatsby-adapter-netlify end`
27+
export const NETLIFY_PLUGIN_MARKER_START = `# @netlify/plugin-gatsby redirects start`
28+
export const NETLIFY_PLUGIN_MARKER_END = `# @netlify/plugin-gatsby redirects end`
29+
export const GATSBY_PLUGIN_MARKER_START = `## Created with gatsby-plugin-netlify`
30+
31+
export async function injectEntries(
32+
fileName: string,
33+
content: string
34+
): Promise<void> {
2735
await fs.ensureFile(fileName)
2836

29-
const data = await fs.readFile(fileName, `utf8`)
30-
const [initial = ``, rest = ``] = data.split(MARKER_START)
31-
const [, final = ``] = rest.split(MARKER_END)
32-
const out = [
33-
initial === EOL ? `` : initial,
34-
initial.endsWith(EOL) ? `` : EOL,
35-
MARKER_START,
36-
EOL,
37-
content,
38-
EOL,
39-
MARKER_END,
40-
final.startsWith(EOL) ? `` : EOL,
41-
final === EOL ? `` : final,
42-
]
43-
.filter(Boolean)
44-
.join(``)
45-
.replace(
46-
/# @netlify\/plugin-gatsby redirects start(.|\n|\r)*# @netlify\/plugin-gatsby redirects end/gm,
47-
``
48-
)
49-
.replace(/## Created with gatsby-plugin-netlify(.|\n|\r)*$/gm, ``)
50-
51-
await fs.outputFile(fileName, out)
37+
const tmpFile = join(
38+
await fs.mkdtemp(join(tmpdir(), basename(fileName))),
39+
`out.txt`
40+
)
41+
42+
let tail = ``
43+
let insideNetlifyPluginGatsby = false
44+
let insideGatsbyPluginNetlify = false
45+
let insideGatsbyAdapterNetlify = false
46+
let injectedEntries = false
47+
48+
const annotatedContent = `${ADAPTER_MARKER_START}\n${content}\n${ADAPTER_MARKER_END}\n`
49+
50+
function getContentToAdd(final: boolean): string {
51+
const lines = tail.split(`\n`)
52+
tail = ``
53+
54+
let contentToAdd = ``
55+
for (let i = 0; i < lines.length; i++) {
56+
const line = lines[i]
57+
58+
if (!final && i === lines.length - 1) {
59+
tail = line
60+
break
61+
}
62+
63+
let skipLine =
64+
insideGatsbyAdapterNetlify ||
65+
insideGatsbyPluginNetlify ||
66+
insideNetlifyPluginGatsby
67+
68+
if (line.includes(ADAPTER_MARKER_START)) {
69+
skipLine = true
70+
insideGatsbyAdapterNetlify = true
71+
} else if (line.includes(ADAPTER_MARKER_END)) {
72+
insideGatsbyAdapterNetlify = false
73+
contentToAdd += annotatedContent
74+
injectedEntries = true
75+
} else if (line.includes(NETLIFY_PLUGIN_MARKER_START)) {
76+
insideNetlifyPluginGatsby = true
77+
skipLine = true
78+
} else if (line.includes(NETLIFY_PLUGIN_MARKER_END)) {
79+
insideNetlifyPluginGatsby = false
80+
} else if (line.includes(GATSBY_PLUGIN_MARKER_START)) {
81+
insideGatsbyPluginNetlify = true
82+
skipLine = true
83+
}
84+
85+
if (!skipLine) {
86+
contentToAdd += line + `\n`
87+
}
88+
}
89+
90+
return contentToAdd
91+
}
92+
93+
const streamReplacer = new Transform({
94+
transform(chunk, _encoding, callback): void {
95+
tail = tail + chunk.toString()
96+
97+
try {
98+
callback(null, getContentToAdd(false))
99+
} catch (e) {
100+
callback(e)
101+
}
102+
},
103+
flush(callback): void {
104+
try {
105+
let contentToAdd = getContentToAdd(true)
106+
if (!injectedEntries) {
107+
contentToAdd += annotatedContent
108+
}
109+
callback(null, contentToAdd)
110+
} catch (e) {
111+
callback(e)
112+
}
113+
},
114+
})
115+
116+
await new Promise<void>((resolve, reject) => {
117+
const writeStream = fs.createWriteStream(tmpFile)
118+
const pipeline = fs
119+
.createReadStream(fileName)
120+
.pipe(streamReplacer)
121+
.pipe(writeStream)
122+
123+
pipeline.on(`finish`, resolve)
124+
pipeline.on(`error`, reject)
125+
streamReplacer.on(`error`, reject)
126+
})
127+
128+
// remove previous file and move new file from tmp to final path
129+
await fs.remove(fileName)
130+
await fs.move(tmpFile, fileName)
52131
}
53132

54133
export async function handleRoutesManifest(

0 commit comments

Comments
 (0)
Please sign in to comment.