Stabilize more tests (#15470)

jest retries seem to be masking test failures, like the `auto-export` one (and maybe others). I turned it off for now. The `auto-export` test fails when retries are turned off.
the output of this test failure was a bit unhelpful so I also improved it.
Many tests have anonymous page functions.

package.json

@@ -112,8 +112,8 @@
     "release": "6.0.1",
     "request-promise-core": "1.1.2",
     "rimraf": "2.6.3",
-    "selenium-standalone": "6.17.0",
-    "selenium-webdriver": "4.0.0-alpha.5",
+    "selenium-standalone": "6.18.0",
+    "selenium-webdriver": "4.0.0-alpha.7",
     "shell-quote": "1.7.2",
     "styled-components": "5.1.0",
     "styled-jsx-plugin-postcss": "2.0.1",

run-tests.js

@@ -136,7 +136,7 @@ const TIMINGS_API = `https://next-timings.jjsweb.site/api/timings`
         {
           stdio: 'inherit',
           env: {
-            JEST_RETRY_TIMES: 2,
+            JEST_RETRY_TIMES: 0,
             ...process.env,
             ...(isAzure
               ? {

test/integration/amphtml/pages/only-amp.js

@@ -1,7 +1,9 @@
 export const config = { amp: true }
 
-export default () => (
-  <div>
-    <p id="only-amp">Only AMP for me...</p>
-  </div>
-)
+export default function Page() {
+  return (
+    <div>
+      <p id="only-amp">Only AMP for me...</p>
+    </div>
+  )
+}

test/integration/auto-export/pages/[post]/[cmnt].js

@@ -5,17 +5,17 @@ if (typeof window !== 'undefined') {
   const origWarn = window.console.warn
   const origError = window.console.error
   window.console.warn = function (...args) {
-    window.caughtWarns.push(1)
+    window.caughtWarns.push(args)
     origWarn(...args)
   }
   window.console.error = function (...args) {
-    window.caughtWarns.push(1)
+    window.caughtWarns.push(args)
     origError(...args)
   }
   window.pathnames = []
 }
 
-export default () => {
+export default function Page() {
   if (typeof window !== 'undefined') {
     window.pathnames.push(window.location.pathname)
   }

test/integration/auto-export/pages/[post]/index.js

@@ -1,6 +1,6 @@
 import { useRouter } from 'next/router'
 
-export default () => {
+export default function Page() {
   const { query } = useRouter()
 
   return <p>post: {query.post}</p>

test/integration/auto-export/test/index.test.js

@@ -81,9 +81,8 @@ describe('Auto Export', () => {
 
     it('should not show hydration warning from mismatching asPath', async () => {
       const browser = await webdriver(appPort, '/zeit/cmnt-1')
-
-      const numCaught = await browser.eval(`window.caughtWarns.length`)
-      expect(numCaught).toBe(0)
+      const caughtWarns = await browser.eval(`window.caughtWarns`)
+      expect(caughtWarns).toEqual([])
     })
   })
 })

test/integration/build-indicator/pages/a.js

@@ -1 +1,3 @@
-export default () => <p>Hello from a</p>
+export default function Page() {
+  return <p>Hello from a</p>
+}

test/integration/build-indicator/pages/b.js

@@ -1 +1,3 @@
-export default () => <p>Hello from b</p>
+export default function Page() {
+  return <p>Hello from b</p>
+}

test/integration/build-indicator/pages/index.js

@@ -1,12 +1,14 @@
 import Link from 'next/link'
 
-export default () => (
-  <>
-    <Link href="/a">
-      <a id="to-a">Go to a</a>
-    </Link>
-    <Link href="/b">
-      <a id="to-b">Go to b</a>
-    </Link>
-  </>
-)
+export default function Page() {
+  return (
+    <>
+      <Link href="/a">
+        <a id="to-a">Go to a</a>
+      </Link>
+      <Link href="/b">
+        <a id="to-b">Go to b</a>
+      </Link>
+    </>
+  )
+}

test/integration/dynamic-optional-routing/test/index.test.js

@@ -10,14 +10,15 @@ import {
   nextBuild,
   nextStart,
   renderViaHTTP,
-  waitFor,
+  check,
 } from 'next-test-utils'
 import { join } from 'path'
 
 jest.setTimeout(1000 * 60 * 2)
 
 let app
 let appPort
+let stderr
 const appDir = join(__dirname, '../')
 
 const DUMMY_PAGE = 'export default () => null'
@@ -187,9 +188,10 @@ function runInvalidPagesTests(buildFn) {
     const invalidRoute = appDir + 'pages/index.js'
     try {
       await fs.outputFile(invalidRoute, DUMMY_PAGE, 'utf-8')
-      const { stderr } = await buildFn(appDir)
-      await expect(stderr).toMatch(
-        'You cannot define a route with the same specificity as a optional catch-all route'
+      await buildFn(appDir)
+      await check(
+        () => stderr,
+        /You cannot define a route with the same specificity as a optional catch-all route/
       )
     } finally {
       await fs.unlink(invalidRoute)
@@ -200,9 +202,10 @@ function runInvalidPagesTests(buildFn) {
     const invalidRoute = appDir + 'pages/nested.js'
     try {
       await fs.outputFile(invalidRoute, DUMMY_PAGE, 'utf-8')
-      const { stderr } = await buildFn(appDir)
-      await expect(stderr).toMatch(
-        'You cannot define a route with the same specificity as a optional catch-all route'
+      await buildFn(appDir)
+      await check(
+        () => stderr,
+        /You cannot define a route with the same specificity as a optional catch-all route/
       )
     } finally {
       await fs.unlink(invalidRoute)
@@ -213,8 +216,8 @@ function runInvalidPagesTests(buildFn) {
     const invalidRoute = appDir + 'pages/nested/[...param].js'
     try {
       await fs.outputFile(invalidRoute, DUMMY_PAGE, 'utf-8')
-      const { stderr } = await buildFn(appDir)
-      await expect(stderr).toMatch(/You cannot use both .+ at the same level/)
+      await buildFn(appDir)
+      await check(() => stderr, /You cannot use both .+ at the same level/)
     } finally {
       await fs.unlink(invalidRoute)
     }
@@ -224,9 +227,10 @@ function runInvalidPagesTests(buildFn) {
     const invalidRoute = appDir + 'pages/invalid/[[param]].js'
     try {
       await fs.outputFile(invalidRoute, DUMMY_PAGE, 'utf-8')
-      const { stderr } = await buildFn(appDir)
-      await expect(stderr).toMatch(
-        'Optional route parameters are not yet supported'
+      await buildFn(appDir)
+      await check(
+        () => stderr,
+        /Optional route parameters are not yet supported/
       )
     } finally {
       await fs.unlink(invalidRoute)
@@ -245,14 +249,12 @@ describe('Dynamic Optional Routing', () => {
     runTests()
 
     runInvalidPagesTests(async (appDir) => {
-      let stderr = ''
+      stderr = ''
       await launchApp(appDir, await findPort(), {
         onStderr: (msg) => {
           stderr += msg
         },
       })
-      await waitFor(1000)
-      return { stderr }
     })
   })
 
@@ -272,9 +274,9 @@ describe('Dynamic Optional Routing', () => {
 
     runTests()
 
-    runInvalidPagesTests(async (appDir) =>
-      nextBuild(appDir, [], { stderr: true })
-    )
+    runInvalidPagesTests(async (appDir) => {
+      ;({ stderr } = await nextBuild(appDir, [], { stderr: true }))
+    })
 
     it('should fail to build when param is not explicitly defined', async () => {
       const invalidRoute = appDir + 'pages/invalid/[[...slug]].js'

test/integration/error-in-error/test/index.test.js

@@ -31,7 +31,7 @@ describe('Handles an Error in _error', () => {
 
   it('Handles error during client transition', async () => {
     const browser = await webdriver(port, '/')
-    await browser.elementByCss('a').click()
+    await browser.waitForElementByCss('a').click()
     await waitFor(1000)
     const html = await browser.eval('document.body.innerHTML')
     expect(html).toMatch(/internal server error/i)

test/integration/invalid-href/pages/third.js

@@ -3,7 +3,7 @@ import { useState } from 'react'
 
 const invalidLink = 'https://vercel.com/'
 
-export default () => {
+export default function Page() {
   const { query, ...router } = useRouter()
   const [isDone, setIsDone] = useState(false)
   const { method = 'push' } = query

test/integration/invalid-href/test/index.test.js

@@ -30,24 +30,18 @@ jest.retryTimes(0)
 const showsError = async (pathname, regex, click = false, isWarn = false) => {
   const browser = await webdriver(appPort, pathname)
   try {
+    // wait for page to be built and navigated to
+    await browser.waitForElementByCss('#click-me')
     if (isWarn) {
       await browser.eval(`(function() {
         window.warnLogs = []
         var origWarn = window.console.warn
-        window.console.warn = function() {
-          var warnStr = ''
-          for (var i = 0; i < arguments.length; i++) {
-            if (i > 0) warnStr += ' ';
-            warnStr += arguments[i]
-          }
-          window.warnLogs.push(warnStr)
-          origWarn.apply(undefined, arguments)
+        window.console.warn = (...args) => {
+          window.warnLogs.push(args.join(' '))
+          origWarn.apply(window.console, args)
         }
       })()`)
     }
-    // wait for page to be built and navigated to
-    await waitFor(3000)
-    await browser.waitForElementByCss('#click-me')
     if (click) {
       await browser.elementByCss('#click-me').click()
       await waitFor(500)
@@ -70,6 +64,11 @@ const showsError = async (pathname, regex, click = false, isWarn = false) => {
 const noError = async (pathname, click = false) => {
   const browser = await webdriver(appPort, '/')
   try {
+    await check(async () => {
+      const appReady = await browser.eval('!!window.next.router')
+      console.log('app ready: ', appReady)
+      return appReady ? 'ready' : 'nope'
+    }, 'ready')
     await browser.eval(`(function() {
       window.caughtErrors = []
       window.addEventListener('error', function (error) {
@@ -80,8 +79,6 @@ const noError = async (pathname, click = false) => {
       })
       window.next.router.replace('${pathname}')
     })()`)
-    // wait for page to be built and navigated to
-    await waitFor(3000)
     await browser.waitForElementByCss('#click-me')
     if (click) {
       await browser.elementByCss('#click-me').click()