feat: introduce retry-after detection (#221)

This introduces a --retry flag, which when passed will automatically retry requests that comes back with a HTTP 429, and a retry-after header. I tested this against GitHub , and it appears to work as expected.

Author: JustinBeckwith

README.md

@@ -64,6 +64,10 @@ $ linkinator LOCATIONS [ --arguments ]
     --recurse, -r
         Recursively follow links on the same root domain.
 
+    --retry,
+        Automatically retry requests that return HTTP 429 responses and include
+        a 'retry-after' header. Defaults to false.
+
     --server-root
         When scanning a locally directory, customize the location on disk
         where the server is started.  Defaults to the path passed in [LOCATION].
@@ -183,6 +187,7 @@ Asynchronous method that runs a site wide scan. Options come in the form of an o
 - `concurrency` (number) -  The number of connections to make simultaneously. Defaults to 100.
 - `port` (number) - When the `path` is provided as a local path on disk, the `port` on which to start the temporary web server.  Defaults to a random high range order port.
 - `recurse` (boolean) - By default, all scans are shallow.  Only the top level links on the requested page will be scanned.  By setting `recurse` to `true`, the crawler will follow all links on the page, and continue scanning links **on the same domain** for as long as it can go. Results are cached, so no worries about loops.
+- `retry` (boolean|RetryConfig) - Automatically retry requests that respond with an HTTP 429, and include a `retry-after` header.  The `RetryConfig` option is a placeholder for fine-grained controls to be implemented at a later time, and is only included here to signal forward-compatibility.
 - `serverRoot` (string) - When scanning a locally directory, customize the location on disk
 where the server is started.  Defaults to the path passed in `path`.
 - `timeout` (number) - By default, requests made by linkinator do not time out (or follow the settings of the OS).  This option (in milliseconds) will fail requests after the configured amount of time.

src/cli.ts

@@ -3,7 +3,13 @@
 import * as meow from 'meow';
 import * as updateNotifier from 'update-notifier';
 import chalk = require('chalk');
-import {LinkChecker, LinkState, LinkResult, CheckOptions} from './index';
+import {
+  LinkChecker,
+  LinkState,
+  LinkResult,
+  CheckOptions,
+  RetryInfo,
+} from './index';
 import {promisify} from 'util';
 import {Flags, getConfig} from './config';
 import {Format, Logger, LogLevel} from './logger';
@@ -51,6 +57,10 @@ const cli = meow(
       --recurse, -r
           Recursively follow links on the same root domain.
 
+      --retry,
+          Automatically retry requests that return HTTP 429 responses and include
+          a 'retry-after' header. Defaults to false.
+
       --server-root
           When scanning a locally directory, customize the location on disk
           where the server is started.  Defaults to the path passed in [LOCATION].
@@ -85,6 +95,7 @@ const cli = meow(
       serverRoot: {type: 'string'},
       verbosity: {type: 'string'},
       directoryListing: {type: 'boolean'},
+      retry: {type: 'boolean'},
     },
     booleanDefault: undefined,
   }
@@ -107,6 +118,9 @@ async function main() {
   logger.error(`🏊‍♂️ crawling ${cli.input}`);
 
   const checker = new LinkChecker();
+  checker.on('retry', (info: RetryInfo) => {
+    logger.warn(`Retrying: ${info.url} in ${info.secondsUntilRetry} seconds.`);
+  });
   checker.on('link', (link: LinkResult) => {
     let state = '';
     switch (link.state) {
@@ -132,6 +146,7 @@ async function main() {
     concurrency: Number(flags.concurrency),
     serverRoot: flags.serverRoot,
     directoryListing: flags.directoryListing,
+    retry: flags.retry,
   };
   if (flags.skip) {
     if (typeof flags.skip === 'string') {

src/config.ts

@@ -14,6 +14,7 @@ export interface Flags {
   markdown?: boolean;
   serverRoot?: string;
   directoryListing?: boolean;
+  retry?: boolean;
 }
 
 export async function getConfig(flags: Flags) {

src/index.ts

@@ -1,39 +1,28 @@
 import {EventEmitter} from 'events';
 import {URL} from 'url';
 import * as http from 'http';
-import * as fs from 'fs';
-import * as util from 'util';
-import * as path from 'path';
 
 import {request, GaxiosResponse} from 'gaxios';
-import PQueue, {DefaultAddOptions} from 'p-queue';
-import PriorityQueue from 'p-queue/dist/priority-queue';
-import * as globby from 'glob';
 
+import {Queue} from './queue';
 import {getLinks} from './links';
 import {startWebServer} from './server';
+import {CheckOptions, processOptions} from './options';
 
-const stat = util.promisify(fs.stat);
-const glob = util.promisify(globby);
-
-export interface CheckOptions {
-  concurrency?: number;
-  port?: number;
-  path: string | string[];
-  recurse?: boolean;
-  timeout?: number;
-  markdown?: boolean;
-  linksToSkip?: string[] | ((link: string) => Promise<boolean>);
-  serverRoot?: string;
-  directoryListing?: boolean;
-}
+export {CheckOptions};
 
 export enum LinkState {
   OK = 'OK',
   BROKEN = 'BROKEN',
   SKIPPED = 'SKIPPED',
 }
 
+export interface RetryInfo {
+  url: string;
+  secondsUntilRetry: number;
+  status: number;
+}
+
 export interface LinkResult {
   url: string;
   status?: number;
@@ -53,9 +42,11 @@ interface CrawlOptions {
   crawl: boolean;
   results: LinkResult[];
   cache: Set<string>;
+  delayCache: Map<string, number>;
   checkOptions: CheckOptions;
-  queue: PQueue<PriorityQueue, DefaultAddOptions>;
+  queue: Queue;
   rootPath: string;
+  retry: boolean;
 }
 
 // Spoof a normal looking User-Agent to keep the servers happy
@@ -64,6 +55,12 @@ export const headers = {
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
 };
 
+export declare interface LinkChecker {
+  on(event: 'link', listener: (result: LinkResult) => void): this;
+  on(event: 'pagestart', listener: (link: string) => void): this;
+  on(event: 'retry', listener: (details: RetryInfo) => void): this;
+}
+
 /**
  * Instance class used to perform a crawl job.
  */
@@ -74,7 +71,7 @@ export class LinkChecker extends EventEmitter {
    * @param options Options to use while checking for 404s
    */
   async check(opts: CheckOptions) {
-    const options = await this.processOptions(opts);
+    const options = await processOptions(opts);
     if (!Array.isArray(options.path)) {
       options.path = [options.path];
     }
@@ -101,12 +98,13 @@ export class LinkChecker extends EventEmitter {
       console.log(options);
     }
 
-    const queue = new PQueue({
+    const queue = new Queue({
       concurrency: options.concurrency || 100,
     });
 
     const results = new Array<LinkResult>();
     const initCache: Set<string> = new Set();
+    const delayCache: Map<string, number> = new Map();
 
     for (const path of options.path) {
       const url = new URL(path);
@@ -118,8 +116,10 @@ export class LinkChecker extends EventEmitter {
           checkOptions: options,
           results,
           cache: initCache,
+          delayCache,
           queue,
           rootPath: path,
+          retry: !!opts.retry,
         });
       });
     }
@@ -135,121 +135,6 @@ export class LinkChecker extends EventEmitter {
     return result;
   }
 
-  /**
-   * Validate the provided flags all work with each other.
-   * @param options CheckOptions passed in from the CLI (or API)
-   */
-  private async processOptions(opts: CheckOptions): Promise<CheckOptions> {
-    const options = Object.assign({}, opts);
-
-    // ensure at least one path is provided
-    if (options.path.length === 0) {
-      throw new Error('At least one path must be provided');
-    }
-
-    // normalize options.path to an array of strings
-    if (!Array.isArray(options.path)) {
-      options.path = [options.path];
-    }
-
-    // disable directory listings by default
-    if (options.directoryListing === undefined) {
-      options.directoryListing = false;
-    }
-
-    // Ensure we do not mix http:// and file system paths.  The paths passed in
-    // must all be filesystem paths, or HTTP paths.
-    let isUrlType: boolean | undefined = undefined;
-    for (const path of options.path) {
-      const innerIsUrlType = path.startsWith('http');
-      if (isUrlType === undefined) {
-        isUrlType = innerIsUrlType;
-      } else if (innerIsUrlType !== isUrlType) {
-        throw new Error(
-          'Paths cannot be mixed between HTTP and local filesystem paths.'
-        );
-      }
-    }
-
-    // if there is a server root, make sure there are no HTTP paths
-    if (options.serverRoot && isUrlType) {
-      throw new Error(
-        "'serverRoot' cannot be defined when the 'path' points to an HTTP endpoint."
-      );
-    }
-
-    if (options.serverRoot) {
-      options.serverRoot = path.normalize(options.serverRoot);
-    }
-
-    // expand globs into paths
-    if (!isUrlType) {
-      const paths: string[] = [];
-      for (const filePath of options.path) {
-        // The glob path provided is relative to the serverRoot. For example,
-        // if the serverRoot is test/fixtures/nested, and the glob is "*/*.html",
-        // The glob needs to be calculated from the serverRoot directory.
-        const fullPath = options.serverRoot
-          ? path.join(options.serverRoot, filePath)
-          : filePath;
-        const expandedPaths = await glob(fullPath);
-        if (expandedPaths.length === 0) {
-          throw new Error(
-            `The provided glob "${filePath}" returned 0 results. The current working directory is "${process.cwd()}".`
-          );
-        }
-        // After resolving the globs, the paths need to be returned to their
-        // original form, without the serverRoot included in the path.
-        for (let p of expandedPaths) {
-          p = path.normalize(p);
-          if (options.serverRoot) {
-            const contractedPath = p
-              .split(path.sep)
-              .slice(options.serverRoot.split(path.sep).length)
-              .join(path.sep);
-            paths.push(contractedPath);
-          } else {
-            paths.push(p);
-          }
-        }
-      }
-      options.path = paths;
-    }
-
-    // enable markdown if someone passes a flag/glob right at it
-    if (options.markdown === undefined) {
-      for (const p of options.path) {
-        if (path.extname(p).toLowerCase() === '.md') {
-          options.markdown = true;
-        }
-      }
-    }
-
-    // Figure out which directory should be used as the root for the web server,
-    // and how that impacts the path to the file for the first request.
-    if (!options.serverRoot && !isUrlType) {
-      // if the serverRoot wasn't defined, and there are multiple paths, just
-      // use process.cwd().
-      if (options.path.length > 1) {
-        options.serverRoot = process.cwd();
-      } else {
-        // if there's a single path, try to be smart and figure it out
-        const s = await stat(options.path[0]);
-        options.serverRoot = options.path[0];
-        if (s.isFile()) {
-          const pathParts = options.path[0].split(path.sep);
-          options.path = [path.sep + pathParts[pathParts.length - 1]];
-          options.serverRoot =
-            pathParts.slice(0, pathParts.length - 1).join(path.sep) || '.';
-        } else {
-          options.serverRoot = options.path[0];
-          options.path = '/';
-        }
-      }
-    }
-    return options;
-  }
-
   /**
    * Crawl a given url with the provided options.
    * @pram opts List of options used to do the crawl
@@ -260,7 +145,7 @@ export class LinkChecker extends EventEmitter {
     // explicitly skip non-http[s] links before making the request
     const proto = opts.url.protocol;
     if (proto !== 'http:' && proto !== 'https:') {
-      const r = {
+      const r: LinkResult = {
         url: opts.url.href,
         status: 0,
         state: LinkState.SKIPPED,
@@ -306,6 +191,22 @@ export class LinkChecker extends EventEmitter {
       }
     }
 
+    // Check if this host has been marked for delay due to 429
+    if (opts.delayCache.has(opts.url.host)) {
+      const timeout = opts.delayCache.get(opts.url.host)!;
+      if (timeout > Date.now()) {
+        opts.queue.add(
+          async () => {
+            await this.crawl(opts);
+          },
+          {
+            delay: timeout - Date.now(),
+          }
+        );
+        return;
+      }
+    }
+
     // Perform a HEAD or GET request based on the need to crawl
     let status = 0;
     let state = LinkState.BROKEN;
@@ -322,6 +223,9 @@ export class LinkChecker extends EventEmitter {
         validateStatus: () => true,
         timeout: opts.checkOptions.timeout,
       });
+      if (this.shouldRetryAfter(res, opts)) {
+        return;
+      }
 
       // If we got an HTTP 405, the server may not like HEAD. GET instead!
       if (res.status === 405) {
@@ -333,6 +237,9 @@ export class LinkChecker extends EventEmitter {
           validateStatus: () => true,
           timeout: opts.checkOptions.timeout,
         });
+        if (this.shouldRetryAfter(res, opts)) {
+          return;
+        }
       }
     } catch (err) {
       // request failure: invalid domain name, etc.
@@ -355,6 +262,9 @@ export class LinkChecker extends EventEmitter {
           headers,
           timeout: opts.checkOptions.timeout,
         });
+        if (this.shouldRetryAfter(res, opts)) {
+          return;
+        }
       }
     } catch (ex) {
       failures.push(ex);
@@ -425,17 +335,74 @@ export class LinkChecker extends EventEmitter {
               url: result.url!,
               crawl,
               cache: opts.cache,
+              delayCache: opts.delayCache,
               results: opts.results,
               checkOptions: opts.checkOptions,
               queue: opts.queue,
               parent: opts.url.href,
               rootPath: opts.rootPath,
+              retry: opts.retry,
             });
           });
         }
       }
     }
   }
+  /**
+   * Check the incoming response for a `retry-after` header.  If present,
+   * and if the status was an HTTP 429, calculate the date at which this
+   * request should be retried. Ensure the delayCache knows that we're
+   * going to wait on requests for this entire host.
+   * @param res GaxiosResponse returned from the request
+   * @param opts CrawlOptions used during this request
+   */
+  shouldRetryAfter(res: GaxiosResponse, opts: CrawlOptions): boolean {
+    if (!opts.retry) {
+      return false;
+    }
+
+    const retryAfterRaw = res.headers['retry-after'];
+    if (res.status !== 429 || !retryAfterRaw) {
+      return false;
+    }
+
+    // The `retry-after` header can come in either <seconds> or
+    // A specific date to go check.
+    let retryAfter = Number(retryAfterRaw) * 1000 + Date.now();
+    if (isNaN(retryAfter)) {
+      retryAfter = Date.parse(retryAfterRaw);
+      if (isNaN(retryAfter)) {
+        return false;
+      }
+    }
+
+    // check to see if there is already a request to wait for this host
+    if (opts.delayCache.has(opts.url.host)) {
+      // use whichever time is higher in the cache
+      const currentTimeout = opts.delayCache.get(opts.url.host)!;
+      if (retryAfter > currentTimeout) {
+        opts.delayCache.set(opts.url.host, retryAfter);
+      }
+    } else {
+      opts.delayCache.set(opts.url.host, retryAfter);
+    }
+
+    opts.queue.add(
+      async () => {
+        await this.crawl(opts);
+      },
+      {
+        delay: retryAfter - Date.now(),
+      }
+    );
+    const retryDetails: RetryInfo = {
+      url: opts.url.href,
+      status: res.status,
+      secondsUntilRetry: Math.round((retryAfter - Date.now()) / 1000),
+    };
+    this.emit('retry', retryDetails);
+    return true;
+  }
 }
 
 /**

src/options.ts

@@ -0,0 +1,137 @@
+import * as fs from 'fs';
+import * as util from 'util';
+import * as path from 'path';
+import * as globby from 'glob';
+
+const stat = util.promisify(fs.stat);
+const glob = util.promisify(globby);
+
+export interface CheckOptions {
+  concurrency?: number;
+  port?: number;
+  path: string | string[];
+  recurse?: boolean;
+  timeout?: number;
+  markdown?: boolean;
+  linksToSkip?: string[] | ((link: string) => Promise<boolean>);
+  serverRoot?: string;
+  directoryListing?: boolean;
+  retry?: boolean;
+}
+
+/**
+ * Validate the provided flags all work with each other.
+ * @param options CheckOptions passed in from the CLI (or API)
+ */
+export async function processOptions(
+  opts: CheckOptions
+): Promise<CheckOptions> {
+  const options = Object.assign({}, opts);
+
+  // ensure at least one path is provided
+  if (options.path.length === 0) {
+    throw new Error('At least one path must be provided');
+  }
+
+  // normalize options.path to an array of strings
+  if (!Array.isArray(options.path)) {
+    options.path = [options.path];
+  }
+
+  // disable directory listings by default
+  if (options.directoryListing === undefined) {
+    options.directoryListing = false;
+  }
+
+  // Ensure we do not mix http:// and file system paths.  The paths passed in
+  // must all be filesystem paths, or HTTP paths.
+  let isUrlType: boolean | undefined = undefined;
+  for (const path of options.path) {
+    const innerIsUrlType = path.startsWith('http');
+    if (isUrlType === undefined) {
+      isUrlType = innerIsUrlType;
+    } else if (innerIsUrlType !== isUrlType) {
+      throw new Error(
+        'Paths cannot be mixed between HTTP and local filesystem paths.'
+      );
+    }
+  }
+
+  // if there is a server root, make sure there are no HTTP paths
+  if (options.serverRoot && isUrlType) {
+    throw new Error(
+      "'serverRoot' cannot be defined when the 'path' points to an HTTP endpoint."
+    );
+  }
+
+  if (options.serverRoot) {
+    options.serverRoot = path.normalize(options.serverRoot);
+  }
+
+  // expand globs into paths
+  if (!isUrlType) {
+    const paths: string[] = [];
+    for (const filePath of options.path) {
+      // The glob path provided is relative to the serverRoot. For example,
+      // if the serverRoot is test/fixtures/nested, and the glob is "*/*.html",
+      // The glob needs to be calculated from the serverRoot directory.
+      const fullPath = options.serverRoot
+        ? path.join(options.serverRoot, filePath)
+        : filePath;
+      const expandedPaths = await glob(fullPath);
+      if (expandedPaths.length === 0) {
+        throw new Error(
+          `The provided glob "${filePath}" returned 0 results. The current working directory is "${process.cwd()}".`
+        );
+      }
+      // After resolving the globs, the paths need to be returned to their
+      // original form, without the serverRoot included in the path.
+      for (let p of expandedPaths) {
+        p = path.normalize(p);
+        if (options.serverRoot) {
+          const contractedPath = p
+            .split(path.sep)
+            .slice(options.serverRoot.split(path.sep).length)
+            .join(path.sep);
+          paths.push(contractedPath);
+        } else {
+          paths.push(p);
+        }
+      }
+    }
+    options.path = paths;
+  }
+
+  // enable markdown if someone passes a flag/glob right at it
+  if (options.markdown === undefined) {
+    for (const p of options.path) {
+      if (path.extname(p).toLowerCase() === '.md') {
+        options.markdown = true;
+      }
+    }
+  }
+
+  // Figure out which directory should be used as the root for the web server,
+  // and how that impacts the path to the file for the first request.
+  if (!options.serverRoot && !isUrlType) {
+    // if the serverRoot wasn't defined, and there are multiple paths, just
+    // use process.cwd().
+    if (options.path.length > 1) {
+      options.serverRoot = process.cwd();
+    } else {
+      // if there's a single path, try to be smart and figure it out
+      const s = await stat(options.path[0]);
+      options.serverRoot = options.path[0];
+      if (s.isFile()) {
+        const pathParts = options.path[0].split(path.sep);
+        options.path = [path.sep + pathParts[pathParts.length - 1]];
+        options.serverRoot =
+          pathParts.slice(0, pathParts.length - 1).join(path.sep) || '.';
+      } else {
+        options.serverRoot = options.path[0];
+        options.path = '/';
+      }
+    }
+  }
+  return options;
+}

src/queue.ts

@@ -0,0 +1,52 @@
+import PQueue from 'p-queue';
+
+export interface QueueOptions {
+  concurrency?: number;
+}
+
+export interface QueueItemOptions {
+  delay?: number;
+}
+
+export type AsyncFunction = () => Promise<void>;
+
+export class Queue {
+  private q: PQueue;
+  private activeTimers = 0;
+
+  constructor(options: QueueOptions) {
+    this.q = new PQueue({
+      concurrency: options.concurrency,
+    });
+  }
+
+  add(fn: AsyncFunction, options?: QueueItemOptions) {
+    if (options?.delay) {
+      setTimeout(() => {
+        this.q.add(fn);
+        this.activeTimers--;
+      }, options.delay);
+      this.activeTimers++;
+    } else {
+      this.q.add(fn);
+    }
+  }
+
+  async onIdle() {
+    await this.q.onIdle();
+    await new Promise<void>(resolve => {
+      if (this.activeTimers === 0) {
+        resolve();
+        return;
+      }
+      const timer = setInterval(async () => {
+        if (this.activeTimers === 0 && this.q.size === 0) {
+          await this.q.onIdle();
+          clearInterval(timer);
+          resolve();
+          return;
+        }
+      }, 500);
+    });
+  }
+}

test/fixtures/retry/index.html

@@ -0,0 +1,7 @@
+<html>
+<body>
+<a href="http://fake.local/1">linky</a>
+<a href="http://fake.local/3">linky</a>
+<a href="subpage.html">subpage!</a>
+</body>
+</html>

test/fixtures/retry/subpage.html

@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="http://fake.local/2">linky</a>
+</body>
+</html>

test/fixtures/retryCLI/index.html

@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="http://localhost:3333">linky</a>
+</body>
+</html>

test/zcli.ts test/test.cli.ts

@@ -1,13 +1,28 @@
 import {describe, it} from 'mocha';
 import * as execa from 'execa';
 import {assert} from 'chai';
+import * as http from 'http';
+import * as util from 'util';
+import enableDestroy = require('server-destroy');
 import {LinkResult, LinkState} from '../src/index';
 
 describe('cli', () => {
+  let server: http.Server;
+
+  if (process.env.LINKINATOR_SKIP_CLI_TESTS) {
+    return;
+  }
+
   before(async () => {
     await execa('npm', ['link']);
   });
 
+  afterEach(async () => {
+    if (server) {
+      await util.promisify(server.destroy)();
+    }
+  });
+
   it('should show output for failures', async () => {
     const res = await execa('npx', ['linkinator', 'test/fixtures/basic'], {
       reject: false,
@@ -194,4 +209,35 @@ describe('cli', () => {
     ]);
     assert.strictEqual(res.exitCode, 0);
   });
+
+  it('should warn on retries', async () => {
+    // start a web server to return the 429
+    let requestCount = 0;
+    let firstRequestTime: number;
+    const port = 3333;
+    const delayMillis = 1000;
+    server = http.createServer((_, res) => {
+      if (requestCount === 0) {
+        res.writeHead(429, {
+          'retry-after': 1,
+        });
+        requestCount++;
+        firstRequestTime = Date.now();
+      } else {
+        assert.isAtLeast(Date.now(), firstRequestTime + delayMillis);
+        res.writeHead(200);
+      }
+      res.end();
+    });
+    enableDestroy(server);
+    await new Promise<void>(r => server.listen(port, r));
+
+    const res = await execa('npx', [
+      'linkinator',
+      '--retry',
+      'test/fixtures/retryCLI',
+    ]);
+    assert.strictEqual(res.exitCode, 0);
+    assert.include(res.stdout, `Retrying: http://localhost:${port}`);
+  });
 });

test/test.ts test/test.index.ts

@@ -0,0 +1,202 @@
+import {assert} from 'chai';
+import * as nock from 'nock';
+import * as sinon from 'sinon';
+import {describe, it, afterEach} from 'mocha';
+
+import {check, LinkChecker} from '../src';
+
+nock.disableNetConnect();
+nock.enableNetConnect('localhost');
+
+describe('retries', () => {
+  afterEach(() => {
+    sinon.restore();
+    nock.cleanAll();
+  });
+
+  it('should handle 429s with invalid retry-after headers', async () => {
+    const scope = nock('http://fake.local').get('/').reply(429, undefined, {
+      'retry-after': 'totally-not-valid',
+    });
+    const results = await check({
+      path: 'test/fixtures/basic',
+      retry: true,
+    });
+    assert.ok(!results.passed);
+    scope.done();
+  });
+
+  it('should retry 429s with second based header', async () => {
+    const scope = nock('http://fake.local')
+      .get('/')
+      .reply(429, undefined, {
+        'retry-after': '10',
+      })
+      .get('/')
+      .reply(200);
+
+    const {promise, resolve} = invertedPromise();
+    const checker = new LinkChecker().on('retry', resolve);
+    const clock = sinon.useFakeTimers();
+    const checkPromise = checker.check({
+      path: 'test/fixtures/basic',
+      retry: true,
+    });
+    await promise;
+    await clock.tickAsync(10_000);
+    const results = await checkPromise;
+    assert.ok(results.passed);
+    scope.done();
+  });
+
+  it('should retry 429s after failed HEAD', async () => {
+    const scope = nock('http://fake.local')
+      .head('/')
+      .reply(405)
+      .get('/')
+      .reply(429, undefined, {
+        'retry-after': '10',
+      })
+      .get('/')
+      .reply(200);
+
+    const {promise, resolve} = invertedPromise();
+    const checker = new LinkChecker().on('retry', resolve);
+    const clock = sinon.useFakeTimers();
+    const checkPromise = checker.check({
+      path: 'test/fixtures/basic',
+      retry: true,
+    });
+    await promise;
+    await clock.tickAsync(10000);
+    const results = await checkPromise;
+    assert.ok(results.passed);
+    scope.done();
+  });
+
+  it('should retry 429s with date based header', async () => {
+    const scope = nock('http://fake.local')
+      .get('/')
+      .reply(429, undefined, {
+        'retry-after': '1970-01-01T00:00:10.000Z',
+      })
+      .get('/')
+      .reply(200);
+
+    const {promise, resolve} = invertedPromise();
+    const checker = new LinkChecker().on('retry', resolve);
+    const clock = sinon.useFakeTimers();
+    const checkPromise = checker.check({
+      path: 'test/fixtures/basic',
+      retry: true,
+    });
+    await promise;
+    await clock.tickAsync(10000);
+    const results = await checkPromise;
+    assert.ok(results.passed);
+    scope.done();
+  });
+
+  it('should detect requests to wait on the same host', async () => {
+    const scope = nock('http://fake.local')
+      .get('/1')
+      .reply(429, undefined, {
+        'retry-after': '3',
+      })
+      .get('/1', () => {
+        assert.isAtLeast(Date.now(), 3000);
+        return true;
+      })
+      .reply(200)
+      .get('/2', () => {
+        assert.isAtLeast(Date.now(), 3000);
+        return true;
+      })
+      .reply(200)
+      .get('/3')
+      .reply(429, undefined, {
+        'retry-after': '3',
+      })
+      .get('/3', () => {
+        assert.isAtLeast(Date.now(), 3000);
+        return true;
+      })
+      .reply(200);
+
+    const {promise, resolve} = invertedPromise();
+    const checker = new LinkChecker().on('retry', resolve);
+    const clock = sinon.useFakeTimers();
+    const checkPromise = checker.check({
+      path: 'test/fixtures/retry',
+      recurse: true,
+      retry: true,
+    });
+    await promise;
+    await clock.tickAsync(3000);
+    const results = await checkPromise;
+    assert.ok(results.passed);
+    scope.done();
+  });
+
+  it('should increase timeout for followup requests to a host', async () => {
+    const scope = nock('http://fake.local')
+      .get('/1')
+      .reply(429, undefined, {
+        'retry-after': '3',
+      })
+      .get('/1', () => {
+        // even though the header said to wait 3 seconds, we are checking to
+        // make sure the /3 route reset it to 9 seconds here. This is common
+        // when a flood of requests come through and the retry-after gets
+        // extended.
+        assert.isAtLeast(Date.now(), 9000);
+        return true;
+      })
+      .reply(200)
+      .get('/2', () => {
+        assert.isAtLeast(Date.now(), 9000);
+        return true;
+      })
+      .reply(200)
+      .get('/3')
+      .reply(429, undefined, {
+        'retry-after': '9',
+      })
+      .get('/3', () => {
+        assert.isAtLeast(Date.now(), 9000);
+        return true;
+      })
+      .reply(200);
+
+    const {promise: p1, resolve: r1} = invertedPromise();
+    const {promise: p2, resolve: r2} = invertedPromise();
+    const checker = new LinkChecker().on('retry', info => {
+      if (info.url === 'http://fake.local/1') {
+        r1();
+      } else if (info.url === 'http://fake.local/3') {
+        r2();
+      }
+    });
+    const clock = sinon.useFakeTimers();
+    const checkPromise = checker.check({
+      path: 'test/fixtures/retry',
+      recurse: true,
+      retry: true,
+    });
+    await Promise.all([p1, p2]);
+    await clock.tickAsync(9000);
+    const results = await checkPromise;
+    assert.ok(results.passed);
+    scope.done();
+  });
+
+  function invertedPromise() {
+    let resolve!: () => void;
+    let reject!: (err: Error) => void;
+    const promise = new Promise<void>((innerResolve, innerReject) => {
+      resolve = innerResolve;
+      reject = innerReject;
+    });
+    return {promise, resolve, reject};
+  }
+});