feat: support directory listings (#225)

In addition to providing the directory listing flag, this swaps the underlying HTTP server from `serve-static` to `serve-handler`.  There should be no user facing changes for that swap.

Author: JustinBeckwith

README.md

@@ -45,6 +45,10 @@ $ linkinator LOCATIONS [ --arguments ]
     --config
         Path to the config file to use. Looks for `linkinator.config.json` by default.
 
+    --directory-listing
+        Include an automatic directory index file when linking to a directory.
+        Defaults to 'false'.
+
     --format, -f
         Return the data in CSV or JSON format.
 
@@ -138,6 +142,7 @@ All options are optional. It should look like this:
   "concurrency": 100,
   "timeout": 0,
   "markdown": true,
+  "directoryListing": true,
   "skip": "www.googleapis.com"
 }
 ```
@@ -161,6 +166,7 @@ where the server is started.  Defaults to the path passed in `path`.
 - `timeout` (number) - By default, requests made by linkinator do not time out (or follow the settings of the OS).  This option (in milliseconds) will fail requests after the configured amount of time.
 - `markdown` (boolean) - Automatically parse and scan markdown if scanning from a location on disk.
 - `linksToSkip` (array | function) - An array of regular expression strings that should be skipped, OR an async function that's called for each link with the link URL as its only argument. Return a Promise that resolves to `true` to skip the link or `false` to check it.
+- `directoryListing` (boolean) - Automatically serve a static file listing page when serving a directory.  Defaults to `false`.
 
 #### linkinator.LinkChecker()
 Constructor method that can be used to create a new `LinkChecker` instance.  This is particularly useful if you want to receive events as the crawler crawls.  Exposes the following events:

package.json

@@ -23,27 +23,25 @@
   "dependencies": {
     "chalk": "^4.0.0",
     "cheerio": "^1.0.0-rc.2",
-    "finalhandler": "^1.1.2",
     "gaxios": "^4.0.0",
     "glob": "^7.1.6",
     "jsonexport": "^3.0.0",
     "marked": "^1.2.5",
     "meow": "^8.0.0",
     "p-queue": "^6.2.1",
-    "serve-static": "^1.14.1",
+    "serve-handler": "^6.1.3",
     "server-destroy": "^1.0.1",
     "update-notifier": "^5.0.0"
   },
   "devDependencies": {
     "@types/chai": "^4.2.7",
     "@types/cheerio": "0.22.22",
-    "@types/finalhandler": "^1.1.0",
     "@types/glob": "^7.1.3",
     "@types/marked": "^1.2.0",
     "@types/meow": "^5.0.0",
     "@types/mocha": "^8.0.0",
     "@types/node": "^12.7.12",
-    "@types/serve-static": "^1.13.8",
+    "@types/serve-handler": "^6.1.0",
     "@types/server-destroy": "^1.0.0",
     "@types/sinon": "^9.0.0",
     "@types/update-notifier": "^5.0.0",

src/cli.ts

@@ -35,6 +35,10 @@ const cli = meow(
       --config
           Path to the config file to use. Looks for \`linkinator.config.json\` by default.
 
+      --directory-listing
+          Include an automatic directory index file when linking to a directory.
+          Defaults to 'false'.
+
       --format, -f
           Return the data in CSV or JSON format.
 
@@ -80,6 +84,7 @@ const cli = meow(
       markdown: {type: 'boolean'},
       serverRoot: {type: 'string'},
       verbosity: {type: 'string'},
+      directoryListing: {type: 'boolean'},
     },
     booleanDefault: undefined,
   }
@@ -126,6 +131,7 @@ async function main() {
     markdown: flags.markdown,
     concurrency: Number(flags.concurrency),
     serverRoot: flags.serverRoot,
+    directoryListing: flags.directoryListing,
   };
   if (flags.skip) {
     if (typeof flags.skip === 'string') {

src/config.ts

@@ -13,6 +13,7 @@ export interface Flags {
   timeout?: number;
   markdown?: boolean;
   serverRoot?: string;
+  directoryListing?: boolean;
 }
 
 export async function getConfig(flags: Flags) {

src/index.ts

@@ -25,6 +25,7 @@ export interface CheckOptions {
   markdown?: boolean;
   linksToSkip?: string[] | ((link: string) => Promise<boolean>);
   serverRoot?: string;
+  directoryListing?: boolean;
 }
 
 export enum LinkState {
@@ -82,11 +83,12 @@ export class LinkChecker extends EventEmitter {
     const hasHttpPaths = options.path.find(x => x.startsWith('http'));
     if (!hasHttpPaths) {
       const port = options.port || 5000 + Math.round(Math.random() * 1000);
-      server = await startWebServer(
-        options.serverRoot!,
+      server = await startWebServer({
+        root: options.serverRoot!,
         port,
-        options.markdown
-      );
+        markdown: options.markdown,
+        directoryListing: options.directoryListing,
+      });
       for (let i = 0; i < options.path.length; i++) {
         if (options.path[i].startsWith('/')) {
           options.path[i] = options.path[i].slice(1);
@@ -150,6 +152,11 @@ export class LinkChecker extends EventEmitter {
       options.path = [options.path];
     }
 
+    // disable directory listings by default
+    if (options.directoryListing === undefined) {
+      options.directoryListing = false;
+    }
+
     // Ensure we do not mix http:// and file system paths.  The paths passed in
     // must all be filesystem paths, or HTTP paths.
     let isUrlType: boolean | undefined = undefined;

src/server.ts

@@ -3,34 +3,36 @@ import * as path from 'path';
 import * as util from 'util';
 import * as fs from 'fs';
 import * as marked from 'marked';
-import finalhandler = require('finalhandler');
-import serveStatic = require('serve-static');
+import serve = require('serve-handler');
 import enableDestroy = require('server-destroy');
 
 const readFile = util.promisify(fs.readFile);
 
+export interface WebServerOptions {
+  // The local path that should be mounted as a static web server
+  root: string;
+  // The port on which to start the local web server
+  port: number;
+  // If markdown should be automatically compiled and served
+  markdown?: boolean;
+  // Should directories automatically serve an inde page
+  directoryListing?: boolean;
+}
+
 /**
  * Spin up a local HTTP server to serve static requests from disk
- * @param root The local path that should be mounted as a static web server
- * @param port The port on which to start the local web server
- * @param markdown If markdown should be automatically compiled and served
  * @private
  * @returns Promise that resolves with the instance of the HTTP server
  */
-export async function startWebServer(
-  root: string,
-  port: number,
-  markdown?: boolean
-) {
+export async function startWebServer(options: WebServerOptions) {
   return new Promise<http.Server>((resolve, reject) => {
-    const serve = serveStatic(root);
     const server = http
       .createServer(async (req, res) => {
         const pathParts = req.url!.split('/').filter(x => !!x);
         if (pathParts.length > 0) {
           const ext = path.extname(pathParts[pathParts.length - 1]);
-          if (markdown && ext.toLowerCase() === '.md') {
-            const filePath = path.join(path.resolve(root), req.url!);
+          if (options.markdown && ext.toLowerCase() === '.md') {
+            const filePath = path.join(path.resolve(options.root), req.url!);
             const data = await readFile(filePath, {encoding: 'utf-8'});
             const result = marked(data, {gfm: true});
             res.writeHead(200, {
@@ -40,9 +42,12 @@ export async function startWebServer(
             return;
           }
         }
-        return serve(req, res, finalhandler(req, res) as () => void);
+        return serve(req, res, {
+          public: options.root,
+          directoryListing: options.directoryListing,
+        });
       })
-      .listen(port, () => resolve(server))
+      .listen(options.port, () => resolve(server))
       .on('error', reject);
     enableDestroy(server);
   });

test/fixtures/config/linkinator.config.json

@@ -3,5 +3,6 @@
   "recurse": true,
   "silent": true,
   "concurrency": 17,
-  "skip": "🌳"
+  "skip": "🌳",
+  "directoryListing": false
 }

test/fixtures/directoryIndex/README.md

@@ -0,0 +1 @@
+This has links to a [directory with files](dir1/) and an [empty directory](dir2/).

test/fixtures/directoryIndex/dir1/dir1.md

@@ -0,0 +1 @@
+👋

test/fixtures/directoryIndex/dir2/dir2.md

@@ -480,4 +480,21 @@ describe('linkinator', () => {
     scope.done();
     assert.strictEqual(results.links.length, 2);
   });
+
+  it('should support directory index', async () => {
+    const results = await check({
+      path: 'test/fixtures/directoryIndex/README.md',
+      directoryListing: true,
+    });
+    assert.ok(results.passed);
+    assert.strictEqual(results.links.length, 3);
+  });
+
+  it('should disabling directory index by default', async () => {
+    const results = await check({
+      path: 'test/fixtures/directoryIndex/README.md',
+    });
+    assert.ok(!results.passed);
+    assert.strictEqual(results.links.length, 3);
+  });
 });