fix!: stream CSV results (#507)

Author: JustinBeckwith

package-lock.json

@@ -28,18 +28,15 @@
     "gaxios": "^5.0.0",
     "glob": "^8.0.1",
     "htmlparser2": "^8.0.1",
-    "jsonexport": "^3.2.0",
     "marked": "^4.0.3",
     "meow": "^10.1.1",
     "mime": "^3.0.0",
     "server-destroy": "^1.0.1",
     "update-notifier": "^6.0.0"
   },
   "devDependencies": {
-    "@types/chai": "^4.2.22",
     "@types/escape-html": "^1.0.1",
     "@types/glob": "^7.2.0",
-    "@types/jsonexport": "^3.0.2",
     "@types/marked": "^4.0.0",
     "@types/mime": "^2.0.3",
     "@types/mocha": "^9.0.0",
@@ -48,7 +45,6 @@
     "@types/sinon": "^10.0.6",
     "@types/update-notifier": "^6.0.0",
     "c8": "^7.10.0",
-    "chai": "^4.3.4",
     "codecov": "^3.8.2",
     "execa": "^6.0.0",
     "gts": "^4.0.0",

package.json

@@ -3,7 +3,6 @@
 import meow from 'meow';
 import updateNotifier from 'update-notifier';
 import chalk from 'chalk';
-import jsonexport from 'jsonexport';
 import fs from 'fs';
 import {URL} from 'url';
 import {Flags, getConfig} from './config.js';
@@ -147,6 +146,10 @@ async function main() {
   logger.error(`🏊‍♂️ crawling ${cli.input}`);
 
   const checker = new LinkChecker();
+  if (format === Format.CSV) {
+    const header = 'url,status,state,parent,failureDetails';
+    console.log(header);
+  }
   checker.on('retry', (info: RetryInfo) => {
     logger.warn(`Retrying: ${info.url} in ${info.secondsUntilRetry} seconds.`);
   });
@@ -166,6 +169,16 @@ async function main() {
         logger.info(`${state} ${chalk.gray(link.url)}`);
         break;
     }
+    if (format === Format.CSV) {
+      const showIt = shouldShowResult(link, verbosity);
+      if (showIt) {
+        console.log(
+          `"${link.url}",${link.status},${link.state},"${link.parent || ''}","${
+            link.failureDetails || ''
+          }"`
+        );
+      }
+    }
   });
   const opts: CheckOptions = {
     path: cli.input,
@@ -203,27 +216,14 @@ async function main() {
     ];
   }
   const result = await checker.check(opts);
-  const filteredResults = result.links.filter(link => {
-    switch (link.state) {
-      case LinkState.OK:
-        return verbosity <= LogLevel.WARNING;
-      case LinkState.BROKEN:
-        if (verbosity > LogLevel.DEBUG) {
-          link.failureDetails = undefined;
-        }
-        return verbosity <= LogLevel.ERROR;
-      case LinkState.SKIPPED:
-        return verbosity <= LogLevel.INFO;
-    }
-  });
+  const filteredResults = result.links.filter(link =>
+    shouldShowResult(link, verbosity)
+  );
   if (format === Format.JSON) {
     result.links = filteredResults;
     console.log(JSON.stringify(result, null, 2));
     return;
   } else if (format === Format.CSV) {
-    result.links = filteredResults;
-    const csv = await jsonexport(result.links);
-    console.log(csv);
     return;
   } else {
     // Build a collection scanned links, collated by the parent link used in
@@ -351,4 +351,18 @@ function parseFormat(flags: Flags): Format {
   return Format[flags.format as keyof typeof Format];
 }
 
+function shouldShowResult(link: LinkResult, verbosity: LogLevel) {
+  switch (link.state) {
+    case LinkState.OK:
+      return verbosity <= LogLevel.WARNING;
+    case LinkState.BROKEN:
+      if (verbosity > LogLevel.DEBUG) {
+        link.failureDetails = undefined;
+      }
+      return verbosity <= LogLevel.ERROR;
+    case LinkState.SKIPPED:
+      return verbosity <= LogLevel.INFO;
+  }
+}
+
 main();

src/cli.ts

@@ -1,6 +1,6 @@
 import {describe, it} from 'mocha';
 import {execa} from 'execa';
-import {assert} from 'chai';
+import assert from 'assert';
 import http from 'http';
 import util from 'util';
 import {URL} from 'url';
@@ -98,7 +98,7 @@ describe('cli', function () {
       'csv',
       'test/fixtures/markdown/README.md',
     ]);
-    assert.match(res.stdout, /README.md,200,OK,/);
+    assert.match(res.stdout, /README.md",200,OK,/);
   });
 
   it('should provide JSON if asked nicely', async () => {
@@ -118,7 +118,7 @@ describe('cli', function () {
       '--silent',
       'test/fixtures/markdown/README.md',
     ]);
-    assert.notMatch(res.stdout, /\[/);
+    assert.doesNotMatch(res.stdout, /\[/);
   });
 
   it('should not show 200 links if verbosity is ERROR with JSON', async () => {
@@ -274,7 +274,7 @@ describe('cli', function () {
         requestCount++;
         firstRequestTime = Date.now();
       } else {
-        assert.isAtLeast(Date.now(), firstRequestTime + delayMillis);
+        assert.ok(Date.now() >= firstRequestTime + delayMillis);
         res.writeHead(200);
       }
       res.end();
@@ -288,6 +288,6 @@ describe('cli', function () {
       'test/fixtures/retryCLI',
     ]);
     assert.strictEqual(res.exitCode, 0);
-    assert.include(res.stdout, `Retrying: http://localhost:${port}`);
+    assert.match(res.stdout, new RegExp(`Retrying: http://localhost:${port}/`));
   });
 });

test/test.cli.ts

@@ -1,4 +1,4 @@
-import {assert} from 'chai';
+import assert from 'assert';
 import nock from 'nock';
 import sinon from 'sinon';
 import {describe, it, afterEach} from 'mocha';
@@ -111,12 +111,12 @@ describe('retries', () => {
         'retry-after': '3',
       })
       .get('/1', () => {
-        assert.isAtLeast(Date.now(), 3000);
+        assert.ok(Date.now() >= 3000);
         return true;
       })
       .reply(200)
       .get('/2', () => {
-        assert.isAtLeast(Date.now(), 3000);
+        assert.ok(Date.now() >= 3000);
         return true;
       })
       .reply(200)
@@ -125,7 +125,7 @@ describe('retries', () => {
         'retry-after': '3',
       })
       .get('/3', () => {
-        assert.isAtLeast(Date.now(), 3000);
+        assert.ok(Date.now() >= 3000);
         return true;
       })
       .reply(200);
@@ -158,12 +158,12 @@ describe('retries', () => {
         // make sure the /3 route reset it to 9 seconds here. This is common
         // when a flood of requests come through and the retry-after gets
         // extended.
-        assert.isAtLeast(Date.now(), 9000);
+        assert.ok(Date.now() >= 9000);
         return true;
       })
       .reply(200)
       .get('/2', () => {
-        assert.isAtLeast(Date.now(), 9000);
+        assert.ok(Date.now() >= 9000);
         return true;
       })
       .reply(200)
@@ -172,7 +172,7 @@ describe('retries', () => {
         'retry-after': '9',
       })
       .get('/3', () => {
-        assert.isAtLeast(Date.now(), 9000);
+        assert.ok(Date.now() >= 9000);
         return true;
       })
       .reply(200);