Skip to content

Commit

Permalink
simplify test fixture handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@isaacs
isaacs committed Apr 24, 2020
1 parent e179501 commit 5102576
Show file tree
Hide file tree
Showing 11 changed files with 53 additions and 72 deletions.
9 changes: 0 additions & 9 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion package.json
View file
Expand Up @@ -23,7 +23,6 @@
"console-control-strings": "^1.1.0"
},
"devDependencies": {
"keyfob": "^1.0.0",
"tap": "^14.10.7"
},
"directories": {
Expand Down
26 changes: 13 additions & 13 deletions test/detail-report-test.js
View file
Expand Up @@ -2,67 +2,67 @@

const tap = require('tap')
const Report = require('../')
const fixtures = require('./lib/test-fixtures')
const fixtures = require('./fixtures')

tap.test('it generates a detail report with no vulns', async t => {
const report = Report(fixtures['no-vulns'], {reporter: 'detail', withColor: false})
const report = Report(fixtures('no-vulns'), {reporter: 'detail', withColor: false})
t.match(report.exitCode, 0, 'successful exit code')
t.match(report.report, /found 0 vulnerabilities/, 'no vulns reported')
t.match(report.report, /918 scanned packages/, 'reports scanned count')
})

tap.test('it generates a detail report with one vuln (update action)', async t => {
const report = Report(fixtures['one-vuln-one-pkg'], {reporter: 'detail'})
const report = Report(fixtures('one-vuln-one-pkg'), {reporter: 'detail'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /npm update tough-cookie --depth 6/, 'recommends update command with --depth')
t.match(report.report, /1 scanned package/, 'reports a single scanned pkg')
})

tap.test('it generates a detail report with one vuln (install action)', async t => {
const report = Report(fixtures['one-vuln-install'], {reporter: 'detail'})
const report = Report(fixtures('one-vuln-install'), {reporter: 'detail'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /npm install knex@3\.0\.0/, 'recommends install command')
})

tap.test('it adds a message if a dep isMajor (one vuln)', async t => {
const report = Report(fixtures['one-vuln-install-ismajor'], {reporter: 'detail', withColor: false})
const report = Report(fixtures('one-vuln-install-ismajor'), {reporter: 'detail', withColor: false})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /1 vulnerability requires semver-major dependency updates/, 'reports one semver-major bump')
})

tap.test('it adds a message if a dep isMajor (multiple vulns)', async t => {
const report = Report(fixtures['some-vulns-ismajor'], {reporter: 'detail', withColor: false})
const report = Report(fixtures('some-vulns-ismajor'), {reporter: 'detail', withColor: false})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /6 vulnerabilities require semver-major dependency updates/, 'reports multiple semver-major bumps')
})

tap.test('it generates a detail report with one vuln (install dev dep)', async t => {
const report = Report(fixtures['one-vuln-dev'], {reporter: 'detail'})
const report = Report(fixtures('one-vuln-dev'), {reporter: 'detail'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /npm install --save-dev knex@3\.0\.0/, 'adds --save-dev to recommendation')
})

tap.test('it generates a detail report with one vuln (review dev dep)', async t => {
const report = Report(fixtures['one-vuln-dev-review'], {reporter: 'detail'})
const report = Report(fixtures('one-vuln-dev-review'), {reporter: 'detail'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /knex \[dev\]/, 'mentions the dep and tags it as dev')
t.match(report.report, /Manual Review/, 'reports a manual review requirement')
})

tap.test('it generates a detail report with one vuln, no color', async t => {
const report = Report(fixtures['one-vuln'], {reporter: 'detail', withColor: false})
const report = Report(fixtures('one-vuln'), {reporter: 'detail', withColor: false})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /# Run {2}npm update tough-cookie --depth 6 {2}to resolve 1 vulnerability/, 'individual update command printed')
})

tap.test('it generates a detail report with one vuln, no unicode', async t => {
const report = Report(fixtures['one-vuln'], {reporter: 'detail', withUnicode: false})
const report = Report(fixtures('one-vuln'), {reporter: 'detail', withUnicode: false})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.notMatch(report.report, /┬/, 'prints a fancy table')
})

tap.test('it generates a detail report with some vulns', async t => {
const report = Report(fixtures['some-vulns'], {reporter: 'detail', withColor: false})
const report = Report(fixtures('some-vulns'), {reporter: 'detail', withColor: false})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /Manual Review/, 'expects manual review')
t.match(report.report, /found 12 vulnerabilities/, 'reports vuln count')
Expand All @@ -72,7 +72,7 @@ tap.test('it generates a detail report with some vulns', async t => {
})

tap.test('it generates a detail report with vulns of all severities', async t => {
const report = Report(fixtures['all-severity-vulns'], {reporter: 'detail', withColor: false})
const report = Report(fixtures('all-severity-vulns'), {reporter: 'detail', withColor: false})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /Manual Review/, 'expects manual review')
t.match(report.report, /found 31 vulnerabilities/, 'reports vuln count')
Expand All @@ -82,7 +82,7 @@ tap.test('it generates a detail report with vulns of all severities', async t =>
})

tap.test('it generates a detail report with review vulns, no unicode', async t => {
const report = Report(fixtures['update-review'], {reporter: 'detail', withUnicode: false, withColor: false})
const report = Report(fixtures('update-review'), {reporter: 'detail', withUnicode: false, withColor: false})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.notMatch(report.report, /┬/, 'unicode table not printed')
t.match(report.report, /Manual Review/, 'manual review reported')
Expand Down
4 changes: 4 additions & 0 deletions test/fixtures/index.js
View file
@@ -0,0 +1,4 @@
// not just using require, so that we get it fresh every time.
const { readFileSync } = require('fs')
const readJson = f => JSON.parse(readFileSync(f, 'utf8'))
module.exports = f => readJson(require.resolve(`./${f}.json`))
16 changes: 8 additions & 8 deletions test/install-report-test.js
View file
Expand Up @@ -2,47 +2,47 @@

const tap = require('tap')
const Report = require('../')
const fixtures = require('./lib/test-fixtures')
const fixtures = require('./fixtures')

tap.test('it generates an install report with no vulns', async t => {
const report = Report(fixtures['no-vulns'])
const report = Report(fixtures('no-vulns'))
t.match(report.report, /found .*0.* vulnerabilities/)
t.match(report.exitCode, 0)
})

tap.test('it generates an install report with no vulns, no colors', async t => {
const report = Report(fixtures['no-vulns'], {withColor: false})
const report = Report(fixtures('no-vulns'), {withColor: false})
t.match(report.report, /found 0 vulnerabilities/)
t.match(report.exitCode, 0)
})

tap.test('it generates an install report with one vuln', async t => {
const report = Report(fixtures['one-vuln'], {withColor: false})
const report = Report(fixtures('one-vuln'), {withColor: false})
t.match(report.report, /found 1 high severity vulnerability/)
t.match(report.exitCode, 1)
})

tap.test('recommend `npm audit fix` when install actions present', async t => {
const report = Report(fixtures['one-vuln'], {withColor: false})
const report = Report(fixtures('one-vuln'), {withColor: false})
t.match(report.report, /run `npm audit fix`/)
t.match(report.exitCode, 1)
})

tap.test('it generates an install report with multiple vulns of one type', async t => {
const report = Report(fixtures['some-same-type'], {withColor: false})
const report = Report(fixtures('some-same-type'), {withColor: false})
t.match(report.report, /found 12 high severity vulnerabilities/)
t.match(report.exitCode, 1)
})

tap.test('it generates an install report with more than one vuln', async t => {
const report = Report(fixtures['some-vulns'])
const report = Report(fixtures('some-vulns'))
t.match(report.report, /^found .*12.* vulnerabilities/)
t.match(report.report, /9 .*low.*, 3 .*high.*/)
t.match(report.exitCode, 1)
})

tap.test('it generates an install report with vulns of all severities', async t => {
const report = Report(fixtures['all-severity-vulns'])
const report = Report(fixtures('all-severity-vulns'))
t.match(report.report, /^found .*31.* vulnerabilities/)
t.match(report.report, /16 .*info, 8 .*low.*, 4 .*moderate.*, 2 .*high.*, 1 .*critical.*/)
t.match(report.exitCode, 1)
Expand Down
4 changes: 2 additions & 2 deletions test/json-report-test.js
View file
Expand Up @@ -2,10 +2,10 @@

const tap = require('tap')
const Report = require('../')
const fixtures = require('./lib/test-fixtures')
const fixtures = require('./fixtures')

tap.test('it generates a json report with zero for every severity', async t => {
const reportRaw = Report(fixtures['no-vulns'], {reporter: 'json'})
const reportRaw = Report(fixtures('no-vulns'), {reporter: 'json'})
const exitCode = reportRaw.exitCode
const vulnerabilities = JSON.parse(reportRaw.report).metadata.vulnerabilities

Expand Down
11 changes: 0 additions & 11 deletions test/lib/test-fixtures.js
View file

This file was deleted.

27 changes: 13 additions & 14 deletions test/parseable-report-test.js
View file
Expand Up @@ -2,63 +2,62 @@

const tap = require('tap')
const Report = require('../')
const Keyfob = require('keyfob')

const fixtures = Keyfob.load({ path: 'test/fixtures', fn: require })
const fixtures = require('./fixtures')

tap.test('it generates a parseable report with no vulns', async t => {
const report = Report(fixtures['no-vulns'], {reporter: 'parseable'})
const report = Report(fixtures('no-vulns'), {reporter: 'parseable'})
t.match(report.exitCode, 0, 'successful exit code')
t.equal(report.report.length, 0, 'no vulns reported')
})

tap.test('it generates a parseable report with one vuln (update action)', async t => {
const report = Report(fixtures['one-vuln-one-pkg'], {reporter: 'parseable'})
const report = Report(fixtures('one-vuln-one-pkg'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /\tnpm update tough-cookie --depth 6/, 'recommends update command with --depth')
})

tap.test('it generates a parseable report with one vuln (update action)', async t => {
const report = Report(fixtures['one-vuln'], {reporter: 'parseable'})
const report = Report(fixtures('one-vuln'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /^update/)
t.match(report.report, /npm update tough-cookie --depth 6/)
})

tap.test('it generates a parseable report with one vuln (install action)', async t => {
const report = Report(fixtures['one-vuln-install'], {reporter: 'parseable'})
const report = Report(fixtures('one-vuln-install'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /^install/)
t.match(report.report, /npm install knex@3.0.0/)
})

tap.test('it adds a message if a dep isMajor (one vuln)', async t => {
const report = Report(fixtures['one-vuln-install-ismajor'], {reporter: 'parseable'})
const report = Report(fixtures('one-vuln-install-ismajor'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /\tY\n/, 'reports one semver-major bump')
})

tap.test('it adds a message if a dep isMajor (multiple vulns)', async t => {
const report = Report(fixtures['some-vulns-ismajor'], {reporter: 'parseable'})
const report = Report(fixtures('some-vulns-ismajor'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /\tY\n[\s\w ->@]*\tY\n/, 'reports multiple semver-major bumps')
})

tap.test('it generates a parseable report with one vuln (install dev dep)', async t => {
const report = Report(fixtures['one-vuln-dev'], {reporter: 'parseable'})
const report = Report(fixtures('one-vuln-dev'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /npm install --save-dev knex@3.0.0/)
})

tap.test('it generates a parseable report with one vuln (review dev dep)', async t => {
const report = Report(fixtures['one-vuln-dev-review'], {reporter: 'parseable'})
const report = Report(fixtures('one-vuln-dev-review'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /review\t/, 'expects manual review')
t.match(report.report, /\tknex/)
})

tap.test('it generates a parseable report with some vulns', async t => {
const report = Report(fixtures['some-vulns'], {reporter: 'parseable'})
const report = Report(fixtures('some-vulns'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /review\t/, 'expects manual review')
t.match(report.report, /\tlow/)
Expand All @@ -68,19 +67,19 @@ tap.test('it generates a parseable report with some vulns', async t => {
})

tap.test('it generates a parseable report with review vulns', async t => {
const report = Report(fixtures['update-review'], {reporter: 'parseable'})
const report = Report(fixtures('update-review'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /review\t/, 'expects manual review')
})

tap.test('it generates a parseable report with critical vulns', async t => {
const report = Report(fixtures['some-vulns-critical'], {reporter: 'parseable'})
const report = Report(fixtures('some-vulns-critical'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /\tcritical/)
})

tap.test('it generates a parseable report with multiple resolves on the same update/install action', async t => {
const report = Report(fixtures['some-same-action'], {reporter: 'parseable'})
const report = Report(fixtures('some-same-action'), {reporter: 'parseable'})
t.equal(report.exitCode, 1, 'non-zero exit code')
t.match(report.report, /\tcritical/)
t.match(report.report, /\tlow/)
Expand Down
6 changes: 3 additions & 3 deletions test/quiet-report-test.js
View file
Expand Up @@ -2,15 +2,15 @@

const tap = require('tap')
const Report = require('../')
const fixtures = require('./lib/test-fixtures')
const fixtures = require('./fixtures')

tap.test('it generates an quiet report with no vulns', async t => {
const report = Report(fixtures['no-vulns'], {reporter: 'quiet'})
const report = Report(fixtures('no-vulns'), {reporter: 'quiet'})
t.match(report.report, '')
t.match(report.exitCode, 0)
})
tap.test('it generates an quiet report with one vuln', async t => {
const report = Report(fixtures['one-vuln'], {reporter: 'quiet'})
const report = Report(fixtures('one-vuln'), {reporter: 'quiet'})
t.match(report.report, '')
t.match(report.exitCode, 1)
})
15 changes: 7 additions & 8 deletions test/reporters-test.js
View file
Expand Up @@ -2,32 +2,31 @@

const tap = require('tap')
const Report = require('../')
const Keyfob = require('keyfob')
const {totalVulnCount, severities} = require('../lib/utils')
const fixtures = Keyfob.load({path: 'test/fixtures', fn: require})
const fixtures = require('./fixtures')

tap.test('total vuln count is 0 with no vulns', async t => {
const reportRaw = Report(fixtures['no-vulns'], {reporter: 'json'})
const reportRaw = Report(fixtures('no-vulns'), {reporter: 'json'})
t.equal(totalVulnCount(JSON.parse(reportRaw.report).metadata.vulnerabilities), 0)
})

tap.test('total vuln count is calculated with some vulns', async t => {
const reportRaw = Report(fixtures['some-vulns'], {reporter: 'json'})
const reportRaw = Report(fixtures('some-vulns'), {reporter: 'json'})
t.equal(totalVulnCount(JSON.parse(reportRaw.report).metadata.vulnerabilities), 12)
})

tap.test('total vuln count is calculated with all severity vulns', async t => {
const reportRaw = Report(fixtures['all-severity-vulns'], {reporter: 'json'})
const reportRaw = Report(fixtures('all-severity-vulns'), {reporter: 'json'})
t.equal(totalVulnCount(JSON.parse(reportRaw.report).metadata.vulnerabilities), 31)
})

tap.test('no severities with 0 vulns', async t => {
const reportRaw = Report(fixtures['no-vulns'], {reporter: 'json'})
const reportRaw = Report(fixtures('no-vulns'), {reporter: 'json'})
t.same(severities(JSON.parse(reportRaw.report).metadata.vulnerabilities), [])
})

tap.test('some severities with some vulns', async t => {
const reportRaw = Report(fixtures['some-vulns'], {reporter: 'json'})
const reportRaw = Report(fixtures('some-vulns'), {reporter: 'json'})
t.same(severities(JSON.parse(reportRaw.report).metadata.vulnerabilities),
[
['low', 9],
Expand All @@ -37,7 +36,7 @@ tap.test('some severities with some vulns', async t => {
})

tap.test('all severities with all vulns', async t => {
const reportRaw = Report(fixtures['all-severity-vulns'], {reporter: 'json'})
const reportRaw = Report(fixtures('all-severity-vulns'), {reporter: 'json'})
t.same(severities(JSON.parse(reportRaw.report).metadata.vulnerabilities),
[
['info', 16],
Expand Down
6 changes: 3 additions & 3 deletions test/test-fixtures-test.js
View file
@@ -1,10 +1,10 @@
'use strict'

const tap = require('tap')
const fixtures = require('./lib/test-fixtures')
const fixtures = require('./fixtures')

tap.test('a test should be able to modify a fixture', function (t) {
const fixture = fixtures['one-vuln']
const fixture = fixtures('one-vuln')
fixture.actions.splice(0, 1)
delete fixture.muted
fixture.metadata.vulnerabilities.high = 0
Expand All @@ -21,7 +21,7 @@ tap.test('a test should be able to modify a fixture', function (t) {
})

tap.test('a test should not be able to see how a previous test modified a fixture', function (t) {
const fixture = fixtures['one-vuln']
const fixture = fixtures('one-vuln')
t.equal(fixture.actions.length, 1)
t.is('muted' in fixture, true)
t.same(fixture.metadata.vulnerabilities, {
Expand Down

0 comments on commit 5102576

Please sign in to comment.