Update --on-port test to expect new percentiles output

README.md

@@ -205,6 +205,33 @@ Because an autocannon instance is an `EventEmitter`, it emits several events. th
 * `reqError`: Emitted in the case of a request error e.g. a timeout.
 * `error`: Emitted if there is an error during the setup phase of autocannon.
 
+### results
+
+The results object emitted by `done` and passed to the `autocannon()` callback has these properties:
+
+* `title`: Value of the `title` option passed to `autocannon()`.
+* `url`: The URL that was targeted.
+* `socketPath`: The UNIX Domain Socket or Windows Named Pipe that was targeted, or `undefined`.
+* `requests`: A histogram object containing statistics about the amount of requests that were sent per second.
+* `latency`: A histogram object containing statistics about response latency.
+* `throughput`: A histogram object containing statistics about the response data throughput per second.
+* `duration`: The amount of time the test took, **in seconds**.
+* `errors`: The number of connection errors (including timeouts) that occurred.
+* `timeouts`: The number of connection timeouts that occurred.
+* `start`: A Date object representing when the test started.
+* `finish`: A Date object representing when the test ended.
+* `connections`: The amount of connections used (value of `opts.connections`).
+* `pipelining`: The number of pipelined requests used per connection (value of `opts.pipelining`).
+* `non2xx`: The number of non-2xx response status codes received.
+
+The histogram objects for `requests`, `latency` and `throughput` are [hdr-histogram-percentiles-obj](https://github.com/thekemkid/hdr-histogram-percentiles-obj) objects and have this shape:
+
+* `min`: The lowest value for this statistic.
+* `max`: The highest value for this statistic.
+* `average`: The average (mean) value.
+* `stddev`: The standard deviation.
+* `p*`: The XXth percentile value for this statistic. The percentile properties are: `p2_5`, `p50`, `p75`, `p90`, `p97_5`, `p99`, `p99_9`, `p99_99`, `p99_999`.
+
 ### `Client` API
 
 This object is passed as the first parameter of both the `setupClient` function and the `response` event from an autocannon instance. You can use this to modify the requests you are sending while benchmarking. This is also an `EventEmitter`, with the events and their params listed below.

lib/progressTracker.js

@@ -75,27 +75,31 @@ function track (instance, opts) {
     // if the user doesn't want to render the table, we can just return early
     if (!opts.renderResultsTable) return
 
-    const out = table([
-      asColor(chalk.cyan, ['Stat', 'Avg', 'Stdev', 'Max']),
-      asRow(chalk.bold('Latency (ms)'), result.latency),
-      asRow(chalk.bold('Req/Sec'), result.requests),
-      asRow(chalk.bold('Bytes/Sec'), asBytes(result.throughput))
-    ], {
+    const tableOpts = {
       border: getBorderCharacters('void'),
       columnDefault: {
         paddingLeft: 0,
         paddingRight: 1
       },
       drawHorizontalLine: () => false
-    })
+    }
 
-    logToStream(out)
+    logToStream(table([
+      asColor(chalk.cyan, ['Stat', '2.5%', '50%', '97.5%', '99%', 'Avg', 'Stdev', 'Max']),
+      asLowRow(chalk.bold('Latency'), asMs(result.latency))
+    ], tableOpts))
+    logToStream(table([
+      asColor(chalk.cyan, ['Stat', '1%', '2.5%', '50%', '97.5%', 'Avg', 'Stdev', 'Min']),
+      asHighRow(chalk.bold('Req/Sec'), result.requests),
+      asHighRow(chalk.bold('Bytes/Sec'), asBytes(result.throughput))
+    ], tableOpts))
+    logToStream('Req/Bytes counts sampled once per second.\n')
 
     if (opts.renderLatencyTable) {
       const latency = table([
         asColor(chalk.cyan, ['Percentile', 'Latency (ms)'])
       ].concat(percentiles.map((perc) => {
-        const key = ('p' + perc).replace('.', '')
+        const key = `p${perc}`.replace('.', '_')
         return [
           chalk.bold('' + perc),
           result.latency[key]
@@ -160,21 +164,56 @@ function trackAmount (instance, opts, iOpts) {
   return progressBar
 }
 
-function asRow (name, stat) {
+// create a table row for stats where low values is better
+function asLowRow (name, stat) {
   return [
     name,
+    stat.p2_5,
+    stat.p50,
+    stat.p97_5,
+    stat.p99,
     stat.average,
     stat.stddev,
     typeof stat.max === 'string' ? stat.max : Math.floor(stat.max * 100) / 100
   ]
 }
 
+// create a table row for stats where high values is better
+function asHighRow (name, stat) {
+  return [
+    name,
+    stat.p1,
+    stat.p2_5,
+    stat.p50,
+    stat.p97_5,
+    stat.average,
+    stat.stddev,
+    typeof stat.min === 'string' ? stat.min : Math.floor(stat.min * 100) / 100
+  ]
+}
+
 function asColor (colorise, row) {
   return row.map((entry) => colorise(entry))
 }
 
+function asMs (stat) {
+  const result = Object.create(null)
+  Object.keys(stat).forEach((k) => {
+    result[k] = `${stat[k]} ms`
+  })
+  result.max = typeof stat.max === 'string' ? stat.max : `${Math.floor(stat.max * 100) / 100} ms`
+
+  return result
+}
+
 function asBytes (stat) {
   const result = Object.create(stat)
+
+  percentiles.forEach((p) => {
+    const key = `p${p}`.replace('.', '_')
+    result[key] = prettyBytes(stat[key])
+  })
+
   result.average = prettyBytes(stat.average)
   result.stddev = prettyBytes(stat.stddev)
   result.max = prettyBytes(stat.max)

lib/run.js

@@ -130,9 +130,9 @@ function run (opts, cb) {
         title: opts.title,
         url: opts.url,
         socketPath: opts.socketPath,
-        requests: histAsObj(requests, totalCompletedRequests),
+        requests: addPercentiles(requests, histAsObj(requests, totalCompletedRequests)),
         latency: addPercentiles(latencies, histAsObj(latencies)),
-        throughput: histAsObj(throughput, totalBytes),
+        throughput: addPercentiles(throughput, histAsObj(throughput, totalBytes)),
         errors: errors,
         timeouts: timeouts,
         duration: Math.round((Date.now() - startTime) / 1000),

package.json

@@ -7,7 +7,7 @@
     "autocannon": "autocannon.js"
   },
   "scripts": {
-    "test": "standard && tap test/*.test.js"
+    "test": "standard && tap --timeout 45 test/*.test.js"
   },
   "pre-commit": [
     "test"
@@ -47,7 +47,7 @@
     "color-support": "^1.1.1",
     "has-async-hooks": "^1.0.0",
     "hdr-histogram-js": "^1.1.4",
-    "hdr-histogram-percentiles-obj": "^1.2.0",
+    "hdr-histogram-percentiles-obj": "^2.0.0",
     "http-parser-js": "^0.4.13",
     "hyperid": "^1.4.1",
     "manage-path": "^2.0.0",

test/cli-ipc.test.js

@@ -13,11 +13,15 @@ const lines = [
   /Running 1s test @ http:\/\/example.com\/foo \([^)]*\)$/,
   /10 connections.*$/,
   /$/,
-  /Stat.*Avg.*Stdev.*Max.*$/,
-  /Latency \(ms\).*$/,
+  /Stat.*2\.5%.*50%.*97\.5%.*99%.*Avg.*Stdev.*Max.*$/,
+  /Latency.*$/,
+  /$/,
+  /Stat.*1%.*2\.5%.*50%.*97\.5%.*Avg.*Stdev.*Min.*$/,
   /Req\/Sec.*$/,
   /Bytes\/Sec.*$/,
   /$/,
+  /Req\/Bytes counts sampled once per second.*$/,
+  /$/,
   /.* requests in \d+s, .* read/
 ]
 

test/cli.test.js

@@ -10,11 +10,15 @@ const lines = [
   /Running 1s test @ .*$/,
   /10 connections.*$/,
   /$/,
-  /Stat.*Avg.*Stdev.*Max.*$/,
-  /Latency \(ms\).*$/,
+  /Stat.*2\.5%.*50%.*97\.5%.*99%.*Avg.*Stdev.*Max.*$/,
+  /Latency.*$/,
+  /$/,
+  /Stat.*1%.*2\.5%.*50%.*97\.5%.*Avg.*Stdev.*Min.*$/,
   /Req\/Sec.*$/,
   /Bytes\/Sec.*$/,
   /$/,
+  /Req\/Bytes counts sampled once per second.*$/,
+  /$/,
   /.* requests in \d+s, .* read/
 ]
 

test/envPort.test.js

@@ -10,11 +10,15 @@ const lines = [
   /Running 1s test @ .*$/,
   /10 connections.*$/,
   /$/,
-  /Stat.*Avg.*Stdev.*Max.*$/,
-  /Latency \(ms\).*$/,
+  /Stat.*2\.5%.*50%.*97\.5%.*99%.*Avg.*Stdev.*Max.*$/,
+  /Latency.*$/,
+  /$/,
+  /Stat.*1%.*2\.5%.*50%.*97\.5%.*Avg.*Stdev.*Min.*$/,
   /Req\/Sec.*$/,
   /Bytes\/Sec.*$/,
   /$/,
+  /Req\/Bytes counts sampled once per second.*$/,
+  /$/,
   /.* requests in \d+s, .* read/
 ]
 

test/run.test.js

@@ -24,26 +24,38 @@ test('run', (t) => {
     t.equal(result.pipelining, 1, 'pipelining is the default')
 
     t.ok(result.latency, 'latency exists')
-    t.ok(result.latency.average, 'latency.average exists')
-    t.ok(result.latency.stddev, 'latency.stddev exists')
+    t.type(result.latency.average, 'number', 'latency.average exists')
+    t.type(result.latency.stddev, 'number', 'latency.stddev exists')
     t.ok(result.latency.min >= 0, 'latency.min exists')
-    t.ok(result.latency.max, 'latency.max exists')
+    t.type(result.latency.max, 'number', 'latency.max exists')
+    t.type(result.latency.p2_5, 'number', 'latency.p2_5 (2.5%) exists')
+    t.type(result.latency.p50, 'number', 'latency.p50 (50%) exists')
+    t.type(result.latency.p97_5, 'number', 'latency.p97_5 (97.5%) exists')
+    t.type(result.latency.p99, 'number', 'latency.p99 (99%) exists')
 
     t.ok(result.requests, 'requests exists')
-    t.ok(result.requests.average, 'requests.average exists')
-    t.ok(result.requests.stddev, 'requests.stddev exists')
-    t.ok(result.requests.min, 'requests.min exists')
-    t.ok(result.requests.max, 'requests.max exists')
+    t.type(result.requests.average, 'number', 'requests.average exists')
+    t.type(result.requests.stddev, 'number', 'requests.stddev exists')
+    t.type(result.requests.min, 'number', 'requests.min exists')
+    t.type(result.requests.max, 'number', 'requests.max exists')
     t.ok(result.requests.total >= result.requests.average * 2 / 100 * 95, 'requests.total exists')
-    t.ok(result.requests.sent, 'sent exists')
+    t.type(result.requests.sent, 'number', 'sent exists')
     t.ok(result.requests.sent >= result.requests.total, 'total requests made should be more than or equal to completed requests total')
+    t.type(result.requests.p1, 'number', 'requests.p1 (1%) exists')
+    t.type(result.requests.p2_5, 'number', 'requests.p2_5 (2.5%) exists')
+    t.type(result.requests.p50, 'number', 'requests.p50 (50%) exists')
+    t.type(result.requests.p97_5, 'number', 'requests.p97_5 (97.5%) exists')
 
     t.ok(result.throughput, 'throughput exists')
-    t.ok(result.throughput.average, 'throughput.average exists')
+    t.type(result.throughput.average, 'number', 'throughput.average exists')
     t.type(result.throughput.stddev, 'number', 'throughput.stddev exists')
-    t.ok(result.throughput.min, 'throughput.min exists')
-    t.ok(result.throughput.max, 'throughput.max exists')
+    t.type(result.throughput.min, 'number', 'throughput.min exists')
+    t.type(result.throughput.max, 'number', 'throughput.max exists')
     t.ok(result.throughput.total >= result.throughput.average * 2 / 100 * 95, 'throughput.total exists')
+    t.type(result.throughput.p1, 'number', 'throughput.p1 (1%) exists')
+    t.type(result.throughput.p2_5, 'number', 'throughput.p2_5 (2.5%) exists')
+    t.type(result.throughput.p50, 'number', 'throughput.p50 (50%) exists')
+    t.type(result.throughput.p97_5, 'number', 'throughput.p97_5 (97.5%) exists')
 
     t.ok(result.start, 'start time exists')
     t.ok(result.finish, 'finish time exists')
@@ -75,26 +87,38 @@ test('tracker.stop()', (t) => {
     t.equal(result.pipelining, 1, 'pipelining is the default')
 
     t.ok(result.latency, 'latency exists')
-    t.ok(result.latency.average, 'latency.average exists')
-    t.ok(result.latency.stddev, 'latency.stddev exists')
+    t.type(result.latency.average, 'number', 'latency.average exists')
+    t.type(result.latency.stddev, 'number', 'latency.stddev exists')
     t.ok(result.latency.min >= 0, 'latency.min exists')
-    t.ok(result.latency.max, 'latency.max exists')
+    t.type(result.latency.max, 'number', 'latency.max exists')
+    t.type(result.latency.p2_5, 'number', 'latency.p2_5 (2.5%) exists')
+    t.type(result.latency.p50, 'number', 'latency.p50 (50%) exists')
+    t.type(result.latency.p97_5, 'number', 'latency.p97_5 (97.5%) exists')
+    t.type(result.latency.p99, 'number', 'latency.p99 (99%) exists')
 
     t.ok(result.requests, 'requests exists')
-    t.ok(result.requests.average, 'requests.average exists')
-    t.ok(result.requests.stddev, 'requests.stddev exists')
-    t.ok(result.requests.min, 'requests.min exists')
-    t.ok(result.requests.max, 'requests.max exists')
+    t.type(result.requests.average, 'number', 'requests.average exists')
+    t.type(result.requests.stddev, 'number', 'requests.stddev exists')
+    t.type(result.requests.min, 'number', 'requests.min exists')
+    t.type(result.requests.max, 'number', 'requests.max exists')
     t.ok(result.requests.total >= result.requests.average * 2 / 100 * 95, 'requests.total exists')
-    t.ok(result.requests.sent, 'sent exists')
+    t.type(result.requests.sent, 'number', 'sent exists')
     t.ok(result.requests.sent >= result.requests.total, 'total requests made should be more than or equal to completed requests total')
+    t.type(result.requests.p1, 'number', 'requests.p1 (1%) exists')
+    t.type(result.requests.p2_5, 'number', 'requests.p2_5 (2.5%) exists')
+    t.type(result.requests.p50, 'number', 'requests.p50 (50%) exists')
+    t.type(result.requests.p97_5, 'number', 'requests.p97_5 (97.5%) exists')
 
     t.ok(result.throughput, 'throughput exists')
-    t.ok(result.throughput.average, 'throughput.average exists')
+    t.type(result.throughput.average, 'number', 'throughput.average exists')
     t.type(result.throughput.stddev, 'number', 'throughput.stddev exists')
-    t.ok(result.throughput.min, 'throughput.min exists')
-    t.ok(result.throughput.max, 'throughput.max exists')
+    t.type(result.throughput.min, 'number', 'throughput.min exists')
+    t.type(result.throughput.max, 'number', 'throughput.max exists')
     t.ok(result.throughput.total >= result.throughput.average * 2 / 100 * 95, 'throughput.total exists')
+    t.type(result.throughput.p1, 'number', 'throughput.p1 (1%) exists')
+    t.type(result.throughput.p2_5, 'number', 'throughput.p2_5 (2.5%) exists')
+    t.type(result.throughput.p50, 'number', 'throughput.p50 (50%) exists')
+    t.type(result.throughput.p97_5, 'number', 'throughput.p97_5 (97.5%) exists')
 
     t.ok(result.start, 'start time exists')
     t.ok(result.finish, 'finish time exists')
@@ -279,18 +303,26 @@ for (let i = 1; i <= 5; i++) {
 
       t.ok(result.latency, 'latency exists')
       t.ok(!Number.isNaN(result.latency.average), 'latency.average is not NaN')
-      t.ok(result.latency.average, 'latency.average exists')
-      t.ok(result.latency.stddev, 'latency.stddev exists')
+      t.type(result.latency.average, 'number', 'latency.average exists')
+      t.type(result.latency.stddev, 'number', 'latency.stddev exists')
       t.ok(result.latency.min >= 0, 'latency.min exists')
-      t.ok(result.latency.max, 'latency.max exists')
+      t.type(result.latency.max, 'number', 'latency.max exists')
+      t.type(result.latency.p2_5, 'number', 'latency.p2_5 (2.5%) exists')
+      t.type(result.latency.p50, 'number', 'latency.p50 (50%) exists')
+      t.type(result.latency.p97_5, 'number', 'latency.p97_5 (97.5%) exists')
+      t.type(result.latency.p99, 'number', 'latency.p99 (99%) exists')
 
       t.ok(result.throughput, 'throughput exists')
       t.ok(!Number.isNaN(result.throughput.average), 'throughput.average is not NaN')
-      t.ok(result.throughput.average, 'throughput.average exists')
+      t.type(result.throughput.average, 'number', 'throughput.average exists')
       t.type(result.throughput.stddev, 'number', 'throughput.stddev exists')
-      t.ok(result.throughput.min, 'throughput.min exists')
-      t.ok(result.throughput.max, 'throughput.max exists')
+      t.type(result.throughput.min, 'number', 'throughput.min exists')
+      t.type(result.throughput.max, 'number', 'throughput.max exists')
       t.ok(result.throughput.total >= result.throughput.average * 2 / 100 * 95, 'throughput.total exists')
+      t.type(result.throughput.p1, 'number', 'throughput.p1 (1%) exists')
+      t.type(result.throughput.p2_5, 'number', 'throughput.p2_5 (2.5%) exists')
+      t.type(result.throughput.p50, 'number', 'throughput.p50 (50%) exists')
+      t.type(result.throughput.p97_5, 'number', 'throughput.p97_5 (97.5%) exists')
 
       t.end()
     })
@@ -333,13 +365,22 @@ test('run will exclude non 2xx stats from latency and throughput averages if exc
     t.equal(result.latency.stddev, 0, 'latency.stddev should be 0')
     t.equal(result.latency.min, 0, 'latency.min should be 0')
     t.equal(result.latency.max, 0, 'latency.max should be 0')
+    t.equal(result.latency.p1, 0, 'latency.p1 (1%) should be 0')
+    t.equal(result.latency.p2_5, 0, 'latency.p2_5 (2.5%) should be 0')
+    t.equal(result.latency.p50, 0, 'latency.p50 (50%) should be 0')
+    t.equal(result.latency.p97_5, 0, 'latency.p97_5 (97.5%) should be 0')
+    t.equal(result.latency.p99, 0, 'latency.p99 (99%) should be 0')
 
     t.ok(result.throughput, 'throughput exists')
     t.equal(result.throughput.average, 0, 'throughput.average should be 0')
     t.equal(result.throughput.stddev, 0, 'throughput.stddev should be 0')
     t.equal(result.throughput.min, 0, 'throughput.min should be 0')
     t.equal(result.throughput.max, 0, 'throughput.max should be 0')
     t.equal(result.throughput.total, 0, 'throughput.total should be 0')
+    t.equal(result.throughput.p1, 0, 'throughput.p1 (1%) should be 0')
+    t.equal(result.throughput.p2_5, 0, 'throughput.p2_5 (2.5%) should be 0')
+    t.equal(result.throughput.p50, 0, 'throughput.p50 (50%) should be 0')
+    t.equal(result.throughput.p97_5, 0, 'throughput.p97_5 (97.5%) should be 0')
 
     t.end()
   })