feat: add initial support for brotli

lib/pack.js

@@ -91,6 +91,15 @@ const Pack = warner(class Pack extends Minipass {
       this.zip.on('end', _ => super.end())
       this.zip.on('drain', _ => this[ONDRAIN]())
       this.on('resume', _ => this.zip.resume())
+    } else if (opt.brotli) {
+      if (typeof opt.brotli !== 'object') {
+        opt.brotli = {}
+      }
+      this.zip = new zlib.BrotliCompress(opt.brotli)
+      this.zip.on('data', chunk => super.write(chunk))
+      this.zip.on('end', _ => super.end())
+      this.zip.on('drain', _ => this[ONDRAIN]())
+      this.on('resume', _ => this.zip.resume())
     } else {
       this.on('drain', this[ONDRAIN])
     }

lib/parse.js

@@ -97,6 +97,9 @@ module.exports = warner(class Parser extends EE {
     this.strict = !!opt.strict
     this.maxMetaEntrySize = opt.maxMetaEntrySize || maxMetaEntrySize
     this.filter = typeof opt.filter === 'function' ? opt.filter : noop
+    // Unlike gzip, brotli doesn't have any magic bytes to identify it
+    // Users need to explicitly tell us they're extracting a brotli file
+    this.brotli = opt.brotli
 
     // have to set this so that streams are ok piping into it
     this.writable = true
@@ -356,6 +359,21 @@ module.exports = warner(class Parser extends EE {
         this[BUFFER] = chunk
         return true
       }
+      if (this[UNZIP] === null && this.brotli) {
+        const ended = this[ENDED]
+        this[ENDED] = false
+        this[UNZIP] = new zlib.BrotliDecompress()
+        this[UNZIP].on('data', chunk => this[CONSUMECHUNK](chunk))
+        this[UNZIP].on('error', er => this.abort(er))
+        this[UNZIP].on('end', _ => {
+          this[ENDED] = true
+          this[CONSUMECHUNK]()
+        })
+        this[WRITING] = true
+        const ret = this[UNZIP][ended ? 'end' : 'write'](chunk)
+        this[WRITING] = false
+        return ret
+      }
       for (let i = 0; this[UNZIP] === null && i < gzipHeader.length; i++) {
         if (chunk[i] !== gzipHeader[i]) {
           this[UNZIP] = false

lib/replace.js

@@ -23,7 +23,7 @@ module.exports = (opt_, files, cb) => {
     throw new TypeError('file is required')
   }
 
-  if (opt.gzip) {
+  if (opt.gzip || opt.brotli || opt.file.endsWith('.br') || opt.file.endsWith('.tbr')) {
     throw new TypeError('cannot append to compressed archives')
   }
 

lib/update.js

@@ -13,7 +13,7 @@ module.exports = (opt_, files, cb) => {
     throw new TypeError('file is required')
   }
 
-  if (opt.gzip) {
+  if (opt.gzip || opt.brotli || opt.file.endsWith('.br') || opt.file.endsWith('.tbr')) {
     throw new TypeError('cannot append to compressed archives')
   }
 

test/extract.js

@@ -310,3 +310,30 @@ t.test('sync gzip error edge case test', async t => {
 
   t.end()
 })
+
+t.test('brotli', async t => {
+  const file = path.resolve(__dirname, 'fixtures/example.tbr')
+  const dir = path.resolve(__dirname, 'brotli')
+
+  t.beforeEach(async () => {
+    await mkdirp(dir)
+  })
+
+  t.afterEach(async () => {
+    await rimraf(dir)
+  })
+
+  t.test('fails if brotli', async t => {
+    const expect = new Error("TAR_BAD_ARCHIVE: Unrecognized archive format")
+    t.throws(_ => x({ sync: true, file: file }), expect)
+  })
+
+  t.test('succeeds', t => {
+    x({ sync: true, file: file, C: dir, brotli: true })
+
+    t.same(fs.readdirSync(dir + '/x').sort(),
+        ['1', '10', '2', '3', '4', '5', '6', '7', '8', '9'])
+    t.end()
+  })
+})
+

test/pack.js

@@ -375,6 +375,13 @@ t.test('if gzip is truthy, make it an object', t => {
   t.end()
 })
 
+t.test('if brotli is truthy, make it an object', t => {
+    const opt = { brotli: true }
+    new Pack(opt)
+    t.type(opt.brotli, 'object')
+    t.end()
+})
+
 t.test('gzip, also a very deep path', t => {
   const out = []
 
@@ -454,6 +461,85 @@ t.test('gzip, also a very deep path', t => {
     })
 })
 
+t.test('brotli, also a very deep path', t => {
+    const out = []
+
+    new Pack({
+        cwd: files,
+        brotli: { flush: 1 },
+    })
+        .add('dir')
+        .add('long-path')
+        .on('data', c => out.push(c))
+        .end()
+        .on('end', _ => {
+            const zipped = Buffer.concat(out)
+            const data = zlib.brotliDecompressSync(zipped)
+            const entries = []
+            for (var i = 0; i < data.length; i += 512) {
+                const slice = data.slice(i, i + 512)
+                const h = new Header(slice)
+                if (h.nullBlock) {
+                    entries.push('null block')
+                } else if (h.cksumValid) {
+                    entries.push([h.type, h.path])
+                } else if (entries[entries.length - 1][0] === 'File') {
+                    entries[entries.length - 1].push(slice.toString().replace(/\0.*$/, ''))
+                }
+            }
+
+            const expect = [
+                ['Directory', 'dir/'],
+                ['Directory', 'long-path/'],
+                ['File', 'dir/x'],
+                ['Directory', 'long-path/r/'],
+                ['Directory', 'long-path/r/e/'],
+                ['Directory', 'long-path/r/e/a/'],
+                ['Directory', 'long-path/r/e/a/l/'],
+                ['Directory', 'long-path/r/e/a/l/l/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/'],
+                ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/'],
+                ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/a.txt', 'short\n'],
+                ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'],
+                ['ExtendedHeader', 'PaxHeader/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'],
+                ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222'],
+                ['ExtendedHeader', 'PaxHeader/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccc'],
+                ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccccccccccccc', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'],
+                ['ExtendedHeader', 'PaxHeader/Ω.txt'],
+                ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/Ω.txt', 'Ω'],
+                'null block',
+                'null block',
+            ]
+
+            let ok = true
+            entries.forEach((entry, i) => {
+                ok = ok &&
+                    t.equal(entry[0], expect[i][0]) &&
+                    t.equal(entry[1], expect[i][1]) &&
+                    (!entry[2] || t.equal(entry[2], expect[i][2]))
+            })
+
+            // t.match(entries, expect)
+            t.end()
+        })
+})
+
 t.test('very deep gzip path, sync', t => {
   const pack = new PackSync({
     cwd: files,
@@ -533,6 +619,85 @@ t.test('very deep gzip path, sync', t => {
   t.end()
 })
 
+t.test('very deep brotli path, sync', t => {
+    const pack = new PackSync({
+        cwd: files,
+        brotli: true,
+    }).add('dir')
+        .add('long-path')
+        .end()
+
+    // these do nothing!
+    pack.pause()
+    pack.resume()
+
+    const zipped = pack.read()
+    t.type(zipped, Buffer)
+    const data = zlib.brotliDecompressSync(zipped)
+    const entries = []
+    for (var i = 0; i < data.length; i += 512) {
+        const slice = data.slice(i, i + 512)
+        const h = new Header(slice)
+        if (h.nullBlock) {
+            entries.push('null block')
+        } else if (h.cksumValid) {
+            entries.push([h.type, h.path])
+        } else if (entries[entries.length - 1][0] === 'File') {
+            entries[entries.length - 1].push(slice.toString().replace(/\0.*$/, ''))
+        }
+    }
+
+    const expect = [
+        ['Directory', 'dir/'],
+        ['File', 'dir/x'],
+        ['Directory', 'long-path/'],
+        ['Directory', 'long-path/r/'],
+        ['Directory', 'long-path/r/e/'],
+        ['Directory', 'long-path/r/e/a/'],
+        ['Directory', 'long-path/r/e/a/l/'],
+        ['Directory', 'long-path/r/e/a/l/l/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/'],
+        ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/'],
+        ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/a.txt', 'short\n'],
+        ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'],
+        ['ExtendedHeader', 'PaxHeader/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'],
+        ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222'],
+        ['ExtendedHeader', 'PaxHeader/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccc'],
+        ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccccccccccccc', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'],
+        ['ExtendedHeader', 'PaxHeader/Ω.txt'],
+        ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/Ω.txt', 'Ω'],
+        'null block',
+        'null block',
+    ]
+
+    let ok = true
+    entries.forEach((entry, i) => {
+        ok = ok &&
+            t.equal(entry[0], expect[i][0]) &&
+            t.equal(entry[1], expect[i][1]) &&
+            (!entry[2] || t.equal(entry[2], expect[i][2]))
+    })
+
+    // t.match(entries, expect)
+    t.end()
+})
+
 t.test('write after end', t => {
   const p = new Pack()
   p.end()

test/parse.js

@@ -125,6 +125,30 @@ t.test('fixture tests', t => {
         bs.end(zlib.gzipSync(tardata))
       })
 
+      t.test('compress with brotli all at once', t => {
+        const p = new Parse({
+          maxMetaEntrySize: maxMeta,
+          filter: filter ? (path, entry) => entry.size % 2 !== 0 : null,
+          strict: strict,
+          brotli: {}
+        })
+        trackEvents(t, expect, p)
+        p.end(zlib.brotliCompressSync(tardata))
+      })
+
+      t.test('compress with brotli byte at a time', t => {
+        const bs = new ByteStream()
+        const bp = new Parse({
+          maxMetaEntrySize: maxMeta,
+          filter: filter ? (path, entry) => entry.size % 2 !== 0 : null,
+          strict: strict,
+          brotli: {},
+        })
+        trackEvents(t, expect, bp)
+        bs.pipe(bp)
+        bs.end(zlib.brotliCompressSync(tardata))
+      })
+
       t.test('async chunks', t => {
         const p = new Parse({
           maxMetaEntrySize: maxMeta,

test/replace.js

@@ -23,6 +23,7 @@ const fixtureDef = {
   'zero.tar': Buffer.from(''),
   'empty.tar': Buffer.alloc(512),
   'compressed.tgz': zlib.gzipSync(data),
+  'compressed.tbr': zlib.brotliCompressSync(data),
 }
 
 t.test('basic file add to archive (good or truncated)', t => {
@@ -211,6 +212,30 @@ t.test('cannot append to gzipped archives', async t => {
   }, [path.basename(__filename)], er => t.match(er, expect))
 })
 
+t.test('cannot append to brotli compressed archives', async t => {
+  const dir = t.testdir({
+    'compressed.tbr': fixtureDef['compressed.tbr'],
+  })
+  const file = resolve(dir, 'compressed.tbr')
+
+  const expect = new Error('cannot append to compressed archives')
+  const expectT = new TypeError('cannot append to compressed archives')
+
+  t.throws(_ => r({
+    file,
+    cwd: __dirname,
+    brotli: true,
+  }, [path.basename(__filename)]), expectT)
+
+  t.throws(_ => r({
+    file,
+    cwd: __dirname,
+    sync: true,
+  }, [path.basename(__filename)]), expect)
+
+  t.end()
+})
+
 t.test('other throws', t => {
   t.throws(_ => r({}, ['asdf']), new TypeError('file is required'))
   t.throws(_ => r({ file: 'asdf' }, []),

test/update.js

@@ -9,6 +9,7 @@ const { resolve } = require('path')
 const fixtures = path.resolve(__dirname, 'fixtures')
 const tars = path.resolve(fixtures, 'tars')
 const zlib = require('zlib')
+const r = require("../lib/replace");
 
 const spawn = require('child_process').spawn
 
@@ -22,6 +23,7 @@ const fixtureDef = {
   'zero.tar': Buffer.from(''),
   'empty.tar': Buffer.alloc(512),
   'compressed.tgz': zlib.gzipSync(data),
+  'compressed.tbr': zlib.brotliCompressSync(data),
 }
 
 t.test('basic file add to archive (good or truncated)', t => {
@@ -213,6 +215,30 @@ t.test('cannot append to gzipped archives', t => {
   })
 })
 
+t.test('cannot append to brotli archives', t => {
+  const dir = t.testdir({
+    'compressed.tbr': fixtureDef['compressed.tbr'],
+  })
+  const file = resolve(dir, 'compressed.tbr')
+
+  const expect = new Error('cannot append to compressed archives')
+  const expectT = new TypeError('cannot append to compressed archives')
+
+  t.throws(_ => u({
+    file,
+    cwd: __dirname,
+    brotli: true,
+  }, [path.basename(__filename)]), expectT)
+
+  t.throws(_ => u({
+    file,
+    cwd: __dirname,
+    sync: true,
+  }, [path.basename(__filename)]), expect)
+
+  t.end()
+})
+
 t.test('other throws', t => {
   t.throws(_ => u({}, ['asdf']), new TypeError('file is required'))
   t.throws(_ => u({ file: 'asdf' }, []),