Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
function index(options) {
var readStream = fs.createReadStream(options.dump)
.pipe(lzma.createDecompressor())
.pipe(es.split())
.pipe(es.map(function (data, cb) {
// prepare/massage the data
//console.log(data);
data = massage(data);
var obj = new Object();
// - migrate id from '_id' to 'id'
obj._id = data._id.$oid;
delete data._id;
// Just keep the fields we want to index
// - Main fields (in the mapping)
obj.title = data.title;
obj.DOI = data.DOI;
function load_file(input) {
var start = new Date();
readStreamBao = fs.createReadStream(input)
.pipe(lzma.createDecompressor())
.pipe(es.split())
.pipe(es.map(function (data, cb) {
// prepare/massage the data
// - migrate id from '_id' to 'id'
var obj = JSON.parse(data);
obj.id = obj._id.$oid;
delete obj._id;
// - remove citation data
delete obj.reference;
cb(null, obj)
}));
async.series([
async pipe(destination: T, options?: { end?: boolean }): Promise {
console.log('download');
if (await this.streamIsCompressed) {
let decompressor = LZMA.createDecompressor();
return AsyncRequest.get(this.url).pipe(decompressor).pipe(destination, options);
} else {
return AsyncRequest.get(this.url).pipe(destination, options);
}
}
fs.close(fd, () => {
reject(err);
});
});
req.on('progress', (state) => {
this.stateEmitter.updateState({
progress: state.percent,
speed: utils.formatBytes(state.speed, 2) + '/s',
transferred: utils.formatBytes(state.size.transferred, 2) + ' / ' + utils.formatBytes(state.size.total, 2),
elapsed: state.time.elapsed + ' seconds',
remaining: state.time.remaining + ' seconds'
});
});
const decompressor = lzma.createDecompressor();
req.pipe(decompressor).pipe(fileStream).on('finish', async () => {
try {
await close(fd);
}
catch (error) {
console.error(error);
}
resolve(null);
});
});
}
getStream(): Readable {
const httpStream = Needle.get(this.ressource);
if (this.compressed) {
const decompressor = LZMA.createDecompressor() as Duplex;
return httpStream.pipe(decompressor);
}
const duplex = new Duplex();
return httpStream.pipe(duplex);
}
getStream(): Readable {
const fileStream = this.listingFile.getStream();
if (this.listingFile.name.endsWith('.xz')) {
const decompressor = LZMA.createDecompressor() as Duplex;
return fileStream.pipe(decompressor);
}
return fileStream;
}
export function decompress(stream: Readable): Readable {
const decompressor = LZMA.createDecompressor();
const decompressedStream = stream.pipe(decompressor);
const originalDestroy = decompressedStream.destroy.bind(decompressedStream);
decompressedStream.destroy = () => {
(decompressedStream as any).cleanup();
originalDestroy();
};
return decompressedStream;
}