Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async function batched () {
winston.info(`creating ${totalBatches} csv file `)
let currentBatch = 0
let teacherIndex = 1
while (currentBatch < totalBatches) {
winston.info(`creating batch ${currentBatch}`)
// const csvHeaders = ['username', 'password']
const csvStream = csv.format()
const writableStream = fs.createWriteStream(path.join(__dirname, `${currentBatch}-teacherLogins.csv`))
csvStream.pipe(writableStream)
// csvStream.write(csvHeaders)
let batchIndex = 0
while (batchIndex < batchSize) {
batchIndex++
csvStream.write([`teacher${teacherIndex}`, 'password'])
teacherIndex++
}
csvStream.end()
currentBatch++
}
}
run(outputStream: stream$Writable) {
this.logger.verbose('Starting Export')
if (this.exportConfig.format === 'csv') {
// open a stream to write csv from object
const csvOptions = {
headers: true,
delimiter: this.exportConfig.delimiter,
}
const csvStream = csv
.format(csvOptions)
.transform((row: Inventory): Object => {
this.logger.verbose(`transforming row ${JSON.stringify(row)}`)
let mappedRow = InventoryExporter.inventoryMappings(row)
if (this.exportConfig.headerFields) {
mappedRow = InventoryExporter.applyHeaderFieldsOnRow(
mappedRow,
this.exportConfig.headerFields
)
}
return mappedRow
})
csvStream.pipe(outputStream)
this._fetchInventories(csvStream)
run(outputStream: stream$Writable) {
this.logger.info('Starting Export')
if (this.config.exportFormat === 'csv') {
const csvOptions = {
headers: this.config.csvHeaders,
delimiter: this.config.delimiter,
}
const csvStream = csv.format(csvOptions)
csvStream.pipe(outputStream)
this._getProducts(outputStream, csvStream)
} else {
// This makes the exported data compatible with the importer
// Newlines make data more human-readable
const jsonStream = JSONStream.stringify('{"prices": [\n', ',\n', '\n]}')
jsonStream.pipe(outputStream)
this._getProducts(outputStream, jsonStream)
}
}
return new Promise((resolve, reject) => {
options = options || {};
// const encoding = options.encoding || 'utf8';
// const separator = options.separator || ',';
// const quoteChar = options.quoteChar || '\'';
const worksheet = this.workbook.getWorksheet(options.sheetName || options.sheetId);
const csvStream = fastCsv.format(options);
stream.on('finish', () => {
resolve();
});
csvStream.on('error', reject);
csvStream.pipe(stream);
const {dateFormat, dateUTC} = options;
const map =
options.map ||
(value => {
if (value) {
if (value.text || value.hyperlink) {
return value.hyperlink || value.text || '';
}
if (value.formula || value.result) {
return value.result || '';
const fs = require('fs');
const path = require('path');
const csv = require('fast-csv');
const User = require('./models/user');
fs.createReadStream(path.resolve(__dirname, 'assets', 'snake_case_users.csv'))
.pipe(csv.parse({ headers: true }))
// pipe the parsed input into a csv formatter
.pipe(csv.format({ headers: true }))
// Using the transform function from the formatting stream
.transform((row, next) => {
User.findById(row.id, (err, user) => {
if (err) {
return next(err);
}
return next(null, {
id: row.id,
firstName: row.first_name,
lastName: row.last_name,
address: row.address,
// properties from user
isVerified: user.isVerified,
hasLoggedIn: user.hasLoggedIn,
age: user.age,
});
try {
newTmpDir = await mtcFsUtils.createTmpDir('PS-REPORT-OUTPUT-V2-', config.PsReportTemp)
logger.info(`${functionName}: tmp directory created: ${newTmpDir}`)
} catch (error) {
logger.error(`${functionName}: Failed to created a new tmp directory: ${error.message}`)
throw error // unrecoverable - no work can be done.
}
// Azure!? - check the directory actually exists
await mtcFsUtils.validateDirectory(newTmpDir)
// Create two write streams for our output files
const anomalyOutputStream = fs.createWriteStream(path.join(newTmpDir, 'anomalyReport.csv'), { mode: 0o600 })
const psReportOutputStream = fs.createWriteStream(path.join(newTmpDir, 'psychometricReport.csv'), { mode: 0o600 })
// ... and 2 CSV streams to pipe into them
const anomalyCsvStream = csv.format({ headers: true })
const psReportCsvStream = csv.format({ headers: true })
psReportCsvStream.pipe(psReportOutputStream)
anomalyCsvStream.pipe(anomalyOutputStream)
let anomalyEndDetected = false
let psReportEndDetected = false
function waitForEnd (predicate, cb) {
if (predicate()) {
cb()
} else {
setTimeout(waitForEnd, 250, predicate, cb)
}
}
return new Promise((resolve, reject) => {
// Open the input file for reading
FROM ${sqlService.adminSchema}.[pupil] p
LEFT JOIN ${sqlService.adminSchema}.[school] s
ON p.school_id = s.id
WHERE p.pin IS NOT NULL
AND p.pinExpiresAt > GETUTCDATE()`
const result = await sqlService.query(sql, [])
let pins = result.map(r => R.values(r))
if (pins.length === 0) {
throw new Error('There are no active pins stored in the db!')
}
winston.info('Writing to CSV...')
const csvHeaders = [ 'schoolPin', 'pupilPin' ]
const csvStream = csv.format()
const writableStream = fs.createWriteStream(path.join(__dirname, '../scenarios/data/pupilLogins.csv'))
writableStream.on('finish', function () {
console.log('DONE')
sqlService.drainPool()
})
csvStream.pipe(writableStream)
csvStream.write(csvHeaders)
pins.map(r => csvStream.write(r))
csvStream.end()
} catch (error) {
winston.info(error)
process.exitCode = 1
sqlService.drainPool()
}
var natural = require('natural')
var csv = require('fast-csv')
var fs = require('fs')
var path = require('path')
var request = require('request');
csvStream = csv.format({headers:true})
csvStream
.pipe(fs.createWriteStream(path.resolve('./', 'all_courses.csv')))
.on('end', process.exit);
var lib = require('../lib.js')
var fall = lib.prepare_query('201609');
var winter = lib.prepare_query('201701');
request(fall, function (error, response, body) {
var courses = lib.parse_data(body);
for (var i = 0; i < courses.length; i++) {
console.log('saving:',courses[i].title)
csvStream.write({
course_title:courses[i].title,
run(outputStream: stream$Writable) {
this.logger.info('Starting Export')
if (this.config.exportFormat === 'csv') {
const csvOptions = {
headers: this.headers,
delimiter: this.config.delimiter,
}
const csvStream = csv.format(csvOptions).transform(this._processCode)
csvStream.pipe(outputStream)
this._handleOutput(outputStream, csvStream)
} else {
const jsonStream = JSONStream.stringify()
jsonStream.pipe(outputStream)
this._handleOutput(outputStream, jsonStream)
}
}