Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
let count = 0;
const processor = batchOf(25, async (items) => {
const dynamoItems = toDynamoItems(converters, items);
await batchWrite({ client, table, dynamoItems });
count += items.length;
if (count % 2500 == 0) {
const seconds = since(start);
const itemsPerSecond = count / seconds;
console.log(
`Inserted ${count} records in ${seconds}s - ${itemsPerSecond} records per second`
);
}
});
let rowIndex = 0;
const parser = fastCsv
.parseFile(fileName, { headers: true, delimiter })
.on("error", (error) => console.error(error))
.on("data", async (row) => {
parser.pause();
try {
await processor(row);
rowIndex++;
} catch (err) {
console.log(`Error processing row ${rowIndex}: ${err}`);
} finally {
parser.resume();
}
})
.on("end", async (rowCount) => {
await batch(null, true);
console.log(`Parsed ${rowCount} rows`);
await new Promise((resolve, reject) => {
csv.parseFile(filepath, { delimiter: '\t', headers: true, trim: true })
.on('data', (data) => {
content.push(data)
})
.on('error', reject)
.on('end', resolve)
})
if (content.length === 0) {
await new Promise((resolve, reject) => {
csv.parseFile(filepath, { delimiter: '\t', headers: true, trim: true })
.on('data', (data) => {
content.push(data)
})
.on('error', reject)
.on('end', resolve)
})
if (content.length === 0) {