Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
console.log('Running Crawler test...');
console.log(' ');
const errors = await require('./crawler')(crawler, {
debug: true,
// maxCrawl: 100,
cluster: {
maxConcurrency: 5,
monitor: true
}
});
console.log('\n' + chalk.bgRedBright(` ERROR `));
const dir = path.resolve(__dirname, '../../logs/crawler/');
await fs.ensureDir(dir);
await fs.writeJson(
path.resolve(dir, `${encodeURIComponent(crawler)}.json`),
parse(stringify(errors)),
{
spaces: '\t'
}
);
Object.entries(errors).forEach(([type, errors]) => {
console.log('❌ ' + chalk.underline(type));
errors.forEach(err => {
console.log(`● ${err.url}`);
if (err.pageUrl) console.log(` pageUrl: ${err.pageUrl}`);
switch (type) {
case 'no gzip':
case 'large file': {
let size = err.contentLength;
if (size > 1024 * 1024) {
size = getSize(size);
} else if (size > 1024) {
const setData = (cacheName, cachedData) => {
const data = parse(stringify(cachedData));
// clean components
for (const key in data) {
for (let i = 0, len = data[key].length; i < len; i += 1) {
if (data[key][i].component) {
data[key][i].component = data[key][i].id;
}
}
}
localStorage.setItem(cacheName, stringify(data));
};
this.server.on('message', (messageString, socket) => {
const message = parse(messageString);
if (message.action !== action) {
return;
}
let reply = (error, replyData) => {
this.reply(socket, action, error, replyData);
};
_done(message.data, reply, socket);
});
}
this.client.on(action, (messageString) => {
callback(parse(messageString));
});
}
ipc.of['embark'].on(action, (messageString) => {
callback(parse(messageString));
});
}
restore(cache) {
if (!cache) {
return this;
}
if (typeof cache === 'string') {
const data = Flatted.parse(cache);
if (Symbol.iterator in Object(data)) {
this.cache = new Map(data);
}
}
else if (Array.isArray(cache)) {
this.cache = new Map(cache);
}
return this;
}
fs.readFile(this.dbPath, 'utf8', async (err, file) => {
if (err) {
if (err.code == 'ENOENT') {
await this.saveAll([])
return resolve([])
}
return reject(err)
}
const items = Flatted.parse(file).map(this.model.create)
resolve(items)
})
})
ipc.server.on('message', function(messageString, socket) {
const message = parse(messageString);
if (message.action !== action) {
return;
}
let reply = function(error, replyData) {
self.reply(socket, action, error, replyData);
};
done(message.data, reply, socket);
});
}
this.client.once('message', (messageString) => {
const message = parse(messageString);
if (message.action !== action) {
return;
}
cb(message.error, message.data);
});
}
this.client.on(action, (messageString) => {
callback(parse(messageString));
});
}