Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async _ensureCorrectMaxMemory() {
if (this.maxMemoryBytes) return;
const { totalBytes } = await getMemoryInfo();
if (isAtHome()) {
this.maxMemoryBytes = totalBytes;
} else {
this.maxMemoryBytes = Math.ceil(totalBytes / 4);
// NOTE: Log as AutoscaledPool, so that users are not confused what "Snapshotter" is
log.info(`AutoscaledPool: Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. Use the ${ENV_VARS.MEMORY_MBYTES} environment variable to override it.`); // eslint-disable-line max-len
}
}
}
_socketConnectionHandler(socket) {
this.clientCount++;
log.info('Live view client connected', { clientId: socket.id });
socket.on('disconnect', (reason) => {
this.clientCount--;
log.info('Live view client disconnected', { clientId: socket.id, reason });
});
socket.on('getLastSnapshot', () => {
if (this.lastSnapshot) {
log.debug('Sending live view snapshot', { createdAt: this.lastSnapshot.createdAt, pageUrl: this.lastSnapshot.pageUrl });
this.socketio.emit('snapshot', this.lastSnapshot);
}
});
}
}
socket.on('disconnect', (reason) => {
this.clientCount--;
log.info('Live view client disconnected', { clientId: socket.id, reason });
});
socket.on('getLastSnapshot', () => {
async start() {
this._isRunning = true;
try {
await ensureDir(this.screenshotDirectoryPath);
await promisifyServerListen(this.httpServer)(this.port);
log.info('Live view web server started', { publicUrl: this.liveViewUrl });
} catch (err) {
log.exception(err, 'Live view web server failed to start.');
this._isRunning = false;
}
}
async _addFetchedRequests(source, fetchedRequests) {
const { requestsFromUrl, regex } = source;
const originalLength = this.requests.length;
fetchedRequests.forEach(request => this._addRequest(request));
const fetchedCount = fetchedRequests.length;
const importedCount = this.requests.length - originalLength;
log.info('RequestList: list fetched.', {
requestsFromUrl,
regex,
fetchedCount,
importedCount,
duplicateCount: fetchedCount - importedCount,
sample: JSON.stringify(fetchedRequests.slice(0, 5)),
});
}
.then(() => {
log.info(`Live view is now available at ${this.liveViewUrl}`);
this.httpServer = server;
});
}
this.logInterval = setInterval(() => {
log.info(this.logMessage, this.getCurrent());
}, this.logIntervalMillis);
}
if (isMaxPagesExceeded()) {
log.info(`BasicCrawler: Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests `
+ 'and all requests that were in progress at that time have now finished. '
+ `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
return true;
}
const isFinished = isFinishedFunction
? await isFinishedFunction()
: await this._defaultIsFinishedFunction();
if (isFinished) {
const reason = isFinishedFunction
? 'BasicCrawler: Crawler\'s custom isFinishedFunction() returned true, the crawler will shut down.'
: 'BasicCrawler: All the requests from request list and/or request queue have been processed, the crawler will shut down.';
log.info(reason);
}
return isFinished;
},
};
async _makeSnapshot(page) {
const pageUrl = page.url();
log.info('Making live view snapshot.', { pageUrl });
const [htmlContent, screenshot] = await Promise.all([
page.content(),
page.screenshot({
type: 'jpeg',
quality: 75,
}),
]);
const screenshotIndex = this.lastScreenshotIndex++;
await writeFile(this._getScreenshotPath(screenshotIndex), screenshot);
if (screenshotIndex > this.maxScreenshotFiles - 1) {
this._deleteScreenshot(screenshotIndex - this.maxScreenshotFiles);
}
const snapshot = new Snapshot({ pageUrl, htmlContent, screenshotIndex });