Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
useIncognitoPages,
proxyUrls,
useLiveView,
} = _.defaults({}, options, DEFAULT_OPTIONS);
// Disabling due to memory leak.
const reusePages = false;
checkParamOrThrow(reusePages, 'options.reusePages', 'Boolean');
checkParamOrThrow(maxOpenPagesPerInstance, 'options.maxOpenPagesPerInstance', 'Number');
checkParamOrThrow(retireInstanceAfterRequestCount, 'options.retireInstanceAfterRequestCount', 'Number');
checkParamOrThrow(launchPuppeteerFunction, 'options.launchPuppeteerFunction', 'Function');
checkParamOrThrow(puppeteerOperationTimeoutSecs, 'options.puppeteerOperationTimeoutSecs', 'Number');
checkParamOrThrow(instanceKillerIntervalMillis, 'options.instanceKillerIntervalMillis', 'Maybe Number');
if (instanceKillerIntervalMillis) {
log.deprecated('PuppeteerPool: options.instanceKillerIntervalMillis is deprecated, use options.instanceKillerIntervalSecs instead.');
}
checkParamOrThrow(instanceKillerIntervalSecs, 'options.instanceKillerIntervalSecs', 'Number');
checkParamOrThrow(killInstanceAfterMillis, 'options.killInstanceAfterMillis', 'Maybe Number');
if (killInstanceAfterMillis) {
log.deprecated('PuppeteerPool: options.killInstanceAfterMillis is deprecated, use options.killInstanceAfterSecs instead.');
}
checkParamOrThrow(killInstanceAfterSecs, 'options.killInstanceAfterSecs', 'Number');
checkParamOrThrow(launchPuppeteerOptions, 'options.launchPuppeteerOptions', 'Maybe Object');
checkParamOrThrow(recycleDiskCache, 'options.recycleDiskCache', 'Maybe Boolean');
checkParamOrThrow(useIncognitoPages, 'options.useIncognitoPages', 'Maybe Boolean');
checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe Array');
// Enforce non-empty proxyUrls array
if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');
checkParamOrThrow(useLiveView, 'options.useLiveView', 'Maybe Boolean');
// Config.
export function computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }) {
const normalizedMethod = method.toUpperCase();
const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
if (!useExtendedUniqueKey) {
if (normalizedMethod !== 'GET' && payload) {
// Using log.deprecated to log only once. We should add log.once or some such.
log.deprecated(`We've encountered a ${normalizedMethod} Request with a payload. `
+ 'This is fine. Just letting you know that if your requests point to the same URL '
+ 'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
}
return normalizedUrl;
}
const payloadHash = payload ? hashPayload(payload) : '';
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
}
const hideWebDriver = async (page) => {
log.deprecated('Apify.utils.puppeteer.hideWebDriver() is deprecated. Use launchPuppeteerOptions.stealth instead.');
checkParamOrThrow(page, 'page', 'Object');
await page.evaluateOnNewDocument(() => {
var modifiedNavigator; // eslint-disable-line no-var
try {
if (Navigator.prototype.hasOwnProperty('webdriver')) { // eslint-disable-line no-prototype-builtins
modifiedNavigator = Navigator.prototype;
} else {
modifiedNavigator = Object.create(window.navigator);
Object.defineProperty(window, 'navigator', {
value: modifiedNavigator,
configurable: false,
enumerable: true,
writable: false,
});
}
export async function enqueueLinks(options = {}) {
const {
page,
$,
selector = 'a',
requestQueue,
baseUrl,
pseudoUrls,
userData, // TODO DEPRECATED 2019/06/27
transformRequestFunction,
} = options;
if (userData) {
log.deprecated('options.userData of Apify.utils.enqueueLinks() is deprecated. Use options.transformRequestFunction instead.');
}
checkParamOrThrow(page, 'page', 'Maybe Object');
checkParamOrThrow($, '$', 'Maybe Function');
if (!page && !$) {
throw new Error('One of the parameters "options.page" or "options.$" must be provided!');
}
if (page && $) {
throw new Error('Only one of the parameters "options.page" or "options.$" must be provided!');
}
checkParamOrThrow(selector, 'selector', 'String');
checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
checkParamOrThrow(baseUrl, 'baseUrl', 'Maybe String');
if (baseUrl && page) log.warning('The parameter options.baseUrl can only be used when parsing a Cheerio object. It will be ignored.');
checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');
checkParamOrThrow(userData, 'userData', 'Maybe Object');
async delete() {
log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the dataset and not individual records in the dataset.');
await this.drop();
}
doNotRetry(message) {
log.deprecated('request.doNotRetry is deprecated. Use request.noRetry = true; instead.');
this.noRetry = true;
if (message) throw new Error(message);
}
}
setMinConcurrency(minConcurrency) {
log.deprecated('AutoscaledPool.setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');
this._minConcurrency = minConcurrency;
}
setMaxConcurrency(maxConcurrency) {
log.deprecated('AutoscaledPool.setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');
this._maxConcurrency = maxConcurrency;
}
async delete() {
log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the key-value store and not individual records in the store.');
await this.drop();
}