Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
it('should allow to use Apify proxy', async () => {
process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
process.env[ENV_VARS.PROXY_PORT] = 123;
const mock = sinon.mock(actor);
mock.expects('getApifyProxyUrl')
.once()
.withArgs({
session: 'xxx',
groups: ['yyy'],
groupsParamName: 'options.apifyProxyGroups',
sessionParamName: 'options.apifyProxySession',
})
.returns(null); // Return null so that it doesn't start proxy-chain
try {
await Apify
it('should work', async () => {
process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
process.env[ENV_VARS.PROXY_PORT] = 123;
const pool = new Apify.PuppeteerPool({
maxOpenPagesPerInstance: 3,
retireInstanceAfterRequestCount: 5,
});
const browsers = [];
// Open 6 pages 3 in both browsers.
browsers.push(pool.newPage());
browsers.push(pool.newPage());
browsers.push(pool.newPage());
browsers.push(pool.newPage());
browsers.push(pool.newPage());
browsers.push(pool.newPage());
})
.returns(null); // Return null so that it doesn't start proxy-chain
try {
await Apify
.launchPuppeteer({
useApifyProxy: true,
apifyProxySession: 'xxx',
apifyProxyGroups: ['yyy'],
headless: true,
})
.then(browser => browser.close());
} finally {
mock.verify();
mock.restore();
delete process.env[ENV_VARS.PROXY_PASSWORD];
delete process.env[ENV_VARS.PROXY_HOSTNAME];
delete process.env[ENV_VARS.PROXY_PORT];
}
});
expect(pool.activeInstances[2].activePages).to.be.eql(1);
expect(pool.activeInstances[2].totalPages).to.be.eql(1);
expect(pool.retiredInstances[0].activePages).to.be.eql(3);
expect(pool.retiredInstances[0].totalPages).to.be.eql(5);
// Kill the remaining 3 pages from the 1st browser to see that it gets closed.
await (await browsers[2]).close();
await (await browsers[6]).close();
await (await browsers[7]).close();
await shortSleep(2000);
expect(_.values(pool.retiredInstances).length).to.be.eql(0);
// Cleanup everything.
await pool.destroy();
delete process.env[ENV_VARS.PROXY_PASSWORD];
delete process.env[ENV_VARS.PROXY_HOSTNAME];
delete process.env[ENV_VARS.PROXY_PORT];
});
// For backwards compatibility.
// TODO: remove this when we release v1.0.0
if (!options.groups && options.apifyProxyGroups) {
log.warning('Parameter `apifyProxyGroups` of Apify.getApifyProxyUrl() is deprecated!!! Use `groups` instead!');
options.groups = options.apifyProxyGroups;
}
if (!options.session && options.apifyProxySession) {
log.warning('Parameter `apifyProxySession` of Apify.getApifyProxyUrl() is deprecated!!! Use `session` instead!');
options.session = options.apifyProxySession;
}
const {
groups,
session,
country,
password = process.env[ENV_VARS.PROXY_PASSWORD],
hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME],
port = parseInt(process.env[ENV_VARS.PROXY_PORT] || LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT], 10),
// This is used only internaly. Some other function calling this function use different naming for groups and session
// parameters so we need to override this in error messages.
groupsParamName = 'opts.groups',
sessionParamName = 'opts.session',
countryParamName = 'opts.country',
} = options;
const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;
const throwInvalidProxyValueError = (param) => {
throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);
};
const throwInvalidCountryCode = (code) => {
throw new Error(`The "${code}" option must be a valid two letter country code according to ISO 3166-1 alpha-2`);
}
// Check if apify storage were purge, if not print error
if (!flags.purge) {
const isStorageEmpty = await checkIfStorageIsEmpty();
if (!isStorageEmpty) {
warning('The apify_storage directory contains a previous state, the actor will continue where it left off. '
+ 'To start from the initial state, use --purge parameter to clean the apify_storage directory.');
}
}
// Attach env vars from local config files
const localEnvVars = {
[ENV_VARS.LOCAL_STORAGE_DIR]: DEFAULT_LOCAL_STORAGE_DIR,
};
if (proxy && proxy.password) localEnvVars[ENV_VARS.PROXY_PASSWORD] = proxy.password;
if (userId) localEnvVars[ENV_VARS.USER_ID] = userId;
if (token) localEnvVars[ENV_VARS.TOKEN] = token;
if (localConfig.env) {
const updatedEnv = replaceSecretsValue(localConfig.env);
Object.assign(localEnvVars, updatedEnv);
}
// NOTE: User can overwrite env vars
const env = Object.assign(localEnvVars, process.env);
env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} --max-http-header-size=80000` : '--max-http-header-size=80000';
if (!userId) {
warning('You are not logged in with your Apify Account. Some features like Apify Proxy will not work. Call "apify login" to fix that.');
}
// --max-http-header-size=80000
// Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb.
it('should work with useApifyProxy and other opts', async () => {
process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
const proxies = [];
const useApifyProxy = true;
const apifyProxyGroups = ['GROUP1', 'GROUP2'];
const apifyProxySession = 'session';
const proxy = Apify.getApifyProxyUrl({
groups: apifyProxyGroups,
session: apifyProxySession,
});
const crawler = new Apify.CheerioCrawler({
requestList,
handlePageFunction: async () => {},
useApifyProxy,
apifyProxyGroups,
apifyProxySession,
it('should work', () => {
process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
process.env[ENV_VARS.PROXY_PORT] = 123;
expect(Apify.getApifyProxyUrl({
session: 'XYZ',
groups: ['g1', 'g2', 'g3'],
country: 'US',
})).to.be.eql('http://groups-g1+g2+g3,session-XYZ,country-US:abc123@my.host.com:123');
expect(Apify.getApifyProxyUrl({
session: 'XYZ',
groups: ['g1', 'g2', 'g3'],
})).to.be.eql('http://groups-g1+g2+g3,session-XYZ:abc123@my.host.com:123');
expect(Apify.getApifyProxyUrl({
groups: ['g1', 'g2', 'g3'],
})).to.be.eql('http://groups-g1+g2+g3,session-XYZ:abc123@my.host.com:123');
expect(Apify.getApifyProxyUrl({
groups: ['g1', 'g2', 'g3'],
})).to.be.eql('http://groups-g1+g2+g3:abc123@my.host.com:123');
expect(Apify.getApifyProxyUrl({
session: 'XYZ',
})).to.be.eql('http://session-XYZ:abc123@my.host.com:123');
expect(Apify.getApifyProxyUrl({ country: 'US' })).to.be.eql('http://country-US:abc123@my.host.com:123');
expect(Apify.getApifyProxyUrl()).to.be.eql('http://auto:abc123@my.host.com:123');
delete process.env[ENV_VARS.PROXY_PASSWORD];
delete process.env[ENV_VARS.PROXY_HOSTNAME];
delete process.env[ENV_VARS.PROXY_PORT];
expect(Apify.getApifyProxyUrl({ password: 'xyz' }))
.to.be.eql(`http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`);
expect(() => Apify.getApifyProxyUrl()).to.throw();
expect(Apify.getApifyProxyUrl({
password: 'xyz',
hostname: 'your.host.com',
port: 345,
})).to.be.eql('http://auto:xyz@your.host.com:345');
});
sessionParamName = 'opts.session',
countryParamName = 'opts.country',
} = options;
const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;
const throwInvalidProxyValueError = (param) => {
throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);
};
const throwInvalidCountryCode = (code) => {
throw new Error(`The "${code}" option must be a valid two letter country code according to ISO 3166-1 alpha-2`);
};
checkParamOrThrow(groups, groupsParamName, 'Maybe [String]');
checkParamOrThrow(session, sessionParamName, 'Maybe Number | String');
checkParamOrThrow(country, countryParamName, 'Maybe String');
checkParamOrThrow(password, 'opts.password', 'String', getMissingParamErrorMgs('password', ENV_VARS.PROXY_PASSWORD));
checkParamOrThrow(hostname, 'opts.hostname', 'String', getMissingParamErrorMgs('hostname', ENV_VARS.PROXY_HOSTNAME));
checkParamOrThrow(port, 'opts.port', 'Number', getMissingParamErrorMgs('port', ENV_VARS.PROXY_PORT));
let username;
if (groups || session || country) {
const parts = [];
if (groups && groups.length) {
if (!groups.every(group => APIFY_PROXY_VALUE_REGEX.test(group))) throwInvalidProxyValueError('groups');
parts.push(`groups-${groups.join('+')}`);
}
if (session) {
if (!APIFY_PROXY_VALUE_REGEX.test(session)) throwInvalidProxyValueError('session');
parts.push(`session-${session}`);
}