Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
ws.on('close', () => {
isWsConnected = false;
});
expect(req.url).to.be.eql('/someRunId');
const send = obj => ws.send(JSON.stringify(obj));
setTimeout(() => send({ name: 'name-1', data: [1, 2, 3] }), 50);
setTimeout(() => send({ name: 'name-1', data: { foo: 'bar' } }), 100);
setTimeout(() => send({ name: 'name-2', data: [1] }), 50);
setTimeout(() => send({ name: 'name-2', data: [2] }), 50);
});
process.env[ENV_VARS.ACTOR_EVENTS_WS_URL] = 'ws://localhost:9099/someRunId';
process.env[ENV_VARS.TOKEN] = 'dummy';
// Run main and store received events
expect(isWsConnected).to.be.eql(false);
Apify.main(async () => {
await delayPromise(10); // Here must be short sleep to get following line to later tick
expect(isWsConnected).to.be.eql(true);
Apify.events.on('name-1', data => eventsReceived.push(data));
await delayPromise(1000);
});
// Main will call process.exit() so we must stub it.
const stubbedExit = sinon
.stub(process, 'exit')
.callsFake(async (code) => {
expect(code).to.be.eql(0);
expect(eventsReceived).to.be.eql([[1, 2, 3], { foo: 'bar' }]);
export const main = (userFunc) => {
if (!userFunc || typeof (userFunc) !== 'function') {
throw new Error(`Apify.main() accepts a single parameter that must be a function (was '${userFunc === null ? 'null' : typeof (userFunc)}').`);
}
if (!process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !process.env[ENV_VARS.TOKEN]) {
const dir = path.join(process.cwd(), './apify_storage');
process.env[ENV_VARS.LOCAL_STORAGE_DIR] = dir;
log.warning(`Neither ${ENV_VARS.LOCAL_STORAGE_DIR} nor ${ENV_VARS.TOKEN} environment variable is set, defaulting to ${ENV_VARS.LOCAL_STORAGE_DIR}="${dir}"`); // eslint-disable-line max-len
}
// This is to enable unit tests where process.exit() is mocked and doesn't really exit the process
// Note that mocked process.exit() might throw, so set exited flag before calling it to avoid confusion.
let exited = false;
const exitWithError = (err, exitCode, message) => {
log.exception(err, message);
exited = true;
// console.log(`Exiting with code: ${exitCode}`);
process.exit(exitCode);
};
// Set dummy interval to ensure the process will not be killed while awaiting empty promise:
export const newClient = () => {
const opts = {
userId: process.env[ENV_VARS.USER_ID] || null,
token: process.env[ENV_VARS.TOKEN] || null,
};
// Only set baseUrl if overridden by env var, so that 'https://api.apify.com' is used by default.
// This simplifies local development, which should run against production unless user wants otherwise.
const apiBaseUrl = process.env[ENV_VARS.API_BASE_URL];
if (apiBaseUrl) opts.baseUrl = apiBaseUrl;
return new ApifyClient(opts);
};
// Check if apify storage were purge, if not print error
if (!flags.purge) {
const isStorageEmpty = await checkIfStorageIsEmpty();
if (!isStorageEmpty) {
warning('The apify_storage directory contains a previous state, the actor will continue where it left off. '
+ 'To start from the initial state, use --purge parameter to clean the apify_storage directory.');
}
}
// Attach env vars from local config files
const localEnvVars = {
[ENV_VARS.LOCAL_STORAGE_DIR]: DEFAULT_LOCAL_STORAGE_DIR,
};
if (proxy && proxy.password) localEnvVars[ENV_VARS.PROXY_PASSWORD] = proxy.password;
if (userId) localEnvVars[ENV_VARS.USER_ID] = userId;
if (token) localEnvVars[ENV_VARS.TOKEN] = token;
if (localConfig.env) {
const updatedEnv = replaceSecretsValue(localConfig.env);
Object.assign(localEnvVars, updatedEnv);
}
// NOTE: User can overwrite env vars
const env = Object.assign(localEnvVars, process.env);
env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} --max-http-header-size=80000` : '--max-http-header-size=80000';
if (!userId) {
warning('You are not logged in with your Apify Account. Some features like Apify Proxy will not work. Call "apify login" to fix that.');
}
// --max-http-header-size=80000
// Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb.
// However they did not think about all the sites there with large headers,
// so we put back the old limit of 80kb, which seems to work just fine.
export const ensureTokenOrLocalStorageEnvExists = (storageName) => {
if (!process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !process.env[ENV_VARS.TOKEN]) {
throw new Error(`Cannot use ${storageName} as neither ${ENV_VARS.LOCAL_STORAGE_DIR} nor ${ENV_VARS.TOKEN} environment variable is set. You need to set one these variables in order to enable data storage.`); // eslint-disable-line max-len
}
};
it('should work', async () => {
const mock = sinon.mock(utils);
process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
mock.expects('openLocalStorage').once();
await Apify.openKeyValueStore();
mock.expects('openLocalStorage').once();
Apify.openKeyValueStore('xxx');
mock.expects('openRemoteStorage').once();
Apify.openKeyValueStore('xxx', { forceCloud: true });
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
mock.expects('openRemoteStorage').once();
await Apify.openKeyValueStore();
delete process.env[ENV_VARS.TOKEN];
mock.verify();
mock.restore();
});
});
it('uses correct default if APIFY_API_BASE_URL is not defined', () => {
delete process.env[ENV_VARS.API_BASE_URL];
process.env[ENV_VARS.USER_ID] = 'userId';
process.env[ENV_VARS.TOKEN] = 'token';
const client = utils.newClient();
const opts = client.getOptions();
expect(opts.userId).to.eql('userId');
expect(opts.token).to.eql('token');
expect(opts.baseUrl).to.eql('https://api.apify.com');
});
});
it('throws if DEFAULT_DATASET_ID env var is not defined and we use cloud storage', async () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
process.env[ENV_VARS.DEFAULT_DATASET_ID] = '';
await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);
delete process.env[ENV_VARS.DEFAULT_DATASET_ID];
await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);
delete process.env[ENV_VARS.TOKEN];
});
it('sets default APIFY_LOCAL_STORAGE_DIR', async () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
delete process.env[ENV_VARS.TOKEN];
await testMain({
userFunc: () => {
expect(process.env[ENV_VARS.LOCAL_STORAGE_DIR]).to.eql(path.join(process.cwd(), './apify_storage'));
},
exitCode: 0,
});
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
});
it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage', async () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
process.env[ENV_VARS.TOKEN] = 'xxx';
const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';
await expect(Apify.getValue('KEY')).to.be.rejectedWith(errMsg);
delete process.env[ENV_VARS.TOKEN];
});
});