Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
export const ensureTokenOrLocalStorageEnvExists = (storageName) => {
if (!process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !process.env[ENV_VARS.TOKEN]) {
throw new Error(`Cannot use ${storageName} as neither ${ENV_VARS.LOCAL_STORAGE_DIR} nor ${ENV_VARS.TOKEN} environment variable is set. You need to set one these variables in order to enable data storage.`); // eslint-disable-line max-len
}
};
it('should work', async () => {
const mock = sinon.mock(utils);
process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
mock.expects('openLocalStorage').once();
await Apify.openKeyValueStore();
mock.expects('openLocalStorage').once();
Apify.openKeyValueStore('xxx');
mock.expects('openRemoteStorage').once();
Apify.openKeyValueStore('xxx', { forceCloud: true });
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
mock.expects('openRemoteStorage').once();
await Apify.openKeyValueStore();
delete process.env[ENV_VARS.TOKEN];
it('should work', () => {
const mock = sinon.mock(utils);
process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
mock.expects('openLocalStorage').once();
Apify.openRequestQueue();
mock.expects('openLocalStorage').once();
Apify.openRequestQueue('xxx');
mock.expects('openRemoteStorage').once();
Apify.openRequestQueue('xxx', { forceCloud: true });
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
process.env[ENV_VARS.TOKEN] = 'xxx';
mock.expects('openRemoteStorage').once();
Apify.openRequestQueue();
delete process.env[ENV_VARS.TOKEN];
mock.verify();
mock.restore();
});
});
it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage', async () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
process.env[ENV_VARS.TOKEN] = 'xxx';
const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';
await expect(Apify.getValue('KEY')).to.be.rejectedWith(errMsg);
delete process.env[ENV_VARS.TOKEN];
});
});
export const expectNotUsingLocalStorage = () => expect(process.env[ENV_VARS.LOCAL_STORAGE_DIR]).toBeUndefined();
import path from 'path';
import { promisify } from 'util';
import express from 'express';
import socketio from 'socket.io';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { promisifyServerListen } from 'apify-shared/utilities';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import { addTimeoutToPromise } from '../utils';
import Snapshot from './snapshot';
const writeFile = promisify(fs.writeFile);
const unlink = promisify(fs.unlink);
const ensureDir = promisify(fs.ensureDir);
const LOCAL_STORAGE_DIR = process.env[ENV_VARS.LOCAL_STORAGE_DIR] || '';
const DEFAULT_SCREENSHOT_DIR_PATH = path.resolve(LOCAL_STORAGE_DIR, 'live_view');
/**
* `LiveViewServer` enables serving of browser snapshots via web sockets. It includes its own client
* that provides a simple frontend to viewing the captured snapshots. A snapshot consists of three
* pieces of information, the currently opened URL, the content of the page (HTML) and its screenshot.
*
* ```json
* {
* "pageUrl": "https://www.example.com",
* "htmlContent": " ....",
* "screenshotIndex": 3,
* "createdAt": "2019-04-18T11:50:40.060Z"
* }
* ```
*
const getLocalStorageDir = () => {
const envVar = ENV_VARS.LOCAL_STORAGE_DIR;
return process.env[envVar] || DEFAULT_LOCAL_STORAGE_DIR;
};
const getLocalKeyValueStorePath = (storeId) => {
export const openRequestQueue = (queueIdOrName, options = {}) => {
checkParamOrThrow(queueIdOrName, 'queueIdOrName', 'Maybe String');
checkParamOrThrow(options, 'options', 'Object');
ensureTokenOrLocalStorageEnvExists('request queue');
const { forceCloud = false } = options;
checkParamOrThrow(forceCloud, 'options.forceCloud', 'Boolean');
return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !forceCloud
? openLocalStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueueLocal, queuesCache)
: openRemoteStorage(queueIdOrName, ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, RequestQueue, queuesCache, getOrCreateQueue);
};
export const openKeyValueStore = (storeIdOrName, options = {}) => {
checkParamOrThrow(storeIdOrName, 'storeIdOrName', 'Maybe String');
checkParamOrThrow(options, 'options', 'Object');
ensureTokenOrLocalStorageEnvExists('key value store');
const { forceCloud = false } = options;
checkParamOrThrow(forceCloud, 'options.forceCloud', 'Boolean');
return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !forceCloud
? openLocalStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStoreLocal, storesCache)
: openRemoteStorage(storeIdOrName, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, KeyValueStore, storesCache, getOrCreateKeyValueStore);
};