Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
it('should use ID from ENV variable and not call getOrCreateStoreFunction parameter is not provided', async () => {
const cache = new LruCache({ maxLength: 5 });
class MyStore {
constructor(id) {
this.id = id;
}
}
process.env['some-env'] = 'id-from-env';
const store = await utils.openRemoteStorage(null, 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line
expect(store.id).to.eql('id-from-env');
delete process.env['some-env'];
});
});
it('should return item from cache if available and create new one otherwise', async () => {
const cache = new LruCache({ maxLength: 5 });
class MyStore {
constructor(id) {
this.id = id;
}
}
delete process.env['some-env'];
expect(cache.length()).to.be.eql(0);
const store = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => ({ id: 'some-id' }));
expect(store.id).to.be.eql('some-id');
expect(cache.length()).to.be.eql(1);
const store2 = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line
expect(store2.id).to.be.eql('some-id');
get: function () { // eslint-disable-line object-shorthand
return false;
},
},
});
// Date.prototype.getTimezoneOffset = function () { return -4 * 60; };
} catch (e) {
console.error(e);
}
});
};
/**
* Cache contents of previously injected files to limit file system access.
*/
const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE });
/**
* Injects a JavaScript file into a Puppeteer page.
* Unlike Puppeteer's `addScriptTag` function, this function works on pages
* with arbitrary Cross-Origin Resource Sharing (CORS) policies.
*
* File contents are cached for up to 10 files to limit file system access.
*
* @param {Page} page
* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page"><code>Page</code></a> object.
* @param {String} filePath File path
* @param {Object} [options]
* @param {boolean} [options.surviveNavigations]
* Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
* This does not mean, however, that internal state will be preserved. Just that it will be automatically
* re-injected on each navigation before any other scripts get the chance to execute.
const RECENTLY_HANDLED_CACHE_SIZE = 1000;
// Indicates how long it usually takes for the underlying storage to propagate all writes
// to be available to subsequent reads.
export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
const writeFilePromised = promisify(fs.writeFile);
const readdirPromised = promisify(fs.readdir);
const readFilePromised = promisify(fs.readFile);
const renamePromised = promisify(fs.rename);
const statPromised = promisify(fs.stat);
const emptyDirPromised = promisify(fs.emptyDir);
const { requestQueues } = apifyClient;
const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queues are stored here.
/**
* Helper function to validate params of *.addRequest().
* @ignore
*/
const validateAddRequestParams = (request, opts) => {
checkParamOrThrow(request, 'request', 'Object');
checkParamOrThrow(opts, 'opts', 'Object');
const newRequest = request instanceof Request ? request : new Request(request);
const { forefront = false } = opts;
checkParamOrThrow(forefront, 'opts.forefront', 'Boolean');
if (request.id) throw new Error('Request already has the "id" field set so it cannot be added to the queue!');
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;
export const LOCAL_FILENAME_DIGITS = 9;
export const LOCAL_GET_ITEMS_DEFAULT_LIMIT = 250000;
const MAX_OPENED_STORES = 1000;
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
const writeFilePromised = promisify(fs.writeFile);
const readFilePromised = promisify(fs.readFile);
const readdirPromised = promisify(fs.readdir);
const statPromised = promisify(fs.stat);
const emptyDirPromised = promisify(fs.emptyDir);
const getLocaleFilename = index => `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;
const { datasets } = apifyClient;
const datasetsCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open Datasets are stored here.
/**
* Accepts a JSON serializable object as an input, validates its serializability,
* and validates its serialized size against limitBytes. Optionally accepts its index
* in an array to provide better error messages. Returns serialized object.
*
* @param {Object} item
* @param {Number} limitBytes
* @param {Number} [index]
* @returns {string}
* @ignore
*/
export const checkAndSerialize = (item, limitBytes, index) => {
const s = typeof index === 'number' ? ` at index ${index} ` : ' ';
let payload;
try {
this.queueId = queueId;
this.queueName = queueName;
// Contains a cached list of request IDs from the head of the queue,
// as obtained in the last query. Both key and value is the request ID.
this.queueHeadDict = new ListDictionary();
this.queryQueueHeadPromise = null;
// A set of all request IDs that are currently being handled,
// i.e. which were returned by fetchNextRequest() but not markRequestHandled()
this.inProgress = new Set();
// Contains a list of recently handled requests. It is used to avoid inconsistencies
// caused by delays in the underlying DynamoDB storage.
// Keys are request IDs, values are true.
this.recentlyHandled = new LruCache({ maxLength: RECENTLY_HANDLED_CACHE_SIZE });
// We can trust these numbers only in a case that queue is used by a single client.
// This information is returned by getHead() under the hadMultipleClients property.
this.assumedTotalCount = 0;
this.assumedHandledCount = 0;
// Caching requests to avoid redundant addRequest() calls.
// Key is computed using getRequestId() and value is { id, isHandled }.
// TODO: We could extend the caching to improve performance
// of other operations such as fetchNextRequest().
this.requestsCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
}
import { APIFY_API_BASE_URL } from './constants';
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.keyValueStores;
const MAX_OPENED_STORES = 1000;
const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css', 'csv'];
const readFilePromised = promisify(fs.readFile);
const readdirPromised = promisify(fs.readdir);
const writeFilePromised = promisify(fs.writeFile);
const unlinkPromised = promisify(fs.unlink);
const statPromised = promisify(fs.stat);
const emptyDirPromised = promisify(fs.emptyDir);
const { keyValueStores } = apifyClient;
const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.
/**
* Helper function to validate params of *.getValue().
*
* @ignore
*/
const validateGetValueParams = (key) => {
checkParamOrThrow(key, 'key', 'String');
if (!key) throw new Error('The "key" parameter cannot be empty');
};
/**
* Helper function to validate params of *.setValue().
*
* @ignore
*/
// Contains a list of recently handled requests. It is used to avoid inconsistencies
// caused by delays in the underlying DynamoDB storage.
// Keys are request IDs, values are true.
this.recentlyHandled = new LruCache({ maxLength: RECENTLY_HANDLED_CACHE_SIZE });
// We can trust these numbers only in a case that queue is used by a single client.
// This information is returned by getHead() under the hadMultipleClients property.
this.assumedTotalCount = 0;
this.assumedHandledCount = 0;
// Caching requests to avoid redundant addRequest() calls.
// Key is computed using getRequestId() and value is { id, isHandled }.
// TODO: We could extend the caching to improve performance
// of other operations such as fetchNextRequest().
this.requestsCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
}
it('should return item from cache if available and create new one otherwise', async () => {
const cache = new LruCache({ maxLength: 5 });
class MyStore {}
expect(cache.length()).to.be.eql(0);
const store = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);
expect(store).to.be.instanceOf(MyStore);
expect(cache.length()).to.be.eql(1);
const store2 = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);
expect(store2).to.be.equal(store);
expect(cache.length()).to.be.eql(1);
const store3 = await utils.openLocalStorage('some-other-id', 'some-env', MyStore, cache);
expect(store3).to.not.be.equal(store);
expect(cache.length()).to.be.eql(2);
});
it('should use ID from ENV variable if no parameter is provided', async () => {
const cache = new LruCache({ maxLength: 5 });
class MyStore {
constructor(id) {
this.id = id;
}
}
process.env['some-env'] = 'id-from-env';
const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);
expect(store.id).to.eql('id-from-env');
delete process.env['some-env'];
});