How to use the apify-shared/lru_cache function in apify-shared

To help you get started, we’ve selected a few apify-shared examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github apifytech / apify-js / test / utils.js View on Github external
it('should use ID from ENV variable and not call getOrCreateStoreFunction parameter is not provided', async () => {
        const cache = new LruCache({ maxLength: 5 });
        class MyStore {
            constructor(id) {
                this.id = id;
            }
        }

        process.env['some-env'] = 'id-from-env';

        const store = await utils.openRemoteStorage(null, 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line
        expect(store.id).to.eql('id-from-env');

        delete process.env['some-env'];
    });
});
github apifytech / apify-js / test / utils.js View on Github external
it('should return item from cache if available and create new one otherwise', async () => {
        const cache = new LruCache({ maxLength: 5 });
        class MyStore {
            constructor(id) {
                this.id = id;
            }
        }

        delete process.env['some-env'];

        expect(cache.length()).to.be.eql(0);

        const store = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => ({ id: 'some-id' }));
        expect(store.id).to.be.eql('some-id');
        expect(cache.length()).to.be.eql(1);

        const store2 = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line
        expect(store2.id).to.be.eql('some-id');
github apifytech / apify-js / src / puppeteer_utils.js View on Github external
get: function () { // eslint-disable-line object-shorthand
                        return false;
                    },
                },
            });
            // Date.prototype.getTimezoneOffset = function () { return -4 * 60; };
        } catch (e) {
            console.error(e);
        }
    });
};

/**
 * Cache contents of previously injected files to limit file system access.
 */
const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE });

/**
 * Injects a JavaScript file into a Puppeteer page.
 * Unlike Puppeteer's `addScriptTag` function, this function works on pages
 * with arbitrary Cross-Origin Resource Sharing (CORS) policies.
 *
 * File contents are cached for up to 10 files to limit file system access.
 *
 * @param {Page} page
 *   Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&amp;show=api-class-page"><code>Page</code></a> object.
 * @param {String} filePath File path
 * @param {Object} [options]
 * @param {boolean} [options.surviveNavigations]
 *   Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
 *   This does not mean, however, that internal state will be preserved. Just that it will be automatically
 *   re-injected on each navigation before any other scripts get the chance to execute.
github apifytech / apify-js / src / request_queue.js View on Github external
const RECENTLY_HANDLED_CACHE_SIZE = 1000;

// Indicates how long it usually takes for the underlying storage to propagate all writes
// to be available to subsequent reads.
export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;


const writeFilePromised = promisify(fs.writeFile);
const readdirPromised = promisify(fs.readdir);
const readFilePromised = promisify(fs.readFile);
const renamePromised = promisify(fs.rename);
const statPromised = promisify(fs.stat);
const emptyDirPromised = promisify(fs.emptyDir);

const { requestQueues } = apifyClient;
const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queues are stored here.

/**
 * Helper function to validate params of *.addRequest().
 * @ignore
 */
const validateAddRequestParams = (request, opts) => {
    checkParamOrThrow(request, 'request', 'Object');
    checkParamOrThrow(opts, 'opts', 'Object');

    const newRequest = request instanceof Request ? request : new Request(request);

    const { forefront = false } = opts;

    checkParamOrThrow(forefront, 'opts.forefront', 'Boolean');

    if (request.id) throw new Error('Request already has the "id" field set so it cannot be added to the queue!');
github apifytech / apify-js / src / dataset.js View on Github external
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;
export const LOCAL_FILENAME_DIGITS = 9;
export const LOCAL_GET_ITEMS_DEFAULT_LIMIT = 250000;
const MAX_OPENED_STORES = 1000;
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%

const writeFilePromised = promisify(fs.writeFile);
const readFilePromised = promisify(fs.readFile);
const readdirPromised = promisify(fs.readdir);
const statPromised = promisify(fs.stat);
const emptyDirPromised = promisify(fs.emptyDir);

const getLocaleFilename = index => `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;

const { datasets } = apifyClient;
const datasetsCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open Datasets are stored here.

/**
 * Accepts a JSON serializable object as an input, validates its serializability,
 * and validates its serialized size against limitBytes. Optionally accepts its index
 * in an array to provide better error messages. Returns serialized object.
 *
 * @param {Object} item
 * @param {Number} limitBytes
 * @param {Number} [index]
 * @returns {string}
 * @ignore
 */
export const checkAndSerialize = (item, limitBytes, index) => {
    const s = typeof index === 'number' ? ` at index ${index} ` : ' ';
    let payload;
    try {
github apifytech / apify-js / src / request_queue.js View on Github external
this.queueId = queueId;
        this.queueName = queueName;

        // Contains a cached list of request IDs from the head of the queue,
        // as obtained in the last query. Both key and value is the request ID.
        this.queueHeadDict = new ListDictionary();
        this.queryQueueHeadPromise = null;

        // A set of all request IDs that are currently being handled,
        // i.e. which were returned by fetchNextRequest() but not markRequestHandled()
        this.inProgress = new Set();

        // Contains a list of recently handled requests. It is used to avoid inconsistencies
        // caused by delays in the underlying DynamoDB storage.
        // Keys are request IDs, values are true.
        this.recentlyHandled = new LruCache({ maxLength: RECENTLY_HANDLED_CACHE_SIZE });

        // We can trust these numbers only in a case that queue is used by a single client.
        // This information is returned by getHead() under the hadMultipleClients property.
        this.assumedTotalCount = 0;
        this.assumedHandledCount = 0;

        // Caching requests to avoid redundant addRequest() calls.
        // Key is computed using getRequestId() and value is { id, isHandled }.
        // TODO: We could extend the caching to improve performance
        //       of other operations such as fetchNextRequest().
        this.requestsCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
    }
github apifytech / apify-js / src / key_value_store.js View on Github external
import { APIFY_API_BASE_URL } from './constants';

export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.keyValueStores;
const MAX_OPENED_STORES = 1000;
const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css', 'csv'];

const readFilePromised = promisify(fs.readFile);
const readdirPromised = promisify(fs.readdir);
const writeFilePromised = promisify(fs.writeFile);
const unlinkPromised = promisify(fs.unlink);
const statPromised = promisify(fs.stat);
const emptyDirPromised = promisify(fs.emptyDir);

const { keyValueStores } = apifyClient;
const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.

/**
 * Helper function to validate params of *.getValue().
 *
 * @ignore
 */
const validateGetValueParams = (key) => {
    checkParamOrThrow(key, 'key', 'String');
    if (!key) throw new Error('The "key" parameter cannot be empty');
};

/**
 * Helper function to validate params of *.setValue().
 *
 * @ignore
 */
github apifytech / apify-js / src / request_queue.js View on Github external
// Contains a list of recently handled requests. It is used to avoid inconsistencies
        // caused by delays in the underlying DynamoDB storage.
        // Keys are request IDs, values are true.
        this.recentlyHandled = new LruCache({ maxLength: RECENTLY_HANDLED_CACHE_SIZE });

        // We can trust these numbers only in a case that queue is used by a single client.
        // This information is returned by getHead() under the hadMultipleClients property.
        this.assumedTotalCount = 0;
        this.assumedHandledCount = 0;

        // Caching requests to avoid redundant addRequest() calls.
        // Key is computed using getRequestId() and value is { id, isHandled }.
        // TODO: We could extend the caching to improve performance
        //       of other operations such as fetchNextRequest().
        this.requestsCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
    }
github apifytech / apify-js / test / utils.js View on Github external
it('should return item from cache if available and create new one otherwise', async () => {
        const cache = new LruCache({ maxLength: 5 });
        class MyStore {}

        expect(cache.length()).to.be.eql(0);

        const store = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);
        expect(store).to.be.instanceOf(MyStore);
        expect(cache.length()).to.be.eql(1);

        const store2 = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);
        expect(store2).to.be.equal(store);
        expect(cache.length()).to.be.eql(1);

        const store3 = await utils.openLocalStorage('some-other-id', 'some-env', MyStore, cache);
        expect(store3).to.not.be.equal(store);
        expect(cache.length()).to.be.eql(2);
    });
github apifytech / apify-js / test / utils.js View on Github external
it('should use ID from ENV variable if no parameter is provided', async () => {
        const cache = new LruCache({ maxLength: 5 });
        class MyStore {
            constructor(id) {
                this.id = id;
            }
        }

        process.env['some-env'] = 'id-from-env';

        const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);
        expect(store.id).to.eql('id-from-env');

        delete process.env['some-env'];
    });