How to use the memorious.util.make_key function in memorious

To help you get started, we’ve selected a few memorious examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github alephdata / memorious / memorious / logic / context.py View on Github external
def skip_incremental(self, *criteria):
        """Perform an incremental check on a set of criteria.

        This can be used to execute a part of a crawler only once per an
        interval (which is specified by the ``expire`` setting). If the
        operation has already been performed (and should thus be skipped),
        this will return ``True``. If the operation needs to be executed,
        the returned value will be ``False``.
        """
        if not self.incremental:
            return False

        # this is pure convenience, and will probably backfire at some point.
        key = make_key(*criteria)
        if key is None:
            return False

        if self.check_tag(key):
            return True

        self.set_tag(key, None)
        return False
github alephdata / memorious / memorious / model / session.py View on Github external
def save(cls, crawler, session):
        session = pickle.dumps(session)
        session = codecs.encode(session, 'base64')
        key = sha1(session).hexdigest()[:15]
        key = make_key(crawler, "session", key)
        cls.conn.set(key, session, ex=QUEUE_EXPIRE)
        return key
github alephdata / memorious / memorious / model / crawler_run.py View on Github external
def record_operation_start(cls, crawler, run_id):
        if not cls.conn.sismember(make_key(crawler, "runs"), run_id):
            cls.conn.sadd(make_key(crawler, "runs"), run_id)
            cls.conn.lpush(make_key(crawler, "runs_list"), run_id)
            cls.conn.set(make_key("run", run_id, "start"), pack_now())
        cls.conn.incr(make_key("run", run_id))
        cls.conn.incr(make_key("run", run_id, "total_ops"))
github alephdata / memorious / memorious / model / crawler_run.py View on Github external
def record_operation_start(cls, crawler, run_id):
        if not cls.conn.sismember(make_key(crawler, "runs"), run_id):
            cls.conn.sadd(make_key(crawler, "runs"), run_id)
            cls.conn.lpush(make_key(crawler, "runs_list"), run_id)
            cls.conn.set(make_key("run", run_id, "start"), pack_now())
        cls.conn.incr(make_key("run", run_id))
        cls.conn.incr(make_key("run", run_id, "total_ops"))
github alephdata / memorious / memorious / logic / context.py View on Github external
def set_tag(self, key, value):
        data = dump_json(value)
        key = make_key(self.crawler, "tag", key)
        return conn.set(key, data, ex=self.crawler.expire)
github alephdata / memorious / memorious / model / crawler_run.py View on Github external
def record_operation_start(cls, crawler, run_id):
        if not cls.conn.sismember(make_key(crawler, "runs"), run_id):
            cls.conn.sadd(make_key(crawler, "runs"), run_id)
            cls.conn.lpush(make_key(crawler, "runs_list"), run_id)
            cls.conn.set(make_key("run", run_id, "start"), pack_now())
        cls.conn.incr(make_key("run", run_id))
        cls.conn.incr(make_key("run", run_id, "total_ops"))
github alephdata / memorious / memorious / model / tag.py View on Github external
def find(cls, crawler, key):
        value = cls.conn.get(make_key(crawler, "tag", key))
        if value is not None:
            return load_json(value)
github alephdata / memorious / memorious / model / session.py View on Github external
def get(cls, crawler, key):
        value = cls.conn.get(make_key(crawler, "session", key))
        if value is not None:
            session = codecs.decode(bytes(value, 'utf-8'), 'base64')
            return pickle.loads(session)
github alephdata / memorious / memorious / model / tag.py View on Github external
def save(cls, crawler, key, value):
        data = dump_json(value)
        key = make_key(crawler, "tag", key)
        cls.conn.set(key, data, ex=crawler.expire)
github alephdata / memorious / memorious / logic / context.py View on Github external
def check_tag(self, key):
        return conn.exists(make_key(self.crawler, "tag", key))