How to use the memorious.model.Crawl function in memorious

To help you get started, we’ve selected a few memorious examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github alephdata / memorious / memorious / logic / context.py View on Github external
def execute(self, data):
        """Execute the crawler and create a database record of having done
        so."""
        if Crawl.is_aborted(self.crawler, self.run_id):
            return

        try:
            Crawl.operation_start(self.crawler, self.stage, self.run_id)
            self.log.info('[%s->%s(%s)]: %s',
                          self.crawler.name,
                          self.stage.name,
                          self.stage.method_name,
                          self.run_id)
            return self.stage.method(self, data)
        except Exception as exc:
            self.emit_exception(exc)
        finally:
            Crawl.operation_end(self.crawler, self.run_id)
            shutil.rmtree(self.work_path)
github alephdata / memorious / memorious / logic / context.py View on Github external
def execute(self, data):
        """Execute the crawler and create a database record of having done
        so."""
        if Crawl.is_aborted(self.crawler, self.run_id):
            return

        try:
            Crawl.operation_start(self.crawler, self.stage, self.run_id)
            self.log.info('[%s->%s(%s)]: %s',
                          self.crawler.name,
                          self.stage.name,
                          self.stage.method_name,
                          self.run_id)
            return self.stage.method(self, data)
        except Exception as exc:
            self.emit_exception(exc)
        finally:
            Crawl.operation_end(self.crawler, self.run_id)
            shutil.rmtree(self.work_path)
github alephdata / memorious / memorious / logic / stage.py View on Github external
def op_count(self):
        """Total operations performed for this stage"""
        return Crawl.op_count(self.crawler, self)
github alephdata / memorious / memorious / logic / crawler.py View on Github external
def flush(self):
        """Delete all run-time data generated by this crawler."""
        Queue.flush(self)
        Event.delete(self)
        Crawl.flush(self)
github alephdata / memorious / memorious / logic / crawler.py View on Github external
def cancel(self):
        Crawl.abort_all(self)
        Queue.flush(self)
github alephdata / memorious / memorious / logic / crawler.py View on Github external
def op_count(self):
        """Total operations performed for this crawler"""
        return Crawl.op_count(self)