How to use the spidermon.Monitor function in spidermon

To help you get started, we’ve selected a few spidermon examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github scrapinghub / spidermon / tests / fixtures / levels.py View on Github external
# ----------------------------------
# Base Monitors
# ----------------------------------
class NoLevelMethodMonitor(Monitor):
    def test(self):
        pass


class HighLevelMethodMonitor(Monitor):
    @monitors.level.high
    def test(self):
        pass


class NormalLevelMethodMonitor(Monitor):
    @monitors.level.normal
    def test(self):
        pass


class LowLevelMethodMonitor(Monitor):
    @monitors.level.low
    def test(self):
        pass


# ----------------------------------
# Monitors
# ----------------------------------
class Monitors:
    class NoLevelMonitor:
github scrapinghub / spidermon / tests / fixtures / cases.py View on Github external
from __future__ import absolute_import
from spidermon import Monitor


class EmptyMonitor(Monitor):
    pass


class Monitor01(Monitor):
    def test_a(self):
        pass

    def test_b(self):
        pass

    def test_c(self):
        pass


class Monitor02(Monitor):
    def test_d(self):
        pass

    def test_e(self):
        pass
github scrapinghub / spidermon / tests / fixtures / cases.py View on Github external
from __future__ import absolute_import
from spidermon import Monitor


class EmptyMonitor(Monitor):
    pass


class Monitor01(Monitor):
    def test_a(self):
        pass

    def test_b(self):
        pass

    def test_c(self):
        pass


class Monitor02(Monitor):
    def test_d(self):
github scrapinghub / spidermon / tests / fixtures / names.py View on Github external
# ----------------------------------
# Monitors
# ----------------------------------
class UnnamedMonitor(Monitor):
    def test_without_name(self):
        pass

    @monitors.name("A Test")
    def test_with_name(self):
        pass


@monitors.name("Class Monitor")
class NamedMonitor(Monitor):
    def test_without_name(self):
        pass

    @monitors.name("A Test")
    def test_with_name(self):
        pass


# ----------------------------------
# Child Suites
# ----------------------------------
class BaseChildSuite(MonitorSuite):
    monitors = [
        UnnamedMonitor,
        NamedMonitor,
        ("Instance Monitor", UnnamedMonitor),
github scrapinghub / spidermon / _examples / monitors.py View on Github external
rules=[
        ('Rule 1', rule_as_function),
        ('Rule 2', rule_as_function),
    ],
    actions=[
        ('Action 1', DummyAction()),
    ]
)
MONITOR_E.add_rule(rule=rule_as_function, name='Rule 3')
MONITOR_E.add_action(action=DummyAction(), name='Action 2')


#---------------------------------------------------------------
# F. Rule levels and action triggers
#---------------------------------------------------------------
MONITOR_F = Monitor(
    name='F. Rule levels and action triggers',
    rules=[
        ('Rule High',   'stats.finish_reason == "finished"', 'HIGH'),
        ('Rule Normal', 'stats.finish_reason != "finished"', 'NORMAL'),
        ('Rule Low',    'stats.finish_reason != "finished"', 'LOW'),
    ],
    actions=[
        ('Action runs always',    DummyAction()),  #  trigger defaults to ALWAYS
        ('Action runs always',    DummyAction(), 'ALWAYS'),
        ('Action runs on passed', DummyAction(), 'PASSED'),
        ('Action runs on failed', DummyAction(), 'FAILED'),
        ('Action runs on error',  DummyAction(), 'ERROR'),
    ]
)
MONITOR_F.add_rule(rule=rule_as_function, name='Rule Low 2', level='LOW')
MONITOR_F.add_action(action=DummyAction(), name='Action on passed', trigger='PASSED')
github scrapinghub / spidermon / _examples / monitors.py View on Github external
]
)


#---------------------------------------------------------------
# B. Adding rules and actions
#---------------------------------------------------------------
MONITOR_B = Monitor(name='B. Adding rules and actions')
MONITOR_B.add_rule(rule=lambda stats: stats.finish_reason == 'finished')
MONITOR_B.add_action(action=MessageAction('finish reason is ok!'))


#---------------------------------------------------------------
# C. Monitor without rules
#---------------------------------------------------------------
MONITOR_C = Monitor(
    name='C. Monitor without rules',
    actions=[
        MessageAction('hi there!'),
    ]
)


#---------------------------------------------------------------
# D. All rule type definitions
#---------------------------------------------------------------
def rule_as_function(stats):
    return stats.finish_reason == 'finished'


class ARule(Rule):
    def check(self, stats):
github scrapinghub / spidermon / _examples / monitors.py View on Github external
lambda stats: stats.finish_reason == 'finished',  # lambda
        'stats.finish_reason == "finished"',  # python expression string
        rule_as_function,  # function
        ARule(),  # Rule
        ATestCase(),  # TestCase
    ],
    actions=[
        DummyAction(),
    ]
)


#---------------------------------------------------------------
# E. Naming rules and actions
#---------------------------------------------------------------
MONITOR_E = Monitor(
    name='E. Naming rules and actions',
    rules=[
        ('Rule 1', rule_as_function),
        ('Rule 2', rule_as_function),
    ],
    actions=[
        ('Action 1', DummyAction()),
    ]
)
MONITOR_E.add_rule(rule=rule_as_function, name='Rule 3')
MONITOR_E.add_action(action=DummyAction(), name='Action 2')


#---------------------------------------------------------------
# F. Rule levels and action triggers
#---------------------------------------------------------------
github okfn-brasil / open-jus / justa / crawler / justa / monitors.py View on Github external
from spidermon import Monitor, MonitorSuite, monitors


class MinimumItemsMonitor(Monitor):
    minimum_expected = {
        'distrito_federal': 433,
        'tjsp_numbers': 826
    }

    @monitors.name('Minimum number of items')
    def test_minimum_number_of_items(self):
        spider = self.data['spider']
        items_extracted = self.data.stats.get('item_scraped_count', 0)
        minimum_expected = self.minimum_expected.get(spider.name, 0)
        self.assertGreaterEqual(items_extracted, minimum_expected)


class JustaMonitorSuite(MonitorSuite):
    monitors = [MinimumItemsMonitor]
github scrapinghub / spidermon / examples / tutorial / tutorial / monitors.py View on Github external
# monitors.py
from spidermon import Monitor, MonitorSuite, monitors
from spidermon.contrib.actions.slack.notifiers import SendSlackMessageSpiderFinished
from spidermon.contrib.monitors.mixins import StatsMonitorMixin

from tutorial.actions import CloseSpiderAction


@monitors.name("Item count")
class ItemCountMonitor(Monitor):
    @monitors.name("Minimum items extracted")
    def test_minimum_number_of_items_extracted(self):
        minimum_threshold = 1000
        item_extracted = getattr(self.data.stats, "item_scraped_count", 0)
        self.assertFalse(
            item_extracted < minimum_threshold,
            msg="Extracted less than {} items".format(minimum_threshold),
        )


@monitors.name("Item validation")
class ItemValidationMonitor(Monitor, StatsMonitorMixin):
    @monitors.name("No item validation errors")
    def test_no_item_validation_errors(self):
        validation_errors = getattr(
            self.data.stats, "spidermon/validation/fields/errors", 0
github scrapinghub / spidermon / examples / tutorial / tutorial / monitors.py View on Github external
    @monitors.name("No item validation errors")
    def test_no_item_validation_errors(self):
        validation_errors = getattr(
            self.data.stats, "spidermon/validation/fields/errors", 0
        )
        self.assertEqual(
            validation_errors,
            0,
            msg="Found validation errors in {} fields".format(validation_errors),
        )

        self.data.stats


@monitors.name("Periodic job stats monitor")
class PeriodicJobStatsMonitor(Monitor, StatsMonitorMixin):
    @monitors.name("Maximum number of errors exceeded")
    def test_number_of_errors(self):
        accepted_num_errors = 20
        num_errors = self.data.stats.get("log_count/ERROR", 0)

        msg = "The job has exceeded the maximum number of errors"
        self.assertLessEqual(num_errors, accepted_num_errors, msg=msg)


class PeriodicMonitorSuite(MonitorSuite):
    monitors = [PeriodicJobStatsMonitor]
    monitors_failed_actions = [CloseSpiderAction]


class SpiderCloseMonitorSuite(MonitorSuite):
    monitors = [ItemCountMonitor, ItemValidationMonitor, PeriodicJobStatsMonitor]