How to use the spidermon.contrib.stats.counters.DictPercentCounter function in spidermon

To help you get started, we’ve selected a few spidermon examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / spider.py View on Github external
# 5xx. internal server errors
        self.internal_server_errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
            target=self.internal_server_errors,
        )

        # >= 6xx. others
        self.others = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
        )

        # errors (4xx + 5xx)
        self.errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_ERRORS, target=self.errors
        )
github scrapinghub / spidermon / spidermon / contrib / stats / counters.py View on Github external
def __init__(self, total):
        super(AttributeDictPercentCounter, self).__init__(total)
        setattr(self, self.__attribute_dict_name__, DictPercentCounter(total))
github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / spider.py View on Github external
)

        # 3xx. redirections
        self.redirections = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
        )

        # 4xx. bad requests
        self.bad_requests = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
        )

        # 5xx. internal server errors
        self.internal_server_errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
            target=self.internal_server_errors,
        )

        # >= 6xx. others
        self.others = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
        )

        # errors (4xx + 5xx)
        self.errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_ERRORS, target=self.errors
        )
github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / validation.py View on Github external
from __future__ import absolute_import
import warnings

from spidermon.contrib.stats.counters import (
    PercentCounter,
    DictPercentCounter,
    AttributeDictPercentCounter,
)
from spidermon.contrib.stats.analyzer import StatsAnalyzer
from spidermon.contrib.scrapy.stats import STATS_DEFAULT_VALIDATION_PREFIX

from .stats import StatsMonitorMixin


class MetaDictPercentCounter(DictPercentCounter):
    def add_values(self, key, subkey, value):
        if key not in self._dict:
            self._create_item(key)
        self[key].add_value(subkey, value)


class ErrorsDictPercentCounter(AttributeDictPercentCounter):
    __attribute_dict_name__ = "fields"


class ErrorsInfo(MetaDictPercentCounter):
    __items_class__ = ErrorsDictPercentCounter


class FieldErrorsDictPercentCounter(AttributeDictPercentCounter):
    __attribute_dict_name__ = "errors"
github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / spider.py View on Github external
self.count = self._stats_analyzer.search(DOWNLOADER_RESPONSE_COUNT + "$").get(
            DOWNLOADER_RESPONSE_COUNT, 0
        )

        # all status codes
        self.all = DictPercentCounter(total=self.count)
        self._add_status_codes(pattern=None, target=self.all)

        # 1xx. informational
        self.informational = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL, target=self.informational
        )

        # 2xx. successful
        self.successful = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
        )

        # 3xx. redirections
        self.redirections = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
        )

        # 4xx. bad requests
        self.bad_requests = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
        )
github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / spider.py View on Github external
# 4xx. bad requests
        self.bad_requests = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
        )

        # 5xx. internal server errors
        self.internal_server_errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
            target=self.internal_server_errors,
        )

        # >= 6xx. others
        self.others = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
        )

        # errors (4xx + 5xx)
        self.errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_ERRORS, target=self.errors
        )
github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / spider.py View on Github external
)

        # 2xx. successful
        self.successful = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
        )

        # 3xx. redirections
        self.redirections = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
        )

        # 4xx. bad requests
        self.bad_requests = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
        )

        # 5xx. internal server errors
        self.internal_server_errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
            target=self.internal_server_errors,
        )

        # >= 6xx. others
        self.others = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
        )
github scrapinghub / spidermon / spidermon / contrib / stats / counters.py View on Github external
def __init__(self, total):
        super(DictPercentCounter, self).__init__(total)
        self._dict = dict()
github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / spider.py View on Github external
def __init__(self, stats):
        self._stats_analyzer = StatsAnalyzer(stats=stats)
        self.count = self._stats_analyzer.search(DOWNLOADER_RESPONSE_COUNT + "$").get(
            DOWNLOADER_RESPONSE_COUNT, 0
        )

        # all status codes
        self.all = DictPercentCounter(total=self.count)
        self._add_status_codes(pattern=None, target=self.all)

        # 1xx. informational
        self.informational = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL, target=self.informational
        )

        # 2xx. successful
        self.successful = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
        )

        # 3xx. redirections
        self.redirections = DictPercentCounter(total=self.count)
github scrapinghub / spidermon / spidermon / contrib / monitors / mixins / spider.py View on Github external
self._add_status_codes(pattern=None, target=self.all)

        # 1xx. informational
        self.informational = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL, target=self.informational
        )

        # 2xx. successful
        self.successful = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
        )

        # 3xx. redirections
        self.redirections = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
        )

        # 4xx. bad requests
        self.bad_requests = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
        )

        # 5xx. internal server errors
        self.internal_server_errors = DictPercentCounter(total=self.count)
        self._add_status_codes(
            pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
            target=self.internal_server_errors,
        )