Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# 5xx. internal server errors
self.internal_server_errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
target=self.internal_server_errors,
)
# >= 6xx. others
self.others = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
)
# errors (4xx + 5xx)
self.errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_ERRORS, target=self.errors
)
def __init__(self, total):
super(AttributeDictPercentCounter, self).__init__(total)
setattr(self, self.__attribute_dict_name__, DictPercentCounter(total))
)
# 3xx. redirections
self.redirections = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
)
# 4xx. bad requests
self.bad_requests = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
)
# 5xx. internal server errors
self.internal_server_errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
target=self.internal_server_errors,
)
# >= 6xx. others
self.others = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
)
# errors (4xx + 5xx)
self.errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_ERRORS, target=self.errors
)
from __future__ import absolute_import
import warnings
from spidermon.contrib.stats.counters import (
PercentCounter,
DictPercentCounter,
AttributeDictPercentCounter,
)
from spidermon.contrib.stats.analyzer import StatsAnalyzer
from spidermon.contrib.scrapy.stats import STATS_DEFAULT_VALIDATION_PREFIX
from .stats import StatsMonitorMixin
class MetaDictPercentCounter(DictPercentCounter):
def add_values(self, key, subkey, value):
if key not in self._dict:
self._create_item(key)
self[key].add_value(subkey, value)
class ErrorsDictPercentCounter(AttributeDictPercentCounter):
__attribute_dict_name__ = "fields"
class ErrorsInfo(MetaDictPercentCounter):
__items_class__ = ErrorsDictPercentCounter
class FieldErrorsDictPercentCounter(AttributeDictPercentCounter):
__attribute_dict_name__ = "errors"
self.count = self._stats_analyzer.search(DOWNLOADER_RESPONSE_COUNT + "$").get(
DOWNLOADER_RESPONSE_COUNT, 0
)
# all status codes
self.all = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=None, target=self.all)
# 1xx. informational
self.informational = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL, target=self.informational
)
# 2xx. successful
self.successful = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
)
# 3xx. redirections
self.redirections = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
)
# 4xx. bad requests
self.bad_requests = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
)
# 4xx. bad requests
self.bad_requests = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
)
# 5xx. internal server errors
self.internal_server_errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
target=self.internal_server_errors,
)
# >= 6xx. others
self.others = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
)
# errors (4xx + 5xx)
self.errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_ERRORS, target=self.errors
)
)
# 2xx. successful
self.successful = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
)
# 3xx. redirections
self.redirections = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
)
# 4xx. bad requests
self.bad_requests = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
)
# 5xx. internal server errors
self.internal_server_errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
target=self.internal_server_errors,
)
# >= 6xx. others
self.others = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
)
def __init__(self, total):
super(DictPercentCounter, self).__init__(total)
self._dict = dict()
def __init__(self, stats):
self._stats_analyzer = StatsAnalyzer(stats=stats)
self.count = self._stats_analyzer.search(DOWNLOADER_RESPONSE_COUNT + "$").get(
DOWNLOADER_RESPONSE_COUNT, 0
)
# all status codes
self.all = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=None, target=self.all)
# 1xx. informational
self.informational = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL, target=self.informational
)
# 2xx. successful
self.successful = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
)
# 3xx. redirections
self.redirections = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=None, target=self.all)
# 1xx. informational
self.informational = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL, target=self.informational
)
# 2xx. successful
self.successful = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
)
# 3xx. redirections
self.redirections = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
)
# 4xx. bad requests
self.bad_requests = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
)
# 5xx. internal server errors
self.internal_server_errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
target=self.internal_server_errors,
)