Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def __init__(self, robots, resolver):
self.robots = robots
self.resolver = resolver
self.q = asyncio.PriorityQueue()
self.ridealong = {}
self.awaiting_work = 0
self.maxhostqps = None
self.delta_t = None
self.next_fetch = cachetools.ttl.TTLCache(10000, 10) # 10 seconds good enough for QPS=0.1 and up
self.frozen_until = cachetools.ttl.TTLCache(10000, 10) # 10 seconds is longer than our typical delay
self.maxhostqps = float(config.read('Crawl', 'MaxHostQPS'))
self.delta_t = 1./self.maxhostqps
self.initialize_budgets()
_, prefetch_dns = fetcher.global_policies()
self.use_ip_key = prefetch_dns
memory.register_debug(self.memory)
Retrieves the public key for this handler with the given kid.
Raises a PublicKeyLoadException on failure.
"""
# If force_refresh is true, we expire all the items in the cache by setting the time to
# the current time + the expiration TTL.
if force_refresh:
self._public_key_cache.expire(time=time.time() + PUBLIC_KEY_CACHE_TTL)
# Retrieve the public key from the cache. If the cache does not contain the public key,
# it will internally call _load_public_key to retrieve it and then save it.
return self._public_key_cache[kid]
class _PublicKeyCache(TTLCache):
def __init__(self, login_service, *args, **kwargs):
super(_PublicKeyCache, self).__init__(*args, **kwargs)
self._login_service = login_service
def __missing__(self, kid):
"""
Loads the public key for this handler from the OIDC service.
Raises PublicKeyLoadException on failure.
"""
keys_url = self._login_service._oidc_config()["jwks_uri"]
# Load the keys.
try:
keys = KEYS()
'hits', 'misses', 'maxsize', 'currsize'
])
class _UnboundCache(dict):
@property
def maxsize(self):
return None
@property
def currsize(self):
return len(self)
class _UnboundTTLCache(TTLCache):
def __init__(self, ttl, timer):
TTLCache.__init__(self, math.inf, ttl, timer)
@property
def maxsize(self):
return None
def _cache(cache, typed):
maxsize = cache.maxsize
def decorator(func):
key = keys.typedkey if typed else keys.hashkey
lock = RLock()
stats = [0, 0]
def ttl_cache(maxsize=128, ttl=600, timer=time.monotonic, typed=False):
"""Decorator to wrap a function with a memoizing callable that saves
up to `maxsize` results based on a Least Recently Used (LRU)
algorithm with a per-item time-to-live (TTL) value.
"""
if maxsize is None:
return _cache(_UnboundTTLCache(ttl, timer), typed)
elif callable(maxsize):
return _cache(TTLCache(128, ttl, timer), typed)(maxsize)
else:
return _cache(TTLCache(maxsize, ttl, timer), typed)
def ttl_cache(maxsize=128, ttl=600, timer=time.monotonic, typed=False):
"""Decorator to wrap a function with a memoizing callable that saves
up to `maxsize` results based on a Least Recently Used (LRU)
algorithm with a per-item time-to-live (TTL) value.
"""
if maxsize is None:
return _cache(_UnboundTTLCache(ttl, timer), typed)
elif callable(maxsize):
return _cache(TTLCache(128, ttl, timer), typed)(maxsize)
else:
return _cache(TTLCache(maxsize, ttl, timer), typed)
def __init__(self, ttl, timer):
TTLCache.__init__(self, math.inf, ttl, timer)
def __init__(self, robots, resolver):
self.robots = robots
self.resolver = resolver
self.q = asyncio.PriorityQueue()
self.ridealong = {}
self.awaiting_work = 0
self.maxhostqps = None
self.delta_t = None
self.next_fetch = cachetools.ttl.TTLCache(10000, 10) # 10 seconds good enough for QPS=0.1 and up
self.frozen_until = cachetools.ttl.TTLCache(10000, 10) # 10 seconds is longer than our typical delay
self.maxhostqps = float(config.read('Crawl', 'MaxHostQPS'))
self.delta_t = 1./self.maxhostqps
self.initialize_budgets()
_, prefetch_dns = fetcher.global_policies()
self.use_ip_key = prefetch_dns
memory.register_debug(self.memory)
def currsize(self):
with self.__timer as time:
self.expire(time)
return super(TTLCache, self).currsize
def __init__(self):
self.seen_set = set()
robots_size = config.read('Robots', 'RobotsCacheSize')
robots_ttl = config.read('Robots', 'RobotsCacheTimeout')
self.robots = cachetools.ttl.TTLCache(robots_size, robots_ttl)
memory.register_debug(self.memory)