How to use the cachetools.ttl.TTLCache function in cachetools

To help you get started, we’ve selected a few cachetools examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github cocrawler / cocrawler / cocrawler / scheduler.py View on Github external
def __init__(self, robots, resolver):
        self.robots = robots
        self.resolver = resolver

        self.q = asyncio.PriorityQueue()
        self.ridealong = {}
        self.awaiting_work = 0
        self.maxhostqps = None
        self.delta_t = None
        self.next_fetch = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds good enough for QPS=0.1 and up
        self.frozen_until = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds is longer than our typical delay
        self.maxhostqps = float(config.read('Crawl', 'MaxHostQPS'))
        self.delta_t = 1./self.maxhostqps
        self.initialize_budgets()

        _, prefetch_dns = fetcher.global_policies()
        self.use_ip_key = prefetch_dns
        memory.register_debug(self.memory)
github quay / quay / oauth / oidc.py View on Github external
Retrieves the public key for this handler with the given kid.

        Raises a PublicKeyLoadException on failure.
        """

        # If force_refresh is true, we expire all the items in the cache by setting the time to
        # the current time + the expiration TTL.
        if force_refresh:
            self._public_key_cache.expire(time=time.time() + PUBLIC_KEY_CACHE_TTL)

        # Retrieve the public key from the cache. If the cache does not contain the public key,
        # it will internally call _load_public_key to retrieve it and then save it.
        return self._public_key_cache[kid]


class _PublicKeyCache(TTLCache):
    def __init__(self, login_service, *args, **kwargs):
        super(_PublicKeyCache, self).__init__(*args, **kwargs)

        self._login_service = login_service

    def __missing__(self, kid):
        """
        Loads the public key for this handler from the OIDC service.

        Raises PublicKeyLoadException on failure.
        """
        keys_url = self._login_service._oidc_config()["jwks_uri"]

        # Load the keys.
        try:
            keys = KEYS()
github tkem / cachetools / cachetools / func.py View on Github external
'hits', 'misses', 'maxsize', 'currsize'
])


class _UnboundCache(dict):

    @property
    def maxsize(self):
        return None

    @property
    def currsize(self):
        return len(self)


class _UnboundTTLCache(TTLCache):
    def __init__(self, ttl, timer):
        TTLCache.__init__(self, math.inf, ttl, timer)

    @property
    def maxsize(self):
        return None


def _cache(cache, typed):
    maxsize = cache.maxsize

    def decorator(func):
        key = keys.typedkey if typed else keys.hashkey
        lock = RLock()
        stats = [0, 0]
github tkem / cachetools / cachetools / func.py View on Github external
def ttl_cache(maxsize=128, ttl=600, timer=time.monotonic, typed=False):
    """Decorator to wrap a function with a memoizing callable that saves
    up to `maxsize` results based on a Least Recently Used (LRU)
    algorithm with a per-item time-to-live (TTL) value.
    """
    if maxsize is None:
        return _cache(_UnboundTTLCache(ttl, timer), typed)
    elif callable(maxsize):
        return _cache(TTLCache(128, ttl, timer), typed)(maxsize)
    else:
        return _cache(TTLCache(maxsize, ttl, timer), typed)
github tkem / cachetools / cachetools / func.py View on Github external
def ttl_cache(maxsize=128, ttl=600, timer=time.monotonic, typed=False):
    """Decorator to wrap a function with a memoizing callable that saves
    up to `maxsize` results based on a Least Recently Used (LRU)
    algorithm with a per-item time-to-live (TTL) value.
    """
    if maxsize is None:
        return _cache(_UnboundTTLCache(ttl, timer), typed)
    elif callable(maxsize):
        return _cache(TTLCache(128, ttl, timer), typed)(maxsize)
    else:
        return _cache(TTLCache(maxsize, ttl, timer), typed)
github tkem / cachetools / cachetools / func.py View on Github external
def __init__(self, ttl, timer):
        TTLCache.__init__(self, math.inf, ttl, timer)
github cocrawler / cocrawler / cocrawler / scheduler.py View on Github external
def __init__(self, robots, resolver):
        self.robots = robots
        self.resolver = resolver

        self.q = asyncio.PriorityQueue()
        self.ridealong = {}
        self.awaiting_work = 0
        self.maxhostqps = None
        self.delta_t = None
        self.next_fetch = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds good enough for QPS=0.1 and up
        self.frozen_until = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds is longer than our typical delay
        self.maxhostqps = float(config.read('Crawl', 'MaxHostQPS'))
        self.delta_t = 1./self.maxhostqps
        self.initialize_budgets()

        _, prefetch_dns = fetcher.global_policies()
        self.use_ip_key = prefetch_dns
        memory.register_debug(self.memory)
github tkem / cachetools / cachetools / ttl.py View on Github external
def currsize(self):
        with self.__timer as time:
            self.expire(time)
            return super(TTLCache, self).currsize
github cocrawler / cocrawler / cocrawler / datalayer.py View on Github external
def __init__(self):
        self.seen_set = set()

        robots_size = config.read('Robots', 'RobotsCacheSize')
        robots_ttl = config.read('Robots', 'RobotsCacheTimeout')
        self.robots = cachetools.ttl.TTLCache(robots_size, robots_ttl)

        memory.register_debug(self.memory)