How to use the cachecontrol.CacheControlAdapter function in CacheControl

To help you get started, we’ve selected a few CacheControl examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github pymedusa / Medusa / medusa / helpers.py View on Github external
def make_session(cache_etags=True, serializer=None, heuristic=None):
    session = requests.Session()

    adapter = CacheControlAdapter(
        DictCache(),
        cache_etags=cache_etags,
        serializer=serializer,
        heuristic=heuristic,
    )

    session.mount('http://', adapter)
    session.mount('https://', adapter)

    session.cache_controller = adapter.controller

    session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'})

    return session
github CenterForOpenScience / osf.io / addons / github / api.py View on Github external
import urllib

import github3
import cachecontrol
from requests.adapters import HTTPAdapter
from requests.exceptions import ConnectionError

from addons.github import settings as github_settings
from addons.github.exceptions import NotFoundError


# Initialize caches
https_cache = cachecontrol.CacheControlAdapter()
default_adapter = HTTPAdapter()


class GitHubClient(object):

    def __init__(self, external_account=None, access_token=None):

        self.access_token = getattr(external_account, 'oauth_key', None) or access_token
        if self.access_token:
            self.gh3 = github3.login(token=self.access_token)
            self.gh3.set_client_id(
                github_settings.CLIENT_ID, github_settings.CLIENT_SECRET
            )
        else:
            self.gh3 = github3.GitHub()
github alin23 / spfy / spfy / mixins / auth.py View on Github external
def get_session(*args, **kwargs):
        session = OAuth2Session(*args, **kwargs)
        cache_adapter = CacheControlAdapter(
            cache=FileCache(CACHE_FILE),
            pool_connections=config.http.connections,
            pool_maxsize=config.http.connections,
            max_retries=config.http.retries,
        )
        session.mount("http://", cache_adapter)
        return session
github omarryhan / pyfy / pyfy / client.py View on Github external
def _create_session(self, max_retries, proxies, backoff_factor, cache):
        sess = Session()
        # Retry only on idempotent methods and only when too many requests
        retries = Retry(total=max_retries, backoff_factor=backoff_factor, status_forcelist=[429], method_whitelist=['GET', 'UPDATE', 'DELETE'])
        retries_adapter = HTTPAdapter(max_retries=retries)
        if cache:
            cache_adapter = CacheControlAdapter(cache_etags=True)
        sess.mount('http://', retries_adapter)
        sess.mount('http://', cache_adapter)
        sess.proxies.update(proxies)  
        return sess
github CenterForOpenScience / osf.io / addons / gitlab / api.py View on Github external
import urllib
import requests

import gitlab
import cachecontrol
from requests.adapters import HTTPAdapter

from addons.gitlab.exceptions import NotFoundError, AuthError
from addons.gitlab.settings import DEFAULT_HOSTS

# Initialize caches
https_cache = cachecontrol.CacheControlAdapter()
default_adapter = HTTPAdapter()

class GitLabClient(object):

    def __init__(self, external_account=None, access_token=None, host=None):
        self.access_token = getattr(external_account, 'oauth_key', None) or access_token
        self.host = getattr(external_account, 'oauth_secret', None) or host or DEFAULT_HOSTS[0]

        if self.access_token:
            self.gitlab = gitlab.Gitlab(self.host, private_token=self.access_token)
        else:
            self.gitlab = gitlab.Gitlab(self.host)

    def user(self, user=None):
        """Fetch a user or the authenticated user.
github github-tooling / ghtopdep / ghtopdep / ghtopdep.py View on Github external
if not repositories:
        destination = "package"
        destinations = "packages"

    repos = []
    more_than_zero_count = 0
    total_repos_count = 0
    spinner = Halo(text="Fetching information about {0}".format(destinations), spinner="dots")
    spinner.start()

    sess = requests.session()
    retries = Retry(
        total=15,
        backoff_factor=15,
        status_forcelist=[429])
    adapter = CacheControlAdapter(max_retries=retries,
                                  cache=FileCache(CACHE_DIR),
                                  heuristic=OneDayHeuristic())
    sess.mount("http://", adapter)
    sess.mount("https://", adapter)

    page_url = get_page_url(sess, url, destination)

    while True:
        response = sess.get(page_url)
        parsed_node = HTMLParser(response.text)
        dependents = parsed_node.css(ITEM_SELECTOR)
        total_repos_count += len(dependents)
        for dep in dependents:
            repo_stars_list = dep.css(STARS_SELECTOR)
            # only for ghost or private? packages
            if repo_stars_list: