How to use the gcsfs.utils.is_retriable function in gcsfs

To help you get started, we’ve selected a few gcsfs examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dask / gcsfs / gcsfs / core.py View on Github external
del kwargs[k]
        json = kwargs.pop('json', None)
        meth = getattr(self.session, method)
        if args:
            path = path.format(*[quote_plus(p) for p in args])
        for retry in range(self.retries):
            try:
                time.sleep(2**retry - 1)
                r = meth(self.base + path, params=kwargs, json=json)
                validate_response(r, path)
                break
            except (HtmlError, RequestException, GoogleAuthError) as e:
                if retry == self.retries - 1:
                    logger.exception("_call out of retries on exception: %s", e)
                    raise e
                if is_retriable(e):
                    logger.debug("_call retrying after exception: %s", e)
                    continue
                logger.exception("_call non-retriable exception: %s", e)
                raise e
        try:
            out = r.json()
        except ValueError:
            out = r.content
        return out
github dask / gcsfs / gcsfs / core.py View on Github external
self.validate_response(status, contents, json, path)
                break
            except (HttpError, RequestException, GoogleAuthError) as e:
                json = None
                if (
                    isinstance(e, HttpError)
                    and e.code == 400
                    and "requester pays" in e.message
                ):
                    msg = "Bucket is requester pays. Set `requester_pays=True` when creating the GCSFileSystem."
                    raise ValueError(msg) from e
                if retry == self.retries - 1:
                    logger.exception("_call out of retries on exception: %s", e)
                    raise e
                if is_retriable(e):
                    logger.debug("_call retrying after exception: %s", e)
                    continue
                logger.exception("_call non-retriable exception: %s", e)
                raise e
        if json_out:
            return json
        elif info_out:
            return info
        else:
            return headers, contents