How to use the gcsfs.utils.HttpError function in gcsfs

To help you get started, we’ve selected a few gcsfs examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dask / gcsfs / gcsfs / utils.py View on Github external
def is_retriable(exception):
    """Returns True if this exception is retriable."""
    errs = list(range(500, 505)) + [
        # Request Timeout
        408,
        # Too Many Requests
        429,
    ]
    errs += [str(e) for e in errs]
    if isinstance(exception, HttpError):
        return exception.code in errs

    return isinstance(exception, RETRIABLE_EXCEPTIONS)
github dask / gcsfs / gcsfs / core.py View on Github external
timeout=self.requests_timeout,
                ) as r:
                    import json

                    status = r.status
                    headers = r.headers
                    info = r.request_info  # for debug only
                    contents = await r.read()
                    try:
                        json = json.loads(contents)
                    except Exception:
                        json = None

                self.validate_response(status, contents, json, path)
                break
            except (HttpError, RequestException, GoogleAuthError) as e:
                json = None
                if (
                    isinstance(e, HttpError)
                    and e.code == 400
                    and "requester pays" in e.message
                ):
                    msg = "Bucket is requester pays. Set `requester_pays=True` when creating the GCSFileSystem."
                    raise ValueError(msg) from e
                if retry == self.retries - 1:
                    logger.exception("_call out of retries on exception: %s", e)
                    raise e
                if is_retriable(e):
                    logger.debug("_call retrying after exception: %s", e)
                    continue
                logger.exception("_call non-retriable exception: %s", e)
                raise e
github dask / gcsfs / gcsfs / core.py View on Github external
except:  # noqa: E722
                # TODO: limit to appropriate exceptions
                msg = content

            if status == 404:
                raise FileNotFoundError
            elif status == 403:
                raise IOError("Forbidden: %s\n%s" % (path, msg))
            elif status == 502:
                raise ProxyError()
            elif "invalid" in str(msg):
                raise ValueError("Bad Request: %s\n%s" % (path, msg))
            elif error:
                raise HttpError(error)
            elif status:
                raise HttpError({"code": status})
            else:
                raise RuntimeError(msg)
github dask / gcsfs / gcsfs / core.py View on Github external
error = json["error"]
                msg = error["message"]
            except:  # noqa: E722
                # TODO: limit to appropriate exceptions
                msg = content

            if status == 404:
                raise FileNotFoundError
            elif status == 403:
                raise IOError("Forbidden: %s\n%s" % (path, msg))
            elif status == 502:
                raise ProxyError()
            elif "invalid" in str(msg):
                raise ValueError("Bad Request: %s\n%s" % (path, msg))
            elif error:
                raise HttpError(error)
            elif status:
                raise HttpError({"code": status})
            else:
                raise RuntimeError(msg)
github dask / gcsfs / gcsfs / utils.py View on Github external
def __init__(self, error_response=None):
        if error_response:
            self.message = error_response.get("message", "")
            self.code = error_response.get("code", None)
        else:
            self.message = ""
            self.code = None
        # Call the base class constructor with the parameters it needs
        super(HttpError, self).__init__(self.message)
github dask / gcsfs / gcsfs / core.py View on Github external
status = r.status
                    headers = r.headers
                    info = r.request_info  # for debug only
                    contents = await r.read()
                    try:
                        json = json.loads(contents)
                    except Exception:
                        json = None

                self.validate_response(status, contents, json, path)
                break
            except (HttpError, RequestException, GoogleAuthError) as e:
                json = None
                if (
                    isinstance(e, HttpError)
                    and e.code == 400
                    and "requester pays" in e.message
                ):
                    msg = "Bucket is requester pays. Set `requester_pays=True` when creating the GCSFileSystem."
                    raise ValueError(msg) from e
                if retry == self.retries - 1:
                    logger.exception("_call out of retries on exception: %s", e)
                    raise e
                if is_retriable(e):
                    logger.debug("_call retrying after exception: %s", e)
                    continue
                logger.exception("_call non-retriable exception: %s", e)
                raise e
        if json_out:
            return json
        elif info_out: