How to use the fsspec.registry.get_filesystem_class function in fsspec

To help you get started, we’ve selected a few fsspec examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github intake / filesystem_spec / fsspec / core.py View on Github external
return out
    x = re.compile(".*[^a-z]+.*")  # test for non protocol-like single word
    bits = (
        [p if "://" in p or x.match(p) else p + "://" for p in path.split("::")]
        if "::" in path
        else [path]
    )
    if len(bits) < 2:
        return []
    # [[url, protocol, kwargs], ...]
    out = []
    previous_bit = None
    previous_protocol = None
    for bit in reversed(bits):
        protocol = split_protocol(bit)[0] or "file"
        cls = get_filesystem_class(protocol)
        extra_kwargs = cls._get_kwargs_from_urls(bit)
        kws = kwargs.get(split_protocol(bit)[0] or "file", {})
        kw = dict(**extra_kwargs, **kws)
        if (
            protocol in {"blockcache", "filecache", "simplecache"}
            and "target_protocol" not in kw
        ):
            bit = previous_bit.replace(previous_protocol, protocol)
        out.append((bit, protocol, kw))
        previous_bit = bit
        previous_protocol = protocol
    out = list(reversed(out))
    # We should only do the url rewrite if the cache is in the middle of the chain
    if out[0][1] in {"blockcache", "filecache", "simplecache"}:
        out[0] = (f"{out[0][1]}://", out[0][1], out[0][2])
    return out
github intake / filesystem_spec / fsspec / core.py View on Github external
inkwargs["target_options"] = kw.copy()
            inkwargs["fo"] = urls
            inkwargs = inkwargs["target_options"]
        protocol = chain[0][1]
    if isinstance(urlpath, (list, tuple)):
        if not urlpath:
            raise ValueError("empty urlpath sequence")
        protocols, paths = zip(*map(split_protocol, urlpath))
        if protocol is None:
            protocol = protocols[0]
            if not all(p == protocol for p in protocols):
                raise ValueError(
                    "When specifying a list of paths, all paths must "
                    "share the same protocol"
                )
        cls = get_filesystem_class(protocol)
        optionss = list(map(cls._get_kwargs_from_urls, urlpath))
        paths = [cls._strip_protocol(u) for u in urlpath]
        options = optionss[0]
        if not all(o == options for o in optionss):
            raise ValueError(
                "When specifying a list of paths, all paths must "
                "share the same file-system options"
            )
        update_storage_options(options, storage_options)
        fs = cls(**options)
        paths = expand_paths_if_needed(paths, mode, num, fs, name_function)

    elif isinstance(urlpath, str) or hasattr(urlpath, "name"):
        protocols, path = split_protocol(urlpath)
        protocol = protocol or protocols
        cls = get_filesystem_class(protocol)
github intake / filesystem_spec / fsspec / core.py View on Github external
def strip_protocol(urlpath):
    """Return only path part of full URL, according to appropriate backend"""
    protocol, _ = split_protocol(urlpath)
    cls = get_filesystem_class(protocol)
    return cls._strip_protocol(urlpath)
github intake / filesystem_spec / fsspec / core.py View on Github external
optionss = list(map(cls._get_kwargs_from_urls, urlpath))
        paths = [cls._strip_protocol(u) for u in urlpath]
        options = optionss[0]
        if not all(o == options for o in optionss):
            raise ValueError(
                "When specifying a list of paths, all paths must "
                "share the same file-system options"
            )
        update_storage_options(options, storage_options)
        fs = cls(**options)
        paths = expand_paths_if_needed(paths, mode, num, fs, name_function)

    elif isinstance(urlpath, str) or hasattr(urlpath, "name"):
        protocols, path = split_protocol(urlpath)
        protocol = protocol or protocols
        cls = get_filesystem_class(protocol)

        options = cls._get_kwargs_from_urls(urlpath)
        path = cls._strip_protocol(urlpath)
        update_storage_options(options, storage_options)
        fs = cls(**options)

        if "w" in mode and expand:
            paths = _expand_paths(path, name_function, num)
        elif "*" in path:
            paths = [f for f in sorted(fs.glob(path)) if not fs.isdir(f)]
        else:
            paths = [path]

    else:
        raise TypeError("url type not understood: %s" % urlpath)
github intake / filesystem_spec / fsspec / spec.py View on Github external
----------
        blob: str

        Returns
        -------
        file system instance, not necessarily of this particular class.
        """
        from .registry import _import_class, get_filesystem_class
        import json

        dic = json.loads(blob)
        protocol = dic.pop("protocol")
        try:
            cls = _import_class(dic.pop("cls"))
        except (ImportError, ValueError, RuntimeError, KeyError):
            cls = get_filesystem_class(protocol)
        return cls(*dic.pop("args", ()), **dic)