How to use the fsspec.asyn.sync function in fsspec

To help you get started, we’ve selected a few fsspec examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dask / gcsfs / gcsfs / core.py View on Github external
def _initiate_upload(self):
        """ Create multi-upload """
        self.location = sync(
            self.gcsfs.loop,
            initiate_upload,
            self.gcsfs,
            self.bucket,
            self.key,
            self.content_type,
            self.metadata,
        )
github intake / filesystem_spec / fsspec / implementations / http.py View on Github external
def __init__(self, fs, url, mode="rb", loop=None, session=None, **kwargs):
        self.asynchronous = kwargs.pop("asynchronous", False)
        self.url = url
        self.loop = loop
        self.session = session
        if mode != "rb":
            raise ValueError
        self.details = {"name": url, "size": None}
        super().__init__(fs=fs, path=url, mode=mode, cache_type="none", **kwargs)
        self.r = sync(self.loop, get, self.session, url, **kwargs)
github intake / filesystem_spec / fsspec / implementations / http.py View on Github external
http/https matching the input URLs.
        size_policy: this argument is deprecated
        storage_options: key-value
            May be credentials, e.g., `{'auth': ('username', 'pword')}` or any
            other parameters passed on to requests
        cache_type, cache_options: defaults used in open
        """
        AbstractFileSystem.__init__(self, asynchronous=asynchronous, loop=loop)
        self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE
        self.simple_links = simple_links
        self.same_schema = same_scheme
        self.cache_type = cache_type
        self.cache_options = cache_options
        self.kwargs = storage_options
        if not asynchronous:
            self._session = sync(self.loop, get_client)
            weakref.finalize(self, sync, self.loop, self.session.close)
        else:
            self._session = None
github dask / gcsfs / gcsfs / core.py View on Github external
def _simple_upload(self):
        """One-shot upload, less than 5MB"""
        self.buffer.seek(0)
        data = self.buffer.read()
        sync(
            self.gcsfs.loop,
            simple_upload,
            self.gcsfs,
            self.bucket,
            self.key,
            data,
            self.metadata,
            self.consistency,
            self.content_type,
        )
github intake / filesystem_spec / fsspec / implementations / http.py View on Github external
size_policy: this argument is deprecated
        storage_options: key-value
            May be credentials, e.g., `{'auth': ('username', 'pword')}` or any
            other parameters passed on to requests
        cache_type, cache_options: defaults used in open
        """
        AbstractFileSystem.__init__(self, asynchronous=asynchronous, loop=loop)
        self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE
        self.simple_links = simple_links
        self.same_schema = same_scheme
        self.cache_type = cache_type
        self.cache_options = cache_options
        self.kwargs = storage_options
        if not asynchronous:
            self._session = sync(self.loop, get_client)
            weakref.finalize(self, sync, self.loop, self.session.close)
        else:
            self._session = None
github dask / gcsfs / gcsfs / core.py View on Github external
if block_size is not None:
            self.default_block_size = block_size
        self.project = project
        self.requester_pays = requester_pays
        self.access = access
        self.credentials = None
        self.heads = {}
        self.lock = threading.Lock()
        self.scope = "https://www.googleapis.com/auth/devstorage." + access
        self.consistency = consistency
        self.token = token
        self.cache_timeout = cache_timeout or kwargs.pop("listings_expiry_time", None)
        self.requests_timeout = requests_timeout
        self.check_credentials = check_connection
        if not asynchronous:
            self._session = sync(self.loop, get_client)
            weakref.finalize(self, sync, self.loop, self.session.close)
        else:
            self._session = None
        self.connect(method=token)
github dask / gcsfs / gcsfs / core.py View on Github external
self.default_block_size = block_size
        self.project = project
        self.requester_pays = requester_pays
        self.access = access
        self.credentials = None
        self.heads = {}
        self.lock = threading.Lock()
        self.scope = "https://www.googleapis.com/auth/devstorage." + access
        self.consistency = consistency
        self.token = token
        self.cache_timeout = cache_timeout or kwargs.pop("listings_expiry_time", None)
        self.requests_timeout = requests_timeout
        self.check_credentials = check_connection
        if not asynchronous:
            self._session = sync(self.loop, get_client)
            weakref.finalize(self, sync, self.loop, self.session.close)
        else:
            self._session = None
        self.connect(method=token)
github dask / gcsfs / gcsfs / core.py View on Github external
def rm(self, path, recursive=False, batchsize=20):
        paths = self.expand_path(path, recursive=recursive)
        sync(self.loop, self._rm, paths, batchsize=batchsize)
github dask / gcsfs / gcsfs / core.py View on Github external
def buckets(self):
        """Return list of available project buckets."""
        return [b["name"] for b in sync(self.loop, self._list_buckets())]