Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
import os
import sys
from .common import ZictBase
def _encode_key(key):
return key.encode('utf-8')
def _decode_key(key):
return key.decode('utf-8')
class LMDB(ZictBase):
""" Mutable Mapping interface to a LMDB database.
Keys must be strings, values must be bytes
Parameters
----------
directory: string
Examples
--------
>>> z = LMDB('/tmp/somedir/') # doctest: +SKIP
>>> z['x'] = b'123' # doctest: +SKIP
>>> z['x'] # doctest: +SKIP
b'123'
"""
def __init__(self, directory):
from __future__ import absolute_import, division, print_function
from heapdict import heapdict
from .common import ZictBase, close
def do_nothing(k, v):
pass
class LRU(ZictBase):
""" Evict Least Recently Used Elements
Parameters
----------
n: int
Number of elements to keep, or total weight if weight= is used
d: MutableMapping
Dictionary in which to hold elements
on_evict: list of callables
Function:: k, v -> action to call on key value pairs prior to eviction
weight: callable
Function:: k, v -> number to determine the size of keeping the item in
the mapping. Defaults to ``(k, v) -> 1``
Examples
--------
def _safe_key(key):
"""
Escape key so as to be usable on all filesystems.
"""
# Even directory separators are unsafe.
return quote(key, safe="")
def _unsafe_key(key):
"""
Undo the escaping done by _safe_key().
"""
return unquote(key)
class File(ZictBase):
""" Mutable Mapping interface to a directory
Keys must be strings, values must be bytes
Note this shouldn't be used for interprocess persistence, as keys
are cached in memory.
Parameters
----------
directory: string
mode: string, ('r', 'w', 'a'), defaults to 'a'
Examples
--------
>>> z = File('myfile') # doctest: +SKIP
>>> z['x'] = b'123' # doctest: +SKIP
from itertools import chain
from .common import ZictBase, close
from .lru import LRU
class Buffer(ZictBase):
""" Buffer one dictionary on top of another
This creates a MutableMapping by combining two MutableMappings, one that
feeds into the other when it overflows, based on an LRU mechanism. When
the first evicts elements these get placed into the second. When an item
is retrieved from the second it is placed back into the first.
Parameters
----------
fast: MutableMapping
slow: MutableMapping
fast_to_slow_callbacks: list of callables
These functions run every time data moves from the fast to the slow
mapping. They take two arguments, a key and a value
slow_to_fast_callbacks: list of callables
These functions run every time data moves form the slow to the fast
from collections import defaultdict
from itertools import chain
import sys
from .common import ZictBase, close
class Sieve(ZictBase):
""" Store values in different mappings based on a selector's
output.
This creates a MutableMapping combining several underlying
MutableMappings for storage. Items are dispatched based on
a selector function provided by the user.
Parameters
----------
mappings: dict of {mapping key: MutableMapping}
selector: callable (key, value) -> mapping key
Examples
--------
>>> small = {}
>>> large = DataBase() # doctest: +SKIP
from .common import ZictBase, close
class Func(ZictBase):
""" Buffer a MutableMapping with a pair of input/output functions
Parameters
----------
dump: callable
Function to call on value as we set it into the mapping
load: callable
Function to call on value as we pull it from the mapping
d: MutableMapping
Examples
--------
>>> def double(x):
... return x * 2
>>> def halve(x):
def device_deserialize(header, frames):
return DeviceSerialized(header["obj-header"], frames)
@nvtx_annotate("SPILL_D2H", color="red", domain="dask_cuda")
def device_to_host(obj: object) -> DeviceSerialized:
header, frames = serialize(obj, serializers=["dask", "pickle"])
return DeviceSerialized(header, frames)
@nvtx_annotate("SPILL_H2D", color="green", domain="dask_cuda")
def host_to_device(s: DeviceSerialized) -> object:
return deserialize(s.header, s.frames)
class DeviceHostFile(ZictBase):
""" Manages serialization/deserialization of objects.
Three LRU cache levels are controlled, for device, host and disk.
Each level takes care of serializing objects once its limit has been
reached and pass it to the subsequent level. Similarly, each cache
may deserialize the object, but storing it back in the appropriate
cache, depending on the type of object being deserialized.
Parameters
----------
device_memory_limit: int
Number of bytes of CUDA device memory for device LRU cache,
spills to host cache once filled.
memory_limit: int
Number of bytes of host memory for host LRU cache, spills to
disk once filled. Setting this to 0 means unlimited host memory,