How to use the distributed.protocol.pickle.loads function in distributed

To help you get started, we’ve selected a few distributed examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dask / distributed / distributed / core.py View on Github external
2.  Truncates the exception and the traceback
    3.  Serializes the exception and traceback or
    4.  If they can't be serialized send string versions
    5.  Format a message and return

    See Also
    --------
    clean_exception: deserialize and unpack message into exception/traceback
    """
    MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
    tblib.pickling_support.install(e, *collect_causes(e))
    tb = get_traceback()
    e2 = truncate_exception(e, MAX_ERROR_LEN)
    try:
        e3 = protocol.pickle.dumps(e2)
        protocol.pickle.loads(e3)
    except Exception:
        e2 = Exception(str(e2))
    e4 = protocol.to_serialize(e2)
    try:
        tb2 = protocol.pickle.dumps(tb)
    except Exception:
        tb = tb2 = "".join(traceback.format_tb(tb))

    if len(tb2) > MAX_ERROR_LEN:
        tb_result = None
    else:
        tb_result = protocol.to_serialize(tb)

    return {"status": status, "exception": e4, "traceback": tb_result, "text": str(e2)}
github dask / distributed / distributed / protocol / numpy.py View on Github external
def deserialize_numpy_maskedarray(header, frames):
    data_header = header["data-header"]
    data_frames = frames[: header["nframes"]]
    data = deserialize_numpy_ndarray(data_header, data_frames)

    if "mask-header" in header:
        mask_header = header["mask-header"]
        mask_frames = frames[header["nframes"] :]
        mask = deserialize_numpy_ndarray(mask_header, mask_frames)
    else:
        mask = np.ma.nomask

    pickled_fv, fill_value = header["fill-value"]
    if pickled_fv:
        fill_value = pickle.loads(fill_value)

    return np.ma.masked_array(data, mask=mask, fill_value=fill_value)
github dask / distributed / distributed / protocol / pickle.py View on Github external
def loads(x, *, buffers=()):
    try:
        if buffers:
            return pickle.loads(x, buffers=buffers)
        else:
            return pickle.loads(x)
    except Exception:
        logger.info("Failed to deserialize %s", x[:10000], exc_info=True)
        raise
github dask / distributed / distributed / client.py View on Github external
def set_error(self, exception, traceback):
        if isinstance(exception, bytes):
            try:
                exception = loads(exception)
            except TypeError:
                exception = Exception("Undeserializable exception", exception)
        if traceback:
            if isinstance(traceback, bytes):
                try:
                    traceback = loads(traceback)
                except (TypeError, AttributeError):
                    traceback = None
        else:
            traceback = None

        self.status = 'error'
        self.exception = exception
        self.traceback = traceback
        self._get_event().set()
github dask / distributed / distributed / worker.py View on Github external
def loads_function(bytes_object):
    """ Load a function from bytes, cache bytes """
    if len(bytes_object) < 100000:
        try:
            result = cache_loads[bytes_object]
        except KeyError:
            result = pickle.loads(bytes_object)
            cache_loads[bytes_object] = result
        return result
    return pickle.loads(bytes_object)
github dask / distributed / distributed / worker.py View on Github external
def _deserialize(function=None, args=None, kwargs=None, task=no_value):
    """ Deserialize task inputs and regularize to func, args, kwargs """
    if function is not None:
        function = loads_function(function)
    if args:
        args = pickle.loads(args)
    if kwargs:
        kwargs = pickle.loads(kwargs)

    if task is not no_value:
        assert not function and not args and not kwargs
        function = execute_task
        args = (task,)

    return function, args or (), kwargs or {}
github dask / distributed / distributed / worker.py View on Github external
def _deserialize(function=None, args=None, kwargs=None, task=no_value):
    """ Deserialize task inputs and regularize to func, args, kwargs """
    if function is not None:
        function = loads_function(function)
    if args:
        args = pickle.loads(args)
    if kwargs:
        kwargs = pickle.loads(kwargs)

    if task is not no_value:
        assert not function and not args and not kwargs
        function = execute_task
        args = (task,)

    return function, args or (), kwargs or {}
github dask / distributed / distributed / protocol / pickle.py View on Github external
def loads(x, *, buffers=()):
    try:
        if buffers:
            return pickle.loads(x, buffers=buffers)
        else:
            return pickle.loads(x)
    except Exception:
        logger.info("Failed to deserialize %s", x[:10000], exc_info=True)
        raise
github dask / distributed / distributed / protocol / numpy.py View on Github external
def deserialize_numpy_ndarray(header, frames):
    with log_errors():
        if header.get("pickle"):
            return pickle.loads(frames[0], buffers=frames[1:])

        (frame,) = frames

        is_custom, dt = header["dtype"]
        if is_custom:
            dt = pickle.loads(dt)
        else:
            dt = np.dtype(dt)

        if header.get("broadcast_to"):
            shape = header["broadcast_to"]
        else:
            shape = header["shape"]

        x = np.ndarray(shape, dtype=dt, buffer=frame, strides=header["strides"])

        return x