Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
2. Truncates the exception and the traceback
3. Serializes the exception and traceback or
4. If they can't be serialized send string versions
5. Format a message and return
See Also
--------
clean_exception: deserialize and unpack message into exception/traceback
"""
MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
tblib.pickling_support.install(e, *collect_causes(e))
tb = get_traceback()
e2 = truncate_exception(e, MAX_ERROR_LEN)
try:
e3 = protocol.pickle.dumps(e2)
protocol.pickle.loads(e3)
except Exception:
e2 = Exception(str(e2))
e4 = protocol.to_serialize(e2)
try:
tb2 = protocol.pickle.dumps(tb)
except Exception:
tb = tb2 = "".join(traceback.format_tb(tb))
if len(tb2) > MAX_ERROR_LEN:
tb_result = None
else:
tb_result = protocol.to_serialize(tb)
return {"status": status, "exception": e4, "traceback": tb_result, "text": str(e2)}
def deserialize_numpy_maskedarray(header, frames):
data_header = header["data-header"]
data_frames = frames[: header["nframes"]]
data = deserialize_numpy_ndarray(data_header, data_frames)
if "mask-header" in header:
mask_header = header["mask-header"]
mask_frames = frames[header["nframes"] :]
mask = deserialize_numpy_ndarray(mask_header, mask_frames)
else:
mask = np.ma.nomask
pickled_fv, fill_value = header["fill-value"]
if pickled_fv:
fill_value = pickle.loads(fill_value)
return np.ma.masked_array(data, mask=mask, fill_value=fill_value)
def loads(x, *, buffers=()):
try:
if buffers:
return pickle.loads(x, buffers=buffers)
else:
return pickle.loads(x)
except Exception:
logger.info("Failed to deserialize %s", x[:10000], exc_info=True)
raise
def set_error(self, exception, traceback):
if isinstance(exception, bytes):
try:
exception = loads(exception)
except TypeError:
exception = Exception("Undeserializable exception", exception)
if traceback:
if isinstance(traceback, bytes):
try:
traceback = loads(traceback)
except (TypeError, AttributeError):
traceback = None
else:
traceback = None
self.status = 'error'
self.exception = exception
self.traceback = traceback
self._get_event().set()
def loads_function(bytes_object):
""" Load a function from bytes, cache bytes """
if len(bytes_object) < 100000:
try:
result = cache_loads[bytes_object]
except KeyError:
result = pickle.loads(bytes_object)
cache_loads[bytes_object] = result
return result
return pickle.loads(bytes_object)
def _deserialize(function=None, args=None, kwargs=None, task=no_value):
""" Deserialize task inputs and regularize to func, args, kwargs """
if function is not None:
function = loads_function(function)
if args:
args = pickle.loads(args)
if kwargs:
kwargs = pickle.loads(kwargs)
if task is not no_value:
assert not function and not args and not kwargs
function = execute_task
args = (task,)
return function, args or (), kwargs or {}
def _deserialize(function=None, args=None, kwargs=None, task=no_value):
""" Deserialize task inputs and regularize to func, args, kwargs """
if function is not None:
function = loads_function(function)
if args:
args = pickle.loads(args)
if kwargs:
kwargs = pickle.loads(kwargs)
if task is not no_value:
assert not function and not args and not kwargs
function = execute_task
args = (task,)
return function, args or (), kwargs or {}
def loads(x, *, buffers=()):
try:
if buffers:
return pickle.loads(x, buffers=buffers)
else:
return pickle.loads(x)
except Exception:
logger.info("Failed to deserialize %s", x[:10000], exc_info=True)
raise
def deserialize_numpy_ndarray(header, frames):
with log_errors():
if header.get("pickle"):
return pickle.loads(frames[0], buffers=frames[1:])
(frame,) = frames
is_custom, dt = header["dtype"]
if is_custom:
dt = pickle.loads(dt)
else:
dt = np.dtype(dt)
if header.get("broadcast_to"):
shape = header["broadcast_to"]
else:
shape = header["shape"]
x = np.ndarray(shape, dtype=dt, buffer=frame, strides=header["strides"])
return x