How to use the ipyparallel.serialize.deserialize_object function in ipyparallel

To help you get started, we’ve selected a few ipyparallel examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github ipython / ipyparallel / ipyparallel / client / client.py View on Github external
md = self.metadata[msg_id]

        if msg_type == 'stream':
            name = content['name']
            s = md[name] or ''
            md[name] = s + content['text']
        elif msg_type == 'error':
            md.update({'error' : self._unwrap_exception(content)})
        elif msg_type == 'execute_input':
            md.update({'execute_input' : content['code']})
        elif msg_type == 'display_data':
            md['outputs'].append(content)
        elif msg_type == 'execute_result':
            md['execute_result'] = content
        elif msg_type == 'data_message':
            data, remainder = serialize.deserialize_object(msg['buffers'])
            md['data'].update(data)
        elif msg_type == 'status':
            # idle message comes after all outputs
            if content['execution_state'] == 'idle':
                future = self._output_futures.get(msg_id)
                if future and not future.done():
                    # TODO: should probably store actual outputs on the Future
                    future.set_result(None)
        else:
            # unhandled msg_type (status, etc.)
            pass
github ipython / ipyparallel / ipyparallel / client / client.py View on Github external
md = self.metadata[msg_id]
                md_msg = dict(
                    content=rcontent,
                    parent_header=parent,
                    header=header,
                    metadata=rec['result_metadata'],
                )
                md.update(self._extract_metadata(md_msg))
                if rec.get('received'):
                    md['received'] = util._parse_date(rec['received'])
                md.update(iodict)

                if rcontent['status'] == 'ok':
                    if header['msg_type'] == 'apply_reply':
                        res,buffers = serialize.deserialize_object(buffers)
                    elif header['msg_type'] == 'execute_reply':
                        res = ExecuteReply(msg_id, rcontent, md)
                    else:
                        raise KeyError("unhandled msg type: %r" % header['msg_type'])
                else:
                    res = self._unwrap_exception(rcontent)
                    failures.append(res)

                self.results[msg_id] = res
                content[msg_id] = res

        if len(theids) == 1 and failures:
            raise failures[0]

        error.collect_exceptions(failures, "result_status")
        return content
github Parsl / parsl / parsl / executors / swift_t.py View on Github external
return

            except Exception as e:
                logger.debug("[MTHREAD] Caught unknown exception: {}".format(e))

            else:

                if msg is None:
                    logger.debug("[MTHREAD] Got None")
                    return

                else:
                    logger.debug("[MTHREAD] Received message: {}".format(msg))
                    task_fut = self.tasks[msg['task_id']]
                    if 'result' in msg:
                        result, _ = deserialize_object(msg['result'])
                        task_fut.set_result(result)

                    elif 'exception' in msg:
                        exception, _ = deserialize_object(msg['exception'])
                        task_fut.set_exception(exception)

            if not self.is_alive:
                break
github ipython / ipyparallel / examples / interengine / bintree.py View on Github external
def deserialize(self, msg):
        """inverse of serialize"""
        return deserialize_object(msg)[0]
github Parsl / parsl / parsl / executors / swift_t.py View on Github external
# Handle received message
            if not msg:
                # Empty message is a die request
                logger.debug("[RUNNER] Received exit request")
                outgoing_q.put(None)
                break
            else:
                # Received a valid message, handle it
                logger.debug("[RUNNER] Got a valid task with ID {}".format(msg["task_id"]))
                try:
                    response_obj = execute_task(msg['buffer'])
                    response = {"task_id": msg["task_id"],
                                "result": serialize_object(response_obj)}

                    logger.debug("[RUNNER] Returing result: {}".format(
                                   deserialize_object(response["result"])))

                except Exception as e:
                    logger.debug("[RUNNER] Caught task exception: {}".format(e))
                    response = {"task_id": msg["task_id"],
                                "exception": serialize_object(e)}

                outgoing_q.put(response)

    logger.debug("[RUNNER] Terminating")
github funcx-faas / funcX / funcx / executors / high_throughput / executor.py View on Github external
if tid == -2 and 'info' in msg:
                            logger.warning("Received info response : {}".format(msg['info']))
                            try:
                                if self.endpoint_db:
                                    self.endpoint_db.put(self.endpoint_id, msg['info'])
                            except Exception as e:
                                logger.exception("Caught error while trying to push data into redis")
                                pass
                            continue

                        if tid == -1 and 'exception' in msg:
                            # TODO: This could be handled better we are essentially shutting down the
                            # client with little indication to the user.
                            logger.warning("Executor shutting down due to version mismatch in interchange")
                            self._executor_exception, _ = deserialize_object(msg['exception'])
                            logger.exception("Exception: {}".format(self._executor_exception))
                            # Set bad state to prevent new tasks from being submitted
                            self._executor_bad_state.set()
                            # We set all current tasks to this exception to make sure that
                            # this is raised in the main context.
                            for task in self.tasks:
                                self.tasks[task].set_exception(self._executor_exception)
                            break

                        logger.warning("YADU: HERE with {}".format(tid))
                        task_fut = self.tasks[tid]

                        if 'result' in msg or 'exception' in msg:
                            task_fut.set_result(msg)
                        else:
                            raise BadMessage("Message received is neither result or exception")
github ipython / ipyparallel / ipyparallel / client / client.py View on Github external
else:
            self.outstanding.remove(msg_id)
        content = msg['content']
        header = msg['header']

        # construct metadata:
        md = self.metadata[msg_id]
        md.update(self._extract_metadata(msg))

        e_outstanding = self._outstanding_dict[md['engine_uuid']]
        if msg_id in e_outstanding:
            e_outstanding.remove(msg_id)

        # construct result:
        if content['status'] == 'ok':
            self.results[msg_id] = serialize.deserialize_object(msg['buffers'])[0]
        elif content['status'] == 'aborted':
            self.results[msg_id] = error.TaskAborted(msg_id)
            out_future = self._output_futures.get(msg_id)
            if out_future and not out_future.done():
                out_future.set_result(None)
        elif content['status'] == 'resubmitted':
            # TODO: handle resubmission
            pass
        else:
            self.results[msg_id] = self._unwrap_exception(content)
        if content['status'] != 'ok' and not content.get('engine_info'):
            # not an engine failure, don't expect output
            out_future = self._output_futures.get(msg_id)
            if out_future and not out_future.done():
                out_future.set_result(None)
        if future: