How to use the tblib.pickling_support function in tblib

To help you get started, we’ve selected a few tblib examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github browsertron / pytest-parallel / pytest_parallel / __init__.py View on Github external
def run(self):
        pickling_support.install()
        while True:
            try:
                index = self.queue.get()
                if index == 'stop':
                    self.queue.task_done()
                    break
            except ConnectionRefusedError:
                time.sleep(.1)
                continue
            item = self.session.items[index]
            try:
                run_test(self.session, item, None)
            except BaseException:
                import pickle
                import sys
github Shopify / ci-queue / python / ciqueue / _pytest / outcomes.py View on Github external
be put them on the redis queue. Then, we swap back in the original exception
when reading off the queue. These operations are performed by
`swap_in_serializable` and `swap_back_original`, respectively.
"""


from __future__ import absolute_import
import dill
try:
    from _pytest import outcomes
except ImportError:
    from _pytest import runner as outcomes
from _pytest._code import code
from tblib import pickling_support

pickling_support.install()


class Skipped(Exception):
    """placeholder for outcomes.Skipped which is not serializable"""


class Failed(Exception):
    """placeholder for outcomes.Failed which is not serializable"""


class UnserializableException(Exception):
    """placeholder for any Exceptions that cannnot be serialized"""


SERIALIZE_TYPES = {outcomes.Skipped: Skipped,
                   outcomes.Failed: Failed}
github OpenNebula / one / src / oca / python / pyone / tester.py View on Github external
# limitations under the License.

from hashlib import md5
from json import load, dumps as json_dumps
from base64 import b64decode, b64encode
from pickle import dumps, loads
from os import path
from . import OneServer
from tblib import pickling_support
from sys import exc_info
from six import reraise
from collections import OrderedDict
from gzip import open
from os import environ

pickling_support.install()


def read_fixture_file(fixture_file):
    f = open(fixture_file, "rt")
    ret = load(f)
    f.close()
    return ret


def write_fixture_file(fixture_file, obj):
    f = open(fixture_file, "wb")
    f.write(json_dumps(obj).encode())
    f.close()

class OneServerTester(OneServer):
    '''
github dask / distributed / distributed / core.py View on Github external
""" Produce message to send back given an exception has occurred

    This does the following:

    1.  Gets the traceback
    2.  Truncates the exception and the traceback
    3.  Serializes the exception and traceback or
    4.  If they can't be serialized send string versions
    5.  Format a message and return

    See Also
    --------
    clean_exception: deserialize and unpack message into exception/traceback
    """
    MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
    tblib.pickling_support.install(e, *collect_causes(e))
    tb = get_traceback()
    e2 = truncate_exception(e, MAX_ERROR_LEN)
    try:
        e3 = protocol.pickle.dumps(e2)
        protocol.pickle.loads(e3)
    except Exception:
        e2 = Exception(str(e2))
    e4 = protocol.to_serialize(e2)
    try:
        tb2 = protocol.pickle.dumps(tb)
    except Exception:
        tb = tb2 = "".join(traceback.format_tb(tb))

    if len(tb2) > MAX_ERROR_LEN:
        tb_result = None
    else:
github dask / dask / dask / multiprocessing.py View on Github external
try:
            typ = type(
                exc.__class__.__name__,
                (RemoteException, type(exc)),
                {"exception_type": type(exc)},
            )
            exceptions[type(exc)] = typ
            return typ(exc, tb)
        except TypeError:
            return exc


try:
    import tblib.pickling_support

    tblib.pickling_support.install()

    def _pack_traceback(tb):
        return tb


except ImportError:

    def _pack_traceback(tb):
        return "".join(traceback.format_tb(tb))

    def reraise(exc, tb):
        exc = remote_exception(exc, tb)
        raise exc


def pack_exception(e, dumps):
github mozilla / OpenWPM / automation / BrowserManager.py View on Github external
import psutil
from multiprocess import Queue
from selenium.common.exceptions import WebDriverException
from six import reraise
from six.moves import cPickle as pickle
from six.moves.queue import Empty as EmptyQueue
from tblib import pickling_support

from .Commands import command_executor
from .DeployBrowsers import deploy_browser
from .Errors import BrowserConfigError, BrowserCrashError, ProfileLoadError
from .SocketInterface import clientsocket
from .utilities.multiprocess_utils import Process, parse_traceback_for_sentry

pickling_support.install()


class Browser:
    """
     The Browser class is responsbile for holding all of the
     configuration and status information on BrowserManager process
     it corresponds to. It also includes a set of methods for managing
     the BrowserManager process and its child processes/threads.
      are the TaskManager configuration settings.
      are per-browser parameter settings (e.g. whether
                      this browser is headless, etc.)
     """

    def __init__(self, manager_params, browser_params):
        # Constants
        self._SPAWN_TIMEOUT = 120  # seconds
github OpenMined / PySyft / syft / workers / websocket_server.py View on Github external
import binascii
from typing import Union
from typing import List

import asyncio
import torch
import websockets
import ssl
import sys
import tblib.pickling_support
import socket
import logging

tblib.pickling_support.install()

import syft as sy
from syft.frameworks.torch.tensors.interpreters import AbstractTensor
from syft.workers.virtual import VirtualWorker
from syft.exceptions import GetNotPermittedError
from syft.exceptions import ResponseSignatureError
from syft.federated import FederatedClient


class WebsocketServerWorker(VirtualWorker, FederatedClient):
    def __init__(
        self,
        hook,
        host: str,
        port: int,
        id: Union[int, str] = 0,
github mozilla / OpenWPM / automation / MPLogger.py View on Github external
import sys
import threading
import time

import dill
import sentry_sdk
import six
from multiprocess import JoinableQueue
from sentry_sdk.integrations.logging import BreadcrumbHandler, EventHandler
from six.moves.queue import Empty as EmptyQueue
from tblib import pickling_support

from .Commands.utils.webdriver_utils import parse_neterror
from .SocketInterface import serversocket

pickling_support.install()

BROWSER_PREFIX = re.compile(r"^BROWSER (-)?\d+:\s*")
# These config variable names should name to lowercase kwargs for MPLogger
ENV_CONFIG_VARS = [
    'LOG_LEVEL_CONSOLE',
    'LOG_LEVEL_FILE',
    'LOG_LEVEL_SENTRY_BREADCRUMB',
    'LOG_LEVEL_SENTRY_EVENT'
]


def _retrive_log_level_from_env(env_var_name):
    """Retrieve log level from `env_var_name`

    Levels from: https://docs.python.org/3/library/logging.html#levels"""
    level = os.getenv(env_var_name, None)
github janelia-flyem / DVIDSparkServices / DVIDSparkServices / subprocess_decorator.py View on Github external
import logging
import threading
from functools import partial
from contextlib import contextmanager
from multiprocessing import Process, TimeoutError, Pipe, Event

import ctypes
from ctypes.util import find_library

try:
    libc = ctypes.cdll.msvcrt # Windows
except OSError:
    libc = ctypes.cdll.LoadLibrary(find_library('c'))

from tblib import pickling_support
pickling_support.install()

class SysExcInfo(tuple):
    """
    A special class for storing sys.exc_info from a child process.
    """
    pass

def execute_in_subprocess(timeout=None, stream_logger=None):
    """
    Excecutes the "decorated" function in a subprocess,
    waits for the result, and returns it.

    Note: Returns a pseudo-decorator.
          For technical reasons, you can't actually use Python @-syntax.
    
    Example: