How to use the multiprocess.forking.ForkingPickler.register function in multiprocess

To help you get started, we’ve selected a few multiprocess examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github uqfoundation / multiprocess / py3.3 / multiprocess / managers.py View on Github external
from time import time as _time

#
# Register some things for pickling
#

def reduce_array(a):
    return array.array, (a.typecode, a.tobytes())
ForkingPickler.register(array.array, reduce_array)

view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
if view_types[0] is not list:       # only needed in Py3.0
    def rebuild_as_list(obj):
        return list, (list(obj),)
    for view_type in view_types:
        ForkingPickler.register(view_type, rebuild_as_list)
        import copyreg
        copyreg.pickle(view_type, rebuild_as_list)

#
# Type for identifying shared objects
#

class Token(object):
    '''
    Type to uniquely indentify a shared object
    '''
    __slots__ = ('typeid', 'address', 'id')

    def __init__(self, typeid, address, id):
        (self.typeid, self.address, self.id) = (typeid, address, id)
github uqfoundation / multiprocess / py2.7 / multiprocess / sharedctypes.py View on Github external
def rebuild_ctype(type_, wrapper, length):
    if length is not None:
        type_ = type_ * length
    ForkingPickler.register(type_, reduce_ctype)
    obj = type_.from_address(wrapper.get_address())
    obj._wrapper = wrapper
    return obj
github uqfoundation / multiprocess / py3.3 / multiprocess / sharedctypes.py View on Github external
def rebuild_ctype(type_, wrapper, length):
    if length is not None:
        type_ = type_ * length
    ForkingPickler.register(type_, reduce_ctype)
    buf = wrapper.create_memoryview()
    obj = type_.from_buffer(buf)
    obj._wrapper = wrapper
    return obj
github uqfoundation / multiprocess / py2.6 / multiprocess / reduction.py View on Github external
s = socket.fromfd(fd, family, type_, proto)
    if s.__class__ is not socket.socket:
        s = socket.socket(_sock=s)
    return s

def reduce_socket(s):
    reduced_handle = reduce_handle(s.fileno())
    return rebuild_socket, (reduced_handle, s.family, s.type, s.proto)

def rebuild_socket(reduced_handle, family, type_, proto):
    fd = rebuild_handle(reduced_handle)
    _sock = fromfd(fd, family, type_, proto)
    close(fd)
    return _sock

ForkingPickler.register(socket.socket, reduce_socket)

#
# Register `_multiprocessing.PipeConnection` with `ForkingPickler`
#

if sys.platform == 'win32':

    def reduce_pipe_connection(conn):
        rh = reduce_handle(conn.fileno())
        return rebuild_pipe_connection, (rh, conn.readable, conn.writable)

    def rebuild_pipe_connection(reduced_handle, readable, writable):
        handle = rebuild_handle(reduced_handle)
        return _multiprocessing.PipeConnection(
            handle, readable=readable, writable=writable
            )
github uqfoundation / multiprocess / py3.2 / multiprocess / managers.py View on Github external
try:
    from dill import PicklingError
except ImportError:
    from pickle import PicklingError
from multiprocess import Process, current_process, active_children, Pool, util, connection
from multiprocess.process import AuthenticationString
from multiprocess.forking import exit, Popen, assert_spawning, ForkingPickler
from multiprocess.util import Finalize, info

#
# Register some things for pickling
#

def reduce_array(a):
    return array.array, (a.typecode, a.tobytes())
ForkingPickler.register(array.array, reduce_array)

view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
if view_types[0] is not list:       # only needed in Py3.0
    def rebuild_as_list(obj):
        return list, (list(obj),)
    for view_type in view_types:
        ForkingPickler.register(view_type, rebuild_as_list)
        import copyreg
        copyreg.pickle(view_type, rebuild_as_list)

#
# Type for identifying shared objects
#

class Token(object):
    '''
github uqfoundation / multiprocess / py2.6 / multiprocess / reduction.py View on Github external
#
# Register `_multiprocessing.Connection` with `ForkingPickler`
#

def reduce_connection(conn):
    rh = reduce_handle(conn.fileno())
    return rebuild_connection, (rh, conn.readable, conn.writable)

def rebuild_connection(reduced_handle, readable, writable):
    handle = rebuild_handle(reduced_handle)
    return _multiprocessing.Connection(
        handle, readable=readable, writable=writable
        )

ForkingPickler.register(_multiprocessing.Connection, reduce_connection)

#
# Register `socket.socket` with `ForkingPickler`
#

def fromfd(fd, family, type_, proto=0):
    s = socket.fromfd(fd, family, type_, proto)
    if s.__class__ is not socket.socket:
        s = socket.socket(_sock=s)
    return s

def reduce_socket(s):
    reduced_handle = reduce_handle(s.fileno())
    return rebuild_socket, (reduced_handle, s.family, s.type, s.proto)

def rebuild_socket(reduced_handle, family, type_, proto):
github uqfoundation / multiprocess / py3.3 / multiprocess / connection.py View on Github external
#
# Make connection and socket objects sharable if possible
#

if sys.platform == 'win32':
    from . import reduction
    ForkingPickler.register(socket.socket, reduction.reduce_socket)
    ForkingPickler.register(Connection, reduction.reduce_connection)
    ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
else:
    try:
        from . import reduction
    except ImportError:
        pass
    else:
        ForkingPickler.register(socket.socket, reduction.reduce_socket)
        ForkingPickler.register(Connection, reduction.reduce_connection)
github uqfoundation / multiprocess / py3.3 / multiprocess / connection.py View on Github external
# Make connection and socket objects sharable if possible
#

if sys.platform == 'win32':
    from . import reduction
    ForkingPickler.register(socket.socket, reduction.reduce_socket)
    ForkingPickler.register(Connection, reduction.reduce_connection)
    ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
else:
    try:
        from . import reduction
    except ImportError:
        pass
    else:
        ForkingPickler.register(socket.socket, reduction.reduce_socket)
        ForkingPickler.register(Connection, reduction.reduce_connection)
github uqfoundation / multiprocess / py2.6 / multiprocess / reduction.py View on Github external
# Register `_multiprocessing.PipeConnection` with `ForkingPickler`
#

if sys.platform == 'win32':

    def reduce_pipe_connection(conn):
        rh = reduce_handle(conn.fileno())
        return rebuild_pipe_connection, (rh, conn.readable, conn.writable)

    def rebuild_pipe_connection(reduced_handle, readable, writable):
        handle = rebuild_handle(reduced_handle)
        return _multiprocessing.PipeConnection(
            handle, readable=readable, writable=writable
            )

    ForkingPickler.register(_multiprocessing.PipeConnection, reduce_pipe_connection)
github uqfoundation / multiprocess / py2.6 / multiprocess / managers.py View on Github external
try:
    from dill import PicklingError
except ImportError:
    try:
        from cPickle import PicklingError
    except ImportError:
        from pickle import PicklingError

#
# Register some things for pickling
#

def reduce_array(a):
    return array.array, (a.typecode, a.tostring())
ForkingPickler.register(array.array, reduce_array)

view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]

#
# Type for identifying shared objects
#

class Token(object):
    '''
    Type to uniquely indentify a shared object
    '''
    __slots__ = ('typeid', 'address', 'id')

    def __init__(self, typeid, address, id):
        (self.typeid, self.address, self.id) = (typeid, address, id)