How to use the msgpack.fallback function in msgpack

To help you get started, we’ve selected a few msgpack examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github HelloZeroNet / ZeroNet / src / Test / TestMsgpack.py View on Github external
    @pytest.mark.parametrize("unpacker_class", [msgpack.Unpacker, msgpack.fallback.Unpacker])
    def testUnpacker(self, unpacker_class):
        unpacker = unpacker_class(raw=False)

        data = msgpack.packb(self.test_data, use_bin_type=True)
        data += msgpack.packb(self.test_data, use_bin_type=True)

        messages = []
        for char in data:
            unpacker.feed(bytes([char]))
            for message in unpacker:
                messages.append(message)

        assert len(messages) == 2
        assert messages[0] == self.test_data
        assert messages[0] == messages[1]
github HelloZeroNet / ZeroNet / src / util / Msgpack.py View on Github external
def getUnpacker(fallback=False, decode=True):
    if fallback:  # Pure Python
        unpacker = msgpack.fallback.Unpacker
    else:
        unpacker = msgpack.Unpacker

    if decode:  # Workaround for backward compatibility: Try to decode bin to str
        unpacker = unpacker(raw=True, object_pairs_hook=objectDecoderHook, max_buffer_size=5 * 1024 * 1024)
    else:
        unpacker = unpacker(raw=False, max_buffer_size=5 * 1024 * 1024)

    return unpacker
github alexcwyu / python-trading / algotrader / utils / msgpack_numpy.py View on Github external
def unpackb(packed, object_hook=decode, encoding='utf-8', **kwargs):
    """
    Unpack a packed object.
    """

    kwargs['object_hook'] = object_hook
    return _unpacker.unpackb(packed, encoding=encoding, **kwargs)
github msgpack / msgpack-python / benchmark / benchmark.py View on Github external
    profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data))
github alexcwyu / python-trading / algotrader / utils / msgpack_numpy.py View on Github external
return np.fromstring(obj['data'],
                                     dtype=np.dtype(obj['type'])).reshape(obj['shape'])
            else:
                return np.fromstring(obj['data'],
                                     dtype=np.dtype(obj['type']))[0]
        elif 'complex' in obj:
            return complex(obj['data'])
        else:
            return obj
    except KeyError:
        return obj


# Maintain support for msgpack < 0.4.0:
if msgpack.version < (0, 4, 0):
    class Packer(_packer.Packer):
        def __init__(self, default=encode,
                     encoding='utf-8',
                     unicode_errors='strict',
                     use_single_float=False,
                     autoreset=1):
            super(Packer, self).__init__(default=default,
                                         encoding=encoding,
                                         unicode_errors=unicode_errors,
                                         use_single_float=use_single_float,
                                         autoreset=1)


    class Unpacker(_unpacker.Unpacker):
        def __init__(self, file_like=None, read_size=0, use_list=None,
                     object_hook=decode,
                     object_pairs_hook=None, list_hook=None, encoding='utf-8',
github NationalSecurityAgency / lemongraph / LemonGraph / serializer.py View on Github external
except NameError:
    xrange = range  # Python 3

# Python 2.6 is generally angry about newer msgpack
# This is enough enough of a hack to make tests pass
try:
    memoryview
except NameError:
    import struct

    # bolt itemsize onto the string class to let
    # it look enough like memoryview for msgpack
    class memoryview_ish(str):
        itemsize = 1
    # and install it into msgpack's namespace
    messagepack.fallback.memoryview = memoryview_ish

    # in addition, looks like struct.unpack_from
    # gets angry when you feed it bytearrays, so
    # monkey patch that too
    def wrap_unpack_from():
        func = struct.unpack_from
        def unpack_from_wrapper(*args, **kwargs):
            if isinstance(args[1], bytearray):
                args = list(args)
                args[1] = str(args[1])
            return func(*args, **kwargs)
        return unpack_from_wrapper
    struct.unpack_from = wrap_unpack_from()

    # probably the right thing to do is to stop supporting 2.6...
github vertexproject / synapse / synapse / lib / msgpack.py View on Github external
import io
import logging
import msgpack
import msgpack.fallback as m_fallback

logger = logging.getLogger(__name__)

# Single Packer object which is reused for performance
pakr = msgpack.Packer(use_bin_type=True, unicode_errors='surrogatepass')
if isinstance(pakr, m_fallback.Packer):  # pragma: no cover
    logger.warning('msgpack is using the pure python fallback implementation. This will impact performance negatively.')
    pakr = None

# synapse.lib.msgpack.un uses a hardcoded subset of these arguments for speed
unpacker_kwargs = {
    'raw': False,
    'use_list': False,
    'max_buffer_size': 2**32 - 1,
    'unicode_errors': 'surrogatepass'
}

def en(item):
    '''
    Use msgpack to serialize a compatible python object.

    Args:
github borgbackup / borg / src / borg / helpers / msgpack.py View on Github external
def is_slow_msgpack():
    import msgpack
    import msgpack.fallback
    return msgpack.Packer is msgpack.fallback.Packer
github krixano / ZeroNet-Windows-Exe-Installer / ZeroBundle / ZeroNet / src / Connection / Connection.py View on Github external
try:
            while not self.closed:
                buff = self.sock.recv(64 * 1024)
                if not buff:
                    break  # Connection closed
                buff_len = len(buff)

                # Statistics
                self.last_recv_time = time.time()
                self.incomplete_buff_recv += 1
                self.bytes_recv += buff_len
                self.server.bytes_recv += buff_len
                req_len += buff_len

                if not self.unpacker:
                    self.unpacker = msgpack.fallback.Unpacker()
                    unpacker_bytes = 0

                self.unpacker.feed(buff)
                unpacker_bytes += buff_len

                while True:
                    try:
                        message = self.unpacker.next()
                    except StopIteration:
                        break
                    if not type(message) is dict:
                        if config.debug_socket:
                            self.log("Invalid message type: %s, content: %r, buffer: %r" % (type(message), message, buff[0:16]))
                        raise Exception("Invalid message type: %s" % type(message))

                    # Stats
github msgpack / msgpack-python / benchmark / benchmark.py View on Github external
def simple(name, data):
    if has_ext:
        packer = _packer.Packer()
        profile("packing %s (ext)" % name, lambda: packer.pack(data))
    packer = fallback.Packer()
    profile("packing %s (fallback)" % name, lambda: packer.pack(data))

    data = packer.pack(data)
    if has_ext:
        profile("unpacking %s (ext)" % name, lambda: _unpacker.unpackb(data))
    profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data))