Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
@pytest.mark.parametrize("unpacker_class", [msgpack.Unpacker, msgpack.fallback.Unpacker])
def testUnpacker(self, unpacker_class):
unpacker = unpacker_class(raw=False)
data = msgpack.packb(self.test_data, use_bin_type=True)
data += msgpack.packb(self.test_data, use_bin_type=True)
messages = []
for char in data:
unpacker.feed(bytes([char]))
for message in unpacker:
messages.append(message)
assert len(messages) == 2
assert messages[0] == self.test_data
assert messages[0] == messages[1]
def getUnpacker(fallback=False, decode=True):
if fallback: # Pure Python
unpacker = msgpack.fallback.Unpacker
else:
unpacker = msgpack.Unpacker
if decode: # Workaround for backward compatibility: Try to decode bin to str
unpacker = unpacker(raw=True, object_pairs_hook=objectDecoderHook, max_buffer_size=5 * 1024 * 1024)
else:
unpacker = unpacker(raw=False, max_buffer_size=5 * 1024 * 1024)
return unpacker
def unpackb(packed, object_hook=decode, encoding='utf-8', **kwargs):
"""
Unpack a packed object.
"""
kwargs['object_hook'] = object_hook
return _unpacker.unpackb(packed, encoding=encoding, **kwargs)
profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data))
return np.fromstring(obj['data'],
dtype=np.dtype(obj['type'])).reshape(obj['shape'])
else:
return np.fromstring(obj['data'],
dtype=np.dtype(obj['type']))[0]
elif 'complex' in obj:
return complex(obj['data'])
else:
return obj
except KeyError:
return obj
# Maintain support for msgpack < 0.4.0:
if msgpack.version < (0, 4, 0):
class Packer(_packer.Packer):
def __init__(self, default=encode,
encoding='utf-8',
unicode_errors='strict',
use_single_float=False,
autoreset=1):
super(Packer, self).__init__(default=default,
encoding=encoding,
unicode_errors=unicode_errors,
use_single_float=use_single_float,
autoreset=1)
class Unpacker(_unpacker.Unpacker):
def __init__(self, file_like=None, read_size=0, use_list=None,
object_hook=decode,
object_pairs_hook=None, list_hook=None, encoding='utf-8',
except NameError:
xrange = range # Python 3
# Python 2.6 is generally angry about newer msgpack
# This is enough enough of a hack to make tests pass
try:
memoryview
except NameError:
import struct
# bolt itemsize onto the string class to let
# it look enough like memoryview for msgpack
class memoryview_ish(str):
itemsize = 1
# and install it into msgpack's namespace
messagepack.fallback.memoryview = memoryview_ish
# in addition, looks like struct.unpack_from
# gets angry when you feed it bytearrays, so
# monkey patch that too
def wrap_unpack_from():
func = struct.unpack_from
def unpack_from_wrapper(*args, **kwargs):
if isinstance(args[1], bytearray):
args = list(args)
args[1] = str(args[1])
return func(*args, **kwargs)
return unpack_from_wrapper
struct.unpack_from = wrap_unpack_from()
# probably the right thing to do is to stop supporting 2.6...
import io
import logging
import msgpack
import msgpack.fallback as m_fallback
logger = logging.getLogger(__name__)
# Single Packer object which is reused for performance
pakr = msgpack.Packer(use_bin_type=True, unicode_errors='surrogatepass')
if isinstance(pakr, m_fallback.Packer): # pragma: no cover
logger.warning('msgpack is using the pure python fallback implementation. This will impact performance negatively.')
pakr = None
# synapse.lib.msgpack.un uses a hardcoded subset of these arguments for speed
unpacker_kwargs = {
'raw': False,
'use_list': False,
'max_buffer_size': 2**32 - 1,
'unicode_errors': 'surrogatepass'
}
def en(item):
'''
Use msgpack to serialize a compatible python object.
Args:
def is_slow_msgpack():
import msgpack
import msgpack.fallback
return msgpack.Packer is msgpack.fallback.Packer
try:
while not self.closed:
buff = self.sock.recv(64 * 1024)
if not buff:
break # Connection closed
buff_len = len(buff)
# Statistics
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += buff_len
self.server.bytes_recv += buff_len
req_len += buff_len
if not self.unpacker:
self.unpacker = msgpack.fallback.Unpacker()
unpacker_bytes = 0
self.unpacker.feed(buff)
unpacker_bytes += buff_len
while True:
try:
message = self.unpacker.next()
except StopIteration:
break
if not type(message) is dict:
if config.debug_socket:
self.log("Invalid message type: %s, content: %r, buffer: %r" % (type(message), message, buff[0:16]))
raise Exception("Invalid message type: %s" % type(message))
# Stats
def simple(name, data):
if has_ext:
packer = _packer.Packer()
profile("packing %s (ext)" % name, lambda: packer.pack(data))
packer = fallback.Packer()
profile("packing %s (fallback)" % name, lambda: packer.pack(data))
data = packer.pack(data)
if has_ext:
profile("unpacking %s (ext)" % name, lambda: _unpacker.unpackb(data))
profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data))