Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def pa_serialize(obj):
return pa.serialize(obj).to_buffer()
def dumps_pyarrow(obj):
"""
Serialize an object.
Returns:
Implementation-dependent bytes-like object
"""
return pa.serialize(obj).to_buffer()
def serialize(obj):
try:
ret = pyarrow.serialize(obj).to_buffer()
except pyarrow.lib.SerializationCallbackError:
print('Could not serialize with pyarrow - falling back to str(obj)')
ret = pyarrow.serialize({'success': True, 'result': str(obj)}).to_buffer()
except:
ret = {'success': False, 'result': traceback.format_exc()}
return ret
def _write_data_to_native_file(filepath, data):
with pa.OSFile(filepath, 'wb') as native_file:
native_file.write(pa.serialize(data).to_buffer())
def dumps_pyarrow(obj):
"""
Serialize an object.
Returns:
Implementation-dependent bytes-like object
"""
return pyarrow.serialize(obj).to_buffer()
def _dump_numpy(self, msg):
return pa.serialize(msg).to_buffer().to_pybytes()
def inmem_serialize(data, name=None):
"""
Serialize data into pyarrow format,
Save to a memory mapped file, return filename
Args:
@data: python object to be serialized.
At least supports native types, dict, list and numpy.array
If data is pyarrow.lib.Buffer, saves directly
@name (Optional):
"""
if name is None:
name = os.path.join(TEMP_FOLDER, str(uuid.uuid4()))
buf = pa.serialize(data).to_buffer()
with pa.MemoryMappedFile.create(name, buf.size) as f:
f.write(buf)
return name.encode()
def _write_data_to_native_file(filepath, data):
with pa.OSFile(filepath, 'wb') as f:
f.write(
pa.serialize(data).to_buffer()
)
def send_new_priorities(self, idxes: np.ndarray, priorities: np.ndarray):
new_priors = [idxes, priorities]
new_priors_id = pa.serialize(new_priors).to_buffer()
self.rep_socket.send(new_priors_id)