How to use the pyarrow.serialize function in pyarrow

To help you get started, we’ve selected a few pyarrow examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github SurrealAI / surreal / surreal / utils / serializer.py View on Github external
def pa_serialize(obj):
    return pa.serialize(obj).to_buffer()
github Lyken17 / Efficient-PyTorch / tools / folder2lmdb.py View on Github external
def dumps_pyarrow(obj):
    """
    Serialize an object.

    Returns:
        Implementation-dependent bytes-like object
    """
    return pa.serialize(obj).to_buffer()
github deepdrive / deepdrive-sim / Content / Scripts / api_server.py View on Github external
def serialize(obj):
    try:
        ret = pyarrow.serialize(obj).to_buffer()
    except pyarrow.lib.SerializationCallbackError:
        print('Could not serialize with pyarrow - falling back to str(obj)')
        ret = pyarrow.serialize({'success': True, 'result': str(obj)}).to_buffer()
    except:
        ret = {'success': False, 'result': traceback.format_exc()}
    return ret
github nismod / smif / src / smif / data_layer / datafile_interface.py View on Github external
def _write_data_to_native_file(filepath, data):
    with pa.OSFile(filepath, 'wb') as native_file:
        native_file.write(pa.serialize(data).to_buffer())
github guanfuchen / semseg / semseg / dataloader / folder2lmdb.py View on Github external
def dumps_pyarrow(obj):
    """
    Serialize an object.
    Returns:
        Implementation-dependent bytes-like object
    """
    return pyarrow.serialize(obj).to_buffer()
github hydro-project / droplet / droplet / shared / serializer.py View on Github external
def _dump_numpy(self, msg):
        return pa.serialize(msg).to_buffer().to_pybytes()
github SurrealAI / surreal / surreal / distributed / inmemory.py View on Github external
def inmem_serialize(data, name=None):
    """
        Serialize data into pyarrow format,
        Save to a memory mapped file, return filename
    Args:
        @data: python object to be serialized. 
               At least supports native types, dict, list and numpy.array
               If data is pyarrow.lib.Buffer, saves directly
        @name (Optional):
    """
    if name is None:
        name = os.path.join(TEMP_FOLDER, str(uuid.uuid4()))
    buf = pa.serialize(data).to_buffer()
    with pa.MemoryMappedFile.create(name, buf.size) as f:
        f.write(buf)
    return name.encode()
github nismod / smif / src / smif / data_layer / datafile_interface.py View on Github external
def _write_data_to_native_file(filepath, data):
        with pa.OSFile(filepath, 'wb') as f:
            f.write(
                pa.serialize(data).to_buffer()
            )
github cyoon1729 / Distributed-Reinforcement-Learning / common / abstract / learner.py View on Github external
def send_new_priorities(self, idxes: np.ndarray, priorities: np.ndarray):
        new_priors = [idxes, priorities]
        new_priors_id = pa.serialize(new_priors).to_buffer()
        self.rep_socket.send(new_priors_id)