How to use the zarr.Blosc function in zarr

To help you get started, we’ve selected a few zarr examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github veugene / data_tools / data_tools / io.py View on Github external
def __init__(self, data_element_shape, dtype, batch_size, filename,
                 array_name, length=None, append=False, kwargs=None):
        import zarr
        super(zarr_array_writer, self).__init__(None, data_element_shape,
                                                dtype, batch_size, length)
        self.filename = filename
        self.array_name = array_name
        self.kwargs = kwargs
        
        # Set up array kwargs
        self.arr_kwargs = {'name': array_name,
                           'chunks': (batch_size,)+data_element_shape,
                           'compressor': zarr.Blosc(cname='lz4',
                                                    clevel=5,
                                                    shuffle=1),
                           'dtype': dtype}
        if self.length is None:
            self.arr_kwargs['shape'] = (1,)+self.data_element_shape
        else:
            self.arr_kwargs['shape'] = (self.length,)+self.data_element_shape
        if kwargs is not None:
            self.arr_kwargs.update(kwargs)
    
        # Open the file for writing.
        self.group = None
        if append:
            self.write_mode = 'a'
        else:
            self.write_mode = 'w'
github dcs4cop / xcube / xcube / core / dsio.py View on Github external
chunks.append(var.shape[i])
                encoding[var_name] = dict(chunks=chunks)
        if packing:
            if encoding:
                for var_name in packing.keys():
                    if var_name in encoding.keys():
                        encoding[var_name].update(dict(packing[var_name]))
                    else:
                        encoding[var_name] = dict(packing[var_name])
            else:
                encoding = {}
                for var_name in packing.keys():
                    encoding[var_name] = dict(packing[var_name])

        if compressor:
            compressor = zarr.Blosc(**compressor)

            if encoding:
                for var_name in encoding.keys():
                    encoding[var_name].update(compressor=compressor)
            else:
                encoding = {var_name: dict(compressor=compressor) for var_name in dataset.data_vars}
        return encoding
github dcs4cop / xcube / e2e / cube-to-zarr.py View on Github external
import cablab as cl
import xarray as xr
import zarr

print(xr.version.version)

cube = cl.Cube.open('D:\\EOData\\cablab-datacube-1.0.0\\low-res')

print('converting to xarray dataset')
ds = cube.data.dataset()

print('writing uncompressed zarr')
ds.to_zarr('D:\\EOData\\cablab-datacube-1.0.0.zarr')

print('writing compressed zarr')
compressor = zarr.Blosc(cname='zstd', clevel=3, shuffle=2)
ds.to_zarr('D:\\EOData\\cablab-datacube-1.0.0.comp.zarr',
           encoding={'cablab-datacube-1.0.0.comp': {'compressor': compressor}})