Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def __init__(self, data_element_shape, dtype, batch_size, filename,
array_name, length=None, append=False, kwargs=None):
import zarr
super(zarr_array_writer, self).__init__(None, data_element_shape,
dtype, batch_size, length)
self.filename = filename
self.array_name = array_name
self.kwargs = kwargs
# Set up array kwargs
self.arr_kwargs = {'name': array_name,
'chunks': (batch_size,)+data_element_shape,
'compressor': zarr.Blosc(cname='lz4',
clevel=5,
shuffle=1),
'dtype': dtype}
if self.length is None:
self.arr_kwargs['shape'] = (1,)+self.data_element_shape
else:
self.arr_kwargs['shape'] = (self.length,)+self.data_element_shape
if kwargs is not None:
self.arr_kwargs.update(kwargs)
# Open the file for writing.
self.group = None
if append:
self.write_mode = 'a'
else:
self.write_mode = 'w'
chunks.append(var.shape[i])
encoding[var_name] = dict(chunks=chunks)
if packing:
if encoding:
for var_name in packing.keys():
if var_name in encoding.keys():
encoding[var_name].update(dict(packing[var_name]))
else:
encoding[var_name] = dict(packing[var_name])
else:
encoding = {}
for var_name in packing.keys():
encoding[var_name] = dict(packing[var_name])
if compressor:
compressor = zarr.Blosc(**compressor)
if encoding:
for var_name in encoding.keys():
encoding[var_name].update(compressor=compressor)
else:
encoding = {var_name: dict(compressor=compressor) for var_name in dataset.data_vars}
return encoding
import cablab as cl
import xarray as xr
import zarr
print(xr.version.version)
cube = cl.Cube.open('D:\\EOData\\cablab-datacube-1.0.0\\low-res')
print('converting to xarray dataset')
ds = cube.data.dataset()
print('writing uncompressed zarr')
ds.to_zarr('D:\\EOData\\cablab-datacube-1.0.0.zarr')
print('writing compressed zarr')
compressor = zarr.Blosc(cname='zstd', clevel=3, shuffle=2)
ds.to_zarr('D:\\EOData\\cablab-datacube-1.0.0.comp.zarr',
encoding={'cablab-datacube-1.0.0.comp': {'compressor': compressor}})