How to use the blosc.unpack_array function in blosc

To help you get started, we’ve selected a few blosc examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Blosc / python-blosc / bench / compare-pack-ptr.py View on Github external
#in_ = np.random.random_integers(0, 100, N)  # random distribution
print(" ", in_)

tic = time.time()
out_ = np.copy(in_)
toc = time.time()
print("  Time for copying array with np.copy():     %.3f s" % (toc-tic,))
print()

for cname in blosc.compressor_list():
    print("Using *** %s *** compressor::" % cname)
    ctic = time.time()
    c = blosc.pack_array(in_, clevel=clevel, shuffle=True, cname=cname)
    ctoc = time.time()
    dtic = time.time()
    out = blosc.unpack_array(c)
    dtoc = time.time()
    assert((in_ == out).all())
    print("  Time for pack_array/unpack_array:     %.3f/%.3f s." % \
          (ctoc-ctic, dtoc-dtic), end='')
    print("\tCompr ratio: %.2f" % (in_.size*in_.dtype.itemsize*1. / len(c)))

    ctic = time.time()
    c = blosc.compress_ptr(in_.__array_interface__['data'][0],
                           in_.size, in_.dtype.itemsize,
                           clevel=clevel, shuffle=True, cname=cname)
    ctoc = time.time()
    out = np.empty(in_.size, dtype=in_.dtype)
    dtic = time.time()
    blosc.decompress_ptr(c, out.__array_interface__['data'][0])
    dtoc = time.time()
    assert((in_ == out).all())
github neurodata / ndstore / scripts / time_serialization_new.py View on Github external
hdf5_ds_values = []
blosc_ds_values = []
numpy_ss_values = []
hdf5_ss_values = []
blosc_ss_values = []
x_axis_values = []

print "-----SERIALIZATION TIME-----"

for i in range(0, ITERATIONS, 1):

  CUBE_VALUE = int(BASE_SIZE*math.pow(2,i))
  x_axis_values.append(CUBE_VALUE)
  print "SIZE:{}".format(CUBE_VALUE)
  cutout_args = (OFFSET[0], OFFSET[0]+CUBE_VALUE, OFFSET[1], OFFSET[1]+CUBE_VALUE, OFFSET[2], OFFSET[2]+Z_SIZE)
  data = blosc.unpack_array(getURL(generateURLBlosc(HOST, TOKEN, CHANNELS, RESOLUTION, cutout_args)))
  # data = np.asarray(range(CUBE_VALUE*CUBE_VALUE*Z_SIZE), dtype=np.uint32).reshape(CUBE_VALUE,CUBE_VALUE,Z_SIZE)

  # NUMPY
  start = time.time()
  fileobj = cStringIO.StringIO()
  np.save(fileobj, data)
  test = zlib.compress(fileobj.getvalue())
  numpy_ts_values.append(time.time()-start)
  # numpy_ss_values.append(sys.getsizeof(data)/sys.getsizeof(test))
  numpy_ss_values.append(sys.getsizeof(test))

  start = time.time()
  test = np.load ( cStringIO.StringIO ( zlib.decompress ( test ) ) )
  numpy_ds_values.append(time.time()-start)
  
  # HDF5
github neurodata / ndstore / scripts / time_serialization.py View on Github external
hdf5_ss_values.append(sys.getsizeof(tmpfile.read()))
  tmpfile.seek(0)
  start = time.time()
  fh5in = h5py.File(tmpfile.name, driver='core', backing_store=True)
  test = np.array(fh5in['TEST']['CUTOUT'])
  hdf5_ds_values.append(time.time()-start)
  fh5in.close()
  tmpfile.close()
  
  # BLOSC
  start = time.time()
  test = blosc.pack_array(data)
  blosc_ts_values.append(time.time()-start)
  blosc_ss_values.append(sys.getsizeof(test))
  start = time.time()
  test = blosc.unpack_array(test)
  blosc_ds_values.append(time.time()-start)


# opening a pdf file
pp = PdfPages('time_serialization.pdf')

# Time Serlization Graph

# plot values
plt.figure(figsize=(10,10))
plt.plot(x_axis_values, numpy_ts_values, color='green', marker='o')
plt.plot(x_axis_values, hdf5_ts_values, color='blue', marker='^')
plt.plot(x_axis_values, blosc_ts_values, color='red', marker='s')

# configure x-axis
plt.xlim(0,5500)
github wehr-lab / autopilot / autopilot / hardware / cameras.py View on Github external
'-pix_fmt': 'yuv420p',
                '-r': str(self.fps),
                '-preset': 'ultrafast',
            },
            verbosity=1
        )

        try:

            for input in iter(self.q.get, 'END'):
                try:

                    if self.given_timestamps:
                        self.timestamps.append(input[0])
                        if self.blosc:
                            vid_out.writeFrame(blosc.unpack_array(input[1]))
                        else:
                            vid_out.writeFrame(input[1])
                    else:
                        self.timestamps.append(datetime.now().isoformat())
                        if self.blosc:
                            vid_out.writeFrame(blosc.unpack_array(input))
                        else:
                            vid_out.writeFrame(input)

                except Exception as e:
                    print(e)
                    traceback.print_tb()
                    # TODO: Too general
                    break

        finally:
github hwang595 / ps_pytorch / src / compression.py View on Github external
def g_decompress(msg):
    if sys.version_info[0] < 3:
        # Python 2.x implementation
        assert isinstance(msg, str)
    else:
        # Python 3.x implementation
        assert isinstance(msg, bytes)
    grad = blosc.unpack_array(msg)
    return grad
github CoffeaTeam / coffea / coffea / processor / dask / __init__.py View on Github external
def setup(self, worker):
        self.cache = Buffer(
            fast={},
            slow=Func(
                dump=blosc.pack_array,
                load=blosc.unpack_array,
                d=Buffer(
                    fast={},
                    slow=LRU(
                        n=self._maxdisk,
                        d=File(os.path.join(worker.local_directory, 'cache')),
                        weight=lambda k, v: len(v),
                    ),
                    n=self._maxcompressed,
                    weight=lambda k, v: len(v),
                ),
            ),
            n=self._maxmem,
            weight=lambda k, v: v.nbytes,
        )
        self.lock = Lock()
        self.hits = 0
github uncbiag / easyreg / easyreg / reg_data_loader_onfly.py View on Github external
if has_label:
                sitk_pair_list[2],_ = self.resize_img(sitk_pair_list[2], is_label=True)
                sitk_pair_list[3],_ = self.resize_img(sitk_pair_list[3], is_label=True)
            pair_list = [sitk.GetArrayFromImage(sitk_pair) for sitk_pair in sitk_pair_list]
            pair_list = [item.astype(np.float32) for item in pair_list]
            img_after_resize = self.img_after_resize if self.img_after_resize is not None else original_sz
            new_spacing=  original_spacing*(original_sz-1)/(np.array(img_after_resize)-1)
            spacing = self._normalize_spacing(new_spacing,img_after_resize, silent_mode=True)


        else:
            zipnp_pair_list = self.pair_list[idx]
            spacing = self.spacing_list[idx]
            original_spacing = self.original_spacing_list[idx]
            original_sz = self.original_sz_list[idx]
            pair_list = [blosc.unpack_array(item) for item in zipnp_pair_list]

        sample = {'image': np.asarray([self.normalize_intensity(pair_list[0]),self.normalize_intensity(pair_list[1])])}
        sample['pair_path'] = pair_path
        if self.load_init_weight:
            sample['init_weight']=self.init_weight_list[idx]

        if has_label:
            try:
                sample ['label']= np.asarray([pair_list[2], pair_list[3]]).astype(np.float32)
            except:
                print(pair_list[2].shape,pair_list[3].shape)
                print(filename)
        # else:
        #     sample['label'] = None
        if self.transform:
            sample['image'] = self.transform(sample['image'])
github uncbiag / easyreg / data_pre / seg_data_loader_online.py View on Github external
sample = {}

        ########################### channel sel
        if self.transform:
            index = [0] if not self.is_brats else [0,1,2,3]
            if self.add_loc:
                index = index + [index[-1]+1 for _ in range(3)]
            if self.is_train:
                sample['image'] = self.transform(data['img'][index].copy())*2-1
            else:
                if not self.use_org_size:
                    sample['image'] = self.transform(data['img'][:,index].copy())*2-1
                else:
                    sample['image'] = self.transform(data['img'][index].copy()) * 2 - 1
            if self.add_resampled:
                sample['resampled_img'] = self.transform(blosc.unpack_array((dic['resampled_img'])))
            if  self.detect_et: #self.use_org_size or self.detect_et:
                sample['checked_label']=self.transform(data['checked_label'].astype(np.int32))
            #sample['image'] = self.transform(data['img'].copy())
            if 'seg'in data and data['seg'] is not None:
                 sample['label'] = self.transform(data['seg'].copy())
            else:
                sample['label'] = self.transform(np.array([-1]).astype((np.int32)))
        # global count
        # count[1] +=1
        # print(count)
        return sample,fname
github jhuapl-boss / intern / ndio / service / boss / v0_4 / volume.py View on Github external
Raises:
            requests.HTTPError
        """

        req = self.get_cutout_request(
            resource, 'GET', 'application/blosc-python',
            url_prefix, auth, resolution, x_range, y_range, z_range, time_range)
        prep = session.prepare_request(req)
        # Hack in Accept header for now.
        prep.headers['Accept'] = 'application/blosc-python'
        #resp = session.send(prep, stream = True, **send_opts)
        resp = session.send(prep, **send_opts)
        
        if resp.status_code == 200:
            return blosc.unpack_array(resp.content)

        msg = ('Get cutout failed on {}, got HTTP response: ({}) - {}'.format(
            resource.name, resp.status_code, resp.text))
        raise HTTPError(msg, request = req, response = resp)