How to use the numcodecs.MsgPack function in numcodecs

To help you get started, we’ve selected a few numcodecs examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github jdfekete / progressivis / progressivis / storage / zarr.py View on Github external
from __future__ import absolute_import, division, print_function

from collections import Iterable

import numpy as np

import zarr
from zarr.attrs import Attributes
from zarr.storage import init_group, contains_group
import numcodecs
zarr.codecs.codec_registry[numcodecs.Pickle.codec_id] = numcodecs.Pickle
zarr.codecs.codec_registry[numcodecs.MsgPack.codec_id] = numcodecs.MsgPack

from progressivis.core.config import get_option
from .base import StorageEngine, Group, Attribute, Dataset


# For now (April 21st 2017), zarr misses two important features: fancy indexing and Boolean indexing.
# These two features are scheduled for inclusion in future releases of zarr.


Group.register(zarr.Group)
Attribute.register(Attributes)
Dataset.register(zarr.Array)

class ZARRGroup(zarr.Group):
    def __init__(self, store, **kwds):
        super(ZARRGroup, self).__init__(store, **kwds)
github jdfekete / progressivis / progressivis / storage / zarr.py View on Github external
def create_dataset(self, name, shape=None, dtype=None, data=None, maxshape=None, **kwds):
        _ = maxshape
        if kwds.get('compression') is None:
            kwds.update(get_option('storage.zarr.filter', {}))
        filters = kwds.get('filters', [])
        filters.append(numcodecs.MsgPack())
        kwds['filters'] = filters
        if dtype == np.dtype('O') and kwds.get('object_codec') is None:
            kwds.update({'object_codec': numcodecs.VLenUTF8()})
        return super(ZARRGroup, self).create_dataset(name, shape=shape, dtype=dtype, data=data, **kwds)