How to use the deepdish.io.ls.NumpyArrayNode function in deepdish

To help you get started, we’ve selected a few deepdish examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github uchicago-cs / deepdish / deepdish / io / ls.py View on Github external
return ObjectNode()
        node = NumpyArrayNode(level.shape, 'unknown')
        return node
    elif isinstance(level, tables.Array):
        stats = {}
        if settings.get('summarize'):
            stats['mean'] = level[:].mean()
            stats['std'] = level[:].std()

        compression = {}
        if settings.get('compression'):
            compression['complib'] = level.filters.complib
            compression['shuffle'] = level.filters.shuffle
            compression['complevel'] = level.filters.complevel

        node = NumpyArrayNode(level.shape, _format_dtype(level.dtype),
                              statistics=stats, compression=compression)

        if hasattr(level._v_attrs, 'zeroarray_dtype'):
            dtype = level._v_attrs.zeroarray_dtype
            node = NumpyArrayNode(tuple(level), _format_dtype(dtype))

        elif hasattr(level._v_attrs, 'strtype'):
            strtype = level._v_attrs.strtype
            itemsize = level._v_attrs.itemsize
            if strtype == b'unicode':
                shape = level.shape[:-1] + (level.shape[-1] // itemsize // 4,)
            elif strtype == b'ascii':
                shape = level.shape

            node = NumpyArrayNode(shape, strtype.decode('ascii'))
github uchicago-cs / deepdish / deepdish / io / ls.py View on Github external
if settings.get('summarize'):
            stats['mean'] = level[:].mean()
            stats['std'] = level[:].std()

        compression = {}
        if settings.get('compression'):
            compression['complib'] = level.filters.complib
            compression['shuffle'] = level.filters.shuffle
            compression['complevel'] = level.filters.complevel

        node = NumpyArrayNode(level.shape, _format_dtype(level.dtype),
                              statistics=stats, compression=compression)

        if hasattr(level._v_attrs, 'zeroarray_dtype'):
            dtype = level._v_attrs.zeroarray_dtype
            node = NumpyArrayNode(tuple(level), _format_dtype(dtype))

        elif hasattr(level._v_attrs, 'strtype'):
            strtype = level._v_attrs.strtype
            itemsize = level._v_attrs.itemsize
            if strtype == b'unicode':
                shape = level.shape[:-1] + (level.shape[-1] // itemsize // 4,)
            elif strtype == b'ascii':
                shape = level.shape

            node = NumpyArrayNode(shape, strtype.decode('ascii'))

        return node
    elif isinstance(level, tables.link.SoftLink):
        node = SoftLinkNode(level.target)
        return node
    else:
github uchicago-cs / deepdish / deepdish / io / ls.py View on Github external
node = NumpyArrayNode(level.shape, _format_dtype(level.dtype),
                              statistics=stats, compression=compression)

        if hasattr(level._v_attrs, 'zeroarray_dtype'):
            dtype = level._v_attrs.zeroarray_dtype
            node = NumpyArrayNode(tuple(level), _format_dtype(dtype))

        elif hasattr(level._v_attrs, 'strtype'):
            strtype = level._v_attrs.strtype
            itemsize = level._v_attrs.itemsize
            if strtype == b'unicode':
                shape = level.shape[:-1] + (level.shape[-1] // itemsize // 4,)
            elif strtype == b'ascii':
                shape = level.shape

            node = NumpyArrayNode(shape, strtype.decode('ascii'))

        return node
    elif isinstance(level, tables.link.SoftLink):
        node = SoftLinkNode(level.target)
        return node
    else:
        return Node()
github uchicago-cs / deepdish / deepdish / io / ls.py View on Github external
for grp in level:
            node.add(grp._v_name, _tree_level(grp, raw=raw, settings=settings))

        for name in level._v_attrs._f_list():
            v = level._v_attrs[name]
            if name == DEEPDISH_IO_VERSION_STR:
                node.header['dd_io_version'] = v

            if name == DEEPDISH_IO_UNPACK:
                node.header['dd_io_unpack'] = v

            if name.startswith(DEEPDISH_IO_PREFIX):
                continue

            if isinstance(v, np.ndarray):
                node.add(name, NumpyArrayNode(v.shape, _format_dtype(v.dtype)))
            else:
                node.add(name, ValueNode(v))

        if (level._v_title.startswith('list:') or
                level._v_title.startswith('tuple:')):
            s = level._v_title.split(':', 1)[1]
            N = int(s)
            lst = ListNode(typename=level._v_title.split(':')[0])
            for i in range(N):
                t = node.children['i{}'.format(i)]
                lst.append(t)
            return lst
        elif level._v_title.startswith('nonetype:'):
            return ValueNode(None)
        elif is_pandas_dataframe(level):
            pandas_type = level._v_attrs['pandas_type']
github uchicago-cs / deepdish / deepdish / io / ls.py View on Github external
new_node = PandasPanelNode(shape)
                return new_node
            # else: it will simply be treated as a dict

        elif level._v_title.startswith('sparse:') and not raw:
            frm = level._v_attrs.format
            dtype = level.data.dtype
            shape = tuple(level.shape[:])
            node = SparseMatrixNode(frm, shape, dtype)
            return node

        return node
    elif isinstance(level, tables.VLArray):
        if level.shape == (1,):
            return ObjectNode()
        node = NumpyArrayNode(level.shape, 'unknown')
        return node
    elif isinstance(level, tables.Array):
        stats = {}
        if settings.get('summarize'):
            stats['mean'] = level[:].mean()
            stats['std'] = level[:].std()

        compression = {}
        if settings.get('compression'):
            compression['complib'] = level.filters.complib
            compression['shuffle'] = level.filters.shuffle
            compression['complevel'] = level.filters.complevel

        node = NumpyArrayNode(level.shape, _format_dtype(level.dtype),
                              statistics=stats, compression=compression)