How to use hdf5plugin - 7 common examples

Default: 0 (for about 8kB per block).
    :param bool lz4:
        Whether to use LZ4_ID compression or not as part of the filter.
        Default: True
    """
    filter_id = BSHUF_ID

    def __init__(self, nelems=0, lz4=True):
        nelems = int(nelems)
        assert nelems % 8 == 0

        lz4_enabled = 2 if lz4 else 0
        self.filter_options = (nelems, lz4_enabled)


class LZ4(_FilterRefClass):
    """h5py.Group.create_dataset's compression and compression_opts arguments for using lz4 filter.

    :param int nelems:
        The number of bytes per block.
        Default: 0 (for 1GB per block).
    """
    filter_id = LZ4_ID

    def __init__(self, nbytes=0):
        nbytes = int(nbytes)
        assert 0 <= nbytes <= 0x7E000000
        self.filter_options = (nbytes,)


def _init_filters():
    """Initialise and register HDF5 filters with h5py

}

        def __hash__(self):
            return hash((self.filter_id, self.filter_options))

        def __len__(self):
            return len(self._kwargs)

        def __iter__(self):
            return iter(self._kwargs)

        def __getitem__(self, item):
            return self._kwargs[item]


class Blosc(_FilterRefClass):
    """h5py.Group.create_dataset's compression and compression_opts arguments for using blosc filter.

    :param str cname:
        `blosclz`, `lz4` (default), `lz4hc`, `zlib`, `zstd`
        Optional: `snappy`, depending on compilation (requires C++11).
    :param int clevel:
        Compression level from 0 no compression to 9 maximum compression.
        Default: 5.
    :param int shuffle: One of:
        - Blosc.NOSHUFFLE (0): No shuffle
        - Blosc.SHUFFLE (1): byte-wise shuffle (default)
        - Blosc.BITSHUFFLE (2): bit-wise shuffle
    """

    NOSHUFFLE = 0
    """Flag to disable data shuffle pre-compression filter"""

'lz4': 1,
        'lz4hc': 2,
        'snappy': 3,
        'zlib': 4,
        'zstd': 5,
    }

    def __init__(self, cname='lz4', clevel=5, shuffle=SHUFFLE):
        compression = self.__COMPRESSIONS[cname]
        clevel = int(clevel)
        assert 0 <= clevel <= 9
        assert shuffle in (self.NOSHUFFLE, self.SHUFFLE, self.BITSHUFFLE)
        self.filter_options = (0, 0, 0, 0, clevel, shuffle, compression)


class Bitshuffle(_FilterRefClass):
    """h5py.Group.create_dataset's compression and compression_opts arguments for using bitshuffle filter.

    :param int nelems:
        The number of elements per block.
        Default: 0 (for about 8kB per block).
    :param bool lz4:
        Whether to use LZ4_ID compression or not as part of the filter.
        Default: True
    """
    filter_id = BSHUF_ID

    def __init__(self, nelems=0, lz4=True):
        nelems = int(nelems)
        assert nelems % 8 == 0

        lz4_enabled = 2 if lz4 else 0

from collections import ChainMap
from functools import partial
from pathlib import Path
from typing import MutableMapping, NamedTuple, Tuple, Optional, Union, Callable

import h5py
import numpy as np

try:
    # hdf5plugin warns if a filter is already loaded. we temporarily surpress
    # that here, then reset the logger level to it's initial version.
    _logger = logging.getLogger('hdf5plugin')
    _initialLevel = _logger.getEffectiveLevel()
    _logger.setLevel(logging.ERROR)
    import hdf5plugin
    assert 'blosc' in hdf5plugin.FILTERS
except (ImportError, ModuleNotFoundError):  # pragma: no cover
    pass
finally:
    _logger.setLevel(_initialLevel)
from xxhash import xxh64_hexdigest

from .. import __version__
from ..constants import DIR_DATA_REMOTE, DIR_DATA_STAGE, DIR_DATA_STORE, DIR_DATA
from ..utils import find_next_prime, random_string, set_blosc_nthreads


set_blosc_nthreads()


# ----------------------------- Configuration ---------------------------------

import time
from collections import ChainMap
from functools import partial
from pathlib import Path
from typing import MutableMapping, NamedTuple, Tuple, Optional, Union, Callable

import h5py
import numpy as np
try:
    # hdf5plugin warns if a filter is already loaded. we temporarily surpress
    # that here, then reset the logger level to it's initial version.
    _logger = logging.getLogger('hdf5plugin')
    _initialLevel = _logger.getEffectiveLevel()
    _logger.setLevel(logging.ERROR)
    import hdf5plugin
    assert 'blosc' in hdf5plugin.FILTERS
except (ImportError, ModuleNotFoundError):  # pragma: no cover
    pass
finally:
    _logger.setLevel(_initialLevel)
from xxhash import xxh64_hexdigest

from .. import __version__
from ..constants import DIR_DATA_REMOTE, DIR_DATA_STAGE, DIR_DATA_STORE, DIR_DATA
from ..utils import find_next_prime, random_string, set_blosc_nthreads


set_blosc_nthreads()


# ----------------------------- Configuration ---------------------------------

assert np.issubdtype(batch_like.dtype, np.number), "invalid non-flattened array subtype"
        auto_chunker = False
        if chunk_size is None:
            auto_chunker = True
            chunk_size = (1, *batch_like.shape[1:])
        chunk_byte_size = np.multiply.reduce(chunk_size) * batch_like.dtype.itemsize
        assert auto_chunker or 10 * (2 ** 10) <= chunk_byte_size < 2 ** 20, \
            f"unrecommended chunk byte size ({chunk_byte_size}) should be in [10KiB,1MiB];" \
            " see http://docs.h5py.org/en/stable/high/dataset.html#chunked-storage"
        if compression == "chunk_lz4":
            dset = fd.create_dataset(
                name=name,
                shape=(max_len, *batch_like.shape[1:]),
                chunks=chunk_size,
                dtype=batch_like.dtype,
                **hdf5plugin.LZ4(nbytes=0)
            )
        else:
            assert compression not in no_compression_flags or len(compression_args) == 0
            dset = fd.create_dataset(
                name=name,
                shape=(max_len, *batch_like.shape[1:]),
                chunks=chunk_size,
                dtype=batch_like.dtype,
                compression=compression if compression not in no_compression_flags else None,
                **compression_args
            )
        dset.attrs["orig_shape"] = batch_like.shape[1:]  # removes batch dim
    else:
        assert thelper.utils.is_scalar(batch_like[0])
        if np.issubdtype(batch_like.dtype, np.number):
            assert compression in no_compression_flags, "cannot compress scalar elements"

#some standard encodings
includes.append('encodings.ascii')
includes.append('encodings.utf_8')
includes.append('encodings.latin_1')
import PyMca5
import hdf5plugin
import silx
import pkg_resources
SILX = True

special_modules = [os.path.dirname(PyMca5.__file__),
                   os.path.dirname(matplotlib.__file__),
                   os.path.dirname(ctypes.__file__),
                   os.path.dirname(fisx.__file__),
                   os.path.dirname(hdf5plugin.__file__),
                   os.path.dirname(silx.__file__),
                   os.path.dirname(pkg_resources.__file__)]

try:
    import tomogui
    special_modules.append(os.path.dirname(tomogui.__file__))
    import freeart
    special_modules.append(os.path.dirname(freeart.__file__))
except:
    pass


excludes = ["Tkinter", "tkinter",
            'tcl','_tkagg', 'Tkconstants',
            "scipy", "Numeric", "numarray", "PyQt5"]