How to use the strax.Chunk function in strax

To help you get started, we’ve selected a few strax examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github AxFoundation / strax / strax / storage / common.py View on Github external
dtype,
                               metadata,
                               chunk_info,
                               time_range,
                               chunk_construction_kwargs) -> strax.Chunk:

        if chunk_info['n'] == 0:
            # No data, no need to load
            data = np.empty(0, dtype=dtype)
        else:
            data = self._read_chunk(backend_key,
                                    chunk_info=chunk_info,
                                    dtype=dtype,
                                    compressor=metadata['compressor'])

        result = strax.Chunk(
            start=chunk_info['start'],
            end=chunk_info['end'],
            run_id=chunk_info['run_id'],
            data=data,
            **chunk_construction_kwargs)

        if time_range:
            if result.start < time_range[0]:
                _, result = result.split(t=time_range[0],
                                         allow_early_split=True)
            if result.end > time_range[1]:
                try:
                    result, _ = result.split(t=time_range[1],
                                             allow_early_split=False)
                except strax.CannotSplit:
                    pass
github AxFoundation / strax / strax / chunk.py View on Github external
data=self.data,
                t=t,
                allow_early_split=allow_early_split)

        common_kwargs = dict(
            run_id=self.run_id,
            dtype=self.dtype,
            data_type=self.data_type,
            data_kind=self.data_kind)

        c1 = strax.Chunk(
            start=self.start,
            end=max(self.start, t),
            data=data1,
            **common_kwargs)
        c2 = strax.Chunk(
            start=max(self.start, t),
            end=max(t, self.end),
            data=data2,
            **common_kwargs)
        return c1, c2
github AxFoundation / strax / strax / processor.py View on Github external
from functools import partial
import logging
import typing as ty
import os
import sys
from concurrent.futures import ProcessPoolExecutor

import numpy as np

import strax
export, __all__ = strax.exporter()

try:
    import npshmex
    SHMExecutor = npshmex.ProcessPoolExecutor
    npshmex.register_array_wrapper(strax.Chunk, 'data')
except ImportError:
    # This is allowed to fail, it only crashes if allow_shm = True
    SHMExecutor = None


@export
class ProcessorComponents(ty.NamedTuple):
    """Specification to assemble a processor"""
    plugins: ty.Dict[str, strax.Plugin]
    loaders: ty.Dict[str, callable]
    savers:  ty.Dict[str, ty.List[strax.Saver]]
    targets: ty.Tuple[str]


class MailboxDict(dict):
    def __init__(self, *args, lazy=False, **kwargs):
github AxFoundation / strax / strax / plugin.py View on Github external
def _fix_output(self, result, start, end, _dtype=None):
        if self.multi_output and _dtype is None:
            if not isinstance(result, dict):
                raise ValueError(
                    f"{self.__class__.__name__} is multi-output and should "
                    "provide a dict output {dtypename: result}")
            return {d: self._fix_output(result[d], start, end, _dtype=d)
                    for d in self.provides}

        if _dtype is None:
            assert not self.multi_output
            _dtype = self.provides[0]

        if not isinstance(result, strax.Chunk):
            if start is None:
                assert len(self.depends_on) == 0
                raise ValueError(
                    "Plugins without dependencies must return full strax "
                    f"Chunks, but {self.__class__.__name__} produced a "
                    f"{type(result)}!")

            result = strax.dict_to_rec(result, dtype=self.dtype_for(_dtype))
            self._check_dtype(result, _dtype)
            result = self.chunk(
                start=start,
                end=end,
                data_type=_dtype,
                data=result)
        return result
github AxFoundation / strax / strax / chunk_arrays.py View on Github external
def _fetch_another(self):
        # Remember to catch StopIteration if you're using this guy!
        # TODO: custom exception?
        try:
            x = next(self.source)
        except StopIteration:
            self._source_exhausted = True
            raise StopIteration
        assert isinstance(x, strax.Chunk), \
            f"Got {type(x)} instead of a strax Chunk!"

        self.last_time_in_buffer = x.end

        self.buffer.append(x)
        self.buffer_items += len(x)
github AxFoundation / strax / strax / plugin.py View on Github external
for s in savers:
                s.save(chunk=results[d], chunk_i=chunk_i)

        # Remove results we do not need to send
        for d in list(results.keys()):
            if d not in self.provides:
                del results[d]

        if self.multi_output:
            for k in self.provides:
                assert k in results
                assert isinstance(results[k], strax.Chunk)
                r0 = results[k]
        else:
            results = r0 = results[self.provides[0]]
            assert isinstance(r0, strax.Chunk)

        return self._fix_output(results, start=r0.start, end=r0.end)