Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
dtype,
metadata,
chunk_info,
time_range,
chunk_construction_kwargs) -> strax.Chunk:
if chunk_info['n'] == 0:
# No data, no need to load
data = np.empty(0, dtype=dtype)
else:
data = self._read_chunk(backend_key,
chunk_info=chunk_info,
dtype=dtype,
compressor=metadata['compressor'])
result = strax.Chunk(
start=chunk_info['start'],
end=chunk_info['end'],
run_id=chunk_info['run_id'],
data=data,
**chunk_construction_kwargs)
if time_range:
if result.start < time_range[0]:
_, result = result.split(t=time_range[0],
allow_early_split=True)
if result.end > time_range[1]:
try:
result, _ = result.split(t=time_range[1],
allow_early_split=False)
except strax.CannotSplit:
pass
data=self.data,
t=t,
allow_early_split=allow_early_split)
common_kwargs = dict(
run_id=self.run_id,
dtype=self.dtype,
data_type=self.data_type,
data_kind=self.data_kind)
c1 = strax.Chunk(
start=self.start,
end=max(self.start, t),
data=data1,
**common_kwargs)
c2 = strax.Chunk(
start=max(self.start, t),
end=max(t, self.end),
data=data2,
**common_kwargs)
return c1, c2
from functools import partial
import logging
import typing as ty
import os
import sys
from concurrent.futures import ProcessPoolExecutor
import numpy as np
import strax
export, __all__ = strax.exporter()
try:
import npshmex
SHMExecutor = npshmex.ProcessPoolExecutor
npshmex.register_array_wrapper(strax.Chunk, 'data')
except ImportError:
# This is allowed to fail, it only crashes if allow_shm = True
SHMExecutor = None
@export
class ProcessorComponents(ty.NamedTuple):
"""Specification to assemble a processor"""
plugins: ty.Dict[str, strax.Plugin]
loaders: ty.Dict[str, callable]
savers: ty.Dict[str, ty.List[strax.Saver]]
targets: ty.Tuple[str]
class MailboxDict(dict):
def __init__(self, *args, lazy=False, **kwargs):
def _fix_output(self, result, start, end, _dtype=None):
if self.multi_output and _dtype is None:
if not isinstance(result, dict):
raise ValueError(
f"{self.__class__.__name__} is multi-output and should "
"provide a dict output {dtypename: result}")
return {d: self._fix_output(result[d], start, end, _dtype=d)
for d in self.provides}
if _dtype is None:
assert not self.multi_output
_dtype = self.provides[0]
if not isinstance(result, strax.Chunk):
if start is None:
assert len(self.depends_on) == 0
raise ValueError(
"Plugins without dependencies must return full strax "
f"Chunks, but {self.__class__.__name__} produced a "
f"{type(result)}!")
result = strax.dict_to_rec(result, dtype=self.dtype_for(_dtype))
self._check_dtype(result, _dtype)
result = self.chunk(
start=start,
end=end,
data_type=_dtype,
data=result)
return result
def _fetch_another(self):
# Remember to catch StopIteration if you're using this guy!
# TODO: custom exception?
try:
x = next(self.source)
except StopIteration:
self._source_exhausted = True
raise StopIteration
assert isinstance(x, strax.Chunk), \
f"Got {type(x)} instead of a strax Chunk!"
self.last_time_in_buffer = x.end
self.buffer.append(x)
self.buffer_items += len(x)
for s in savers:
s.save(chunk=results[d], chunk_i=chunk_i)
# Remove results we do not need to send
for d in list(results.keys()):
if d not in self.provides:
del results[d]
if self.multi_output:
for k in self.provides:
assert k in results
assert isinstance(results[k], strax.Chunk)
r0 = results[k]
else:
results = r0 = results[self.provides[0]]
assert isinstance(r0, strax.Chunk)
return self._fix_output(results, start=r0.start, end=r0.end)