Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
-------
chunkss : dict
Map like {index: chunks}.
arrays : list
List of rechunked arrays.
See Also
--------
common_blockdim
"""
if not args:
return {}, []
arginds = [(asarray(a) if ind is not None else a, ind)
for a, ind in partition(2, args)] # [x, ij, y, jk]
args = list(concat(arginds)) # [(x, ij), (y, jk)]
warn = kwargs.get('warn', True)
arrays, inds = zip(*arginds)
if all(ind == inds[0] for ind in inds) and all(a.chunks == arrays[0].chunks for a in arrays):
return dict(zip(inds[0], arrays[0].chunks)), arrays
nameinds = [(a.name if i is not None else a, i) for a, i in arginds]
blockdim_dict = {a.name: a.chunks
for a, ind in arginds
if ind is not None}
chunkss = broadcast_dimensions(nameinds, blockdim_dict,
consolidate=common_blockdim)
max_parts = max(arg.npartitions for arg, ind in arginds if ind is not None)
nparts = np.prod(list(map(len, chunkss.values())))
>>> argpairs = [('x', 'ij'), ('y', 'ij')]
>>> numblocks = {'x': (2, 1), 'y': (1, 3)}
>>> broadcast_dimensions(argpairs, numblocks)
{'i': 2, 'j': 3}
Works in other contexts too
>>> argpairs = [('x', 'ij'), ('y', 'ij')]
>>> d = {'x': ('Hello', 1), 'y': (1, (2, 3))}
>>> broadcast_dimensions(argpairs, d)
{'i': 'Hello', 'j': (2, 3)}
"""
# List like [('i', 2), ('j', 1), ('i', 1), ('j', 2)]
argpairs2 = [(a, ind) for a, ind in argpairs if ind is not None]
L = concat([zip(inds, dims) for (x, inds), (x, dims)
in join(first, argpairs2, first, numblocks.items())])
g = groupby(0, L)
g = dict((k, set([d for i, d in v])) for k, v in g.items())
g2 = dict((k, v - set(sentinels) if len(v) > 1 else v) for k, v in g.items())
if consolidate:
return valmap(consolidate, g2)
if g2 and not set(map(len, g2.values())) == set([1]):
raise ValueError("Shapes do not align %s" % g)
return valmap(first, g2)
``opt_einsum`` backend with the relevant library installed.
contract_opts
Passed to ``opt_einsum.contract_expression`` or
``opt_einsum.contract_path``.
Returns
-------
scalar or Tensor
"""
check_opt('get', get, _VALID_CONTRACT_GET)
if backend is None:
backend = _CONTRACT_BACKEND
i_ix = tuple(t.inds for t in tensors) # input indices per tensor
total_ix = tuple(concat(i_ix)) # list of all input indices
all_ix = tuple(unique(total_ix))
if output_inds is None:
# sort output indices by input order for efficiency and consistency
o_ix = tuple(_gen_output_inds(total_ix))
else:
o_ix = output_inds
# possibly map indices into the range needed by opt- einsum
eq = _maybe_map_indices_to_alphabet(all_ix, i_ix, o_ix)
if get == 'symbol-map':
return {oe.get_symbol(i): ix for i, ix in enumerate(all_ix)}
if get == 'path-info':
ops = (t.data for t in tensors)
>>> seq = iter([1, 2, 3])
>>> empty, seq = isempty(seq)
>>> empty
False
>>> list(seq) # seq is preserved
[1, 2, 3]
>>> seq = iter([])
>>> empty, seq = isempty(seq)
>>> empty
True
"""
try:
first = next(seq)
return False, concat([[first], seq])
except StopIteration:
return True, False
``opt_einsum`` backend with the relevant library installed.
contract_opts
Passed to ``opt_einsum.contract_expression`` or
``opt_einsum.contract_path``.
Returns
-------
scalar or Tensor
"""
check_opt('get', get, _VALID_CONTRACT_GET)
if backend is None:
backend = _CONTRACT_BACKEND
i_ix = tuple(t.inds for t in tensors) # input indices per tensor
total_ix = tuple(concat(i_ix)) # list of all input indices
all_ix = tuple(unique(total_ix))
if output_inds is None:
# sort output indices by input order for efficiency and consistency
o_ix = tuple(_gen_output_inds(total_ix))
else:
o_ix = output_inds
# possibly map indices into the range needed by opt- einsum
eq = _maybe_map_indices_to_alphabet(all_ix, i_ix, o_ix)
if get == 'symbol-map':
return {oe.get_symbol(i): ix for i, ix in enumerate(all_ix)}
if get == 'path-info':
ops = (t.data for t in tensors)
event_name: str,
block_hashes: List[HexBytes]) -> List[AttributeDict]:
event_abi: Dict[str, any] = self._event_abi_map.get(event_name, None)
if event_abi is None:
event_abi = find_matching_event_abi(self._contract_abi, event_name=event_name)
self._event_abi_map[event_name] = event_abi
_, event_filter_params = construct_event_filter_params(event_abi,
contract_address=self._address)
tasks = []
for block_hash in block_hashes:
event_filter_params["blockHash"] = block_hash.hex()
tasks.append(self._get_logs(event_filter_params))
raw_logs = await asyncio.gather(*tasks, return_exceptions=True)
logs: List[any] = list(cytoolz.concat(raw_logs))
new_entries = []
for log in logs:
event_data: AttributeDict = get_event_data(event_abi, log)
event_data_block_number: int = event_data["blockNumber"]
event_data_tx_hash: HexBytes = event_data["transactionHash"]
if event_data_tx_hash not in self._event_cache:
if event_data_block_number not in self._block_events:
self._block_events[event_data_block_number] = [event_data_tx_hash]
else:
self._block_events[event_data_block_number].append(event_data_tx_hash)
self._event_cache.add(event_data_tx_hash)
new_entries.append(event_data)
else:
self.logger().debug(
f"Duplicate event transaction hash found - '{event_data_tx_hash.hex()}'."
)
# Find 'seedy' examples
examples_with_seeds = list(find_with_terms(stream, seeds,
at_least=10, at_most=1000,
give_up_after=10000))
for eg in examples_with_seeds:
eg.setdefault('meta', {})
eg['meta']['via_seed'] = True
print("Found {} examples with seeds".format(len(examples_with_seeds)))
examples_with_seeds = [task for _, task in model(examples_with_seeds)]
# Rank the stream. Note this is continuous, as model() is a generator.
# As we call model.update(), the ranking of examples changes.
stream = prefer_uncertain(model(stream))
# Prepend 'seedy' examples, if present
if seeds:
log("RECIPE: Prepending examples with seeds to the stream")
stream = cytoolz.concat((examples_with_seeds, stream))
return {
'view_id': 'classification',
'dataset': dataset,
'stream': stream,
'exclude': exclude,
'update': model.update,
'config': {'lang': nlp.lang, 'labels': model.labels}
}
concatenate
"""
n = len(seq)
ndim = len(seq[0].shape)
if axis < 0:
axis = ndim + axis + 1
if axis > ndim:
raise ValueError("Axis must not be greater than number of dimensions"
"\nData has %d dimensions, but got axis=%d" %
(ndim, axis))
if not all(x.shape == seq[0].shape for x in seq):
raise ValueError("Stacked arrays must have the same shape. Got %s",
[x.shape for x in seq])
ind = list(range(ndim))
uc_args = list(concat((x, ind) for x in seq))
_, seq = unify_chunks(*uc_args)
dt = reduce(np.promote_types, [a.dtype for a in seq])
seq = [x.astype(dt) for x in seq]
assert len(set(a.chunks for a in seq)) == 1 # same chunks
chunks = (seq[0].chunks[:axis] + ((1,) * n,) + seq[0].chunks[axis:])
names = [a.name for a in seq]
name = 'stack-' + tokenize(names, axis)
keys = list(product([name], *[range(len(bd)) for bd in chunks]))
inputs = [(names[key[axis + 1]], ) + key[1:axis + 1] + key[axis + 2:]
for key in keys]
values = [(getitem, inp, (slice(None, None, None),) * axis +
(None, ) + (slice(None, None, None), ) * (ndim - axis))
def call_function(func, func_token, args, kwargs, pure=None, nout=None):
dask_key_name = kwargs.pop("dask_key_name", None)
pure = kwargs.pop("pure", pure)
if dask_key_name is None:
name = "%s-%s" % (
funcname(func),
tokenize(func_token, *args, pure=pure, **kwargs),
)
else:
name = dask_key_name
args2, collections = unzip(map(unpack_collections, args), 2)
collections = list(concat(collections))
if kwargs:
dask_kwargs, collections2 = unpack_collections(kwargs)
collections.extend(collections2)
task = (apply, func, list(args2), dask_kwargs)
else:
task = (func,) + args2
graph = HighLevelGraph.from_collections(
name, {name: task}, dependencies=collections
)
nout = nout if nout is not None else None
return Delayed(name, graph, length=nout)