Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
>>> blockdims_from_blockshape((10, 0), (4, 0))
((4, 4, 2), (0,))
"""
if chunks is None:
raise TypeError("Must supply chunks= keyword argument")
if shape is None:
raise TypeError("Must supply shape= keyword argument")
if np.isnan(sum(shape)) or np.isnan(sum(chunks)):
raise ValueError("Array chunk sizes are unknown. shape: %s, chunks: %s"
% (shape, chunks))
if not all(map(is_integer, chunks)):
raise ValueError("chunks can only contain integers.")
if not all(map(is_integer, shape)):
raise ValueError("shape can only contain integers.")
shape = tuple(map(int, shape))
chunks = tuple(map(int, chunks))
return tuple(((bd,) * (d // bd) + ((d % bd,) if d % bd else ())
if d else (0,))
for d, bd in zip(shape, chunks))
dissoc = toolz.curry(toolz.dissoc)
do = toolz.curry(toolz.do)
drop = toolz.curry(toolz.drop)
excepts = toolz.curry(toolz.excepts)
filter = toolz.curry(toolz.filter)
get = toolz.curry(toolz.get)
get_in = toolz.curry(toolz.get_in)
groupby = toolz.curry(toolz.groupby)
interpose = toolz.curry(toolz.interpose)
itemfilter = toolz.curry(toolz.itemfilter)
itemmap = toolz.curry(toolz.itemmap)
iterate = toolz.curry(toolz.iterate)
join = toolz.curry(toolz.join)
keyfilter = toolz.curry(toolz.keyfilter)
keymap = toolz.curry(toolz.keymap)
map = toolz.curry(toolz.map)
mapcat = toolz.curry(toolz.mapcat)
nth = toolz.curry(toolz.nth)
partial = toolz.curry(toolz.partial)
partition = toolz.curry(toolz.partition)
partition_all = toolz.curry(toolz.partition_all)
partitionby = toolz.curry(toolz.partitionby)
peekn = toolz.curry(toolz.peekn)
pluck = toolz.curry(toolz.pluck)
random_sample = toolz.curry(toolz.random_sample)
reduce = toolz.curry(toolz.reduce)
reduceby = toolz.curry(toolz.reduceby)
remove = toolz.curry(toolz.remove)
sliding_window = toolz.curry(toolz.sliding_window)
sorted = toolz.curry(toolz.sorted)
tail = toolz.curry(toolz.tail)
take = toolz.curry(toolz.take)
>>> list(deepmap(inc, [1, 2], n=1))
[2, 3]
>>> list(deepmap(inc, [(1, 2), (3, 4)], n=2))
[(2, 3), (4, 5)]
Works on variadic args too
>>> add = lambda x, y: x + y
>>> list(deepmap(add, [1, 2], [10, 20], n=1))
[11, 22]
"""
n = kwargs.pop('n', 1)
if n == 0:
return func(*data)
if n == 1:
return map(func, *data)
else:
return map(compose(tuple, partial(deepmap, func, n=n-1)), *data)
ndim = len(seq[0].shape)
if axis < 0:
axis = ndim + axis
if axis >= ndim:
msg = ("Axis must be less than than number of dimensions"
"\nData has %d dimensions, but got axis=%d")
raise ValueError(msg % (ndim, axis))
if n == 1:
return seq[0]
if (not allow_unknown_chunksizes and
not all(i == axis or all(x.shape[i] == seq[0].shape[i] for x in seq)
for i in range(ndim))):
if any(map(np.isnan, seq[0].shape)):
raise ValueError("Tried to concatenate arrays with unknown"
" shape %s. To force concatenation pass"
" allow_unknown_chunksizes=True."
% str(seq[0].shape))
raise ValueError("Shapes do not align: %s", [x.shape for x in seq])
inds = [list(range(ndim)) for i in range(n)]
for i, ind in enumerate(inds):
ind[axis] = -(i + 1)
uc_args = list(concat(zip(seq, inds)))
_, seq = unify_chunks(*uc_args, warn=False)
bds = [a.chunks for a in seq]
chunks = (seq[0].chunks[:axis] + (sum([bd[axis] for bd in bds], ()), ) +
numblocks = {a.name: a.numblocks for a, ind in arginds if ind is not None}
argindsstr = list(concat([(a if ind is None else a.name, ind) for a, ind in arginds]))
# Finish up the name
if not out:
out = '%s-%s' % (token or funcname(func).strip('_'),
tokenize(func, out_ind, argindsstr, dtype, **kwargs))
dsk = top(func, out, out_ind, *argindsstr, numblocks=numblocks, **kwargs)
dsks = [a.dask for a, ind in arginds if ind is not None]
chunks = [chunkss[i] for i in out_ind]
if adjust_chunks:
for i, ind in enumerate(out_ind):
if ind in adjust_chunks:
if callable(adjust_chunks[ind]):
chunks[i] = tuple(map(adjust_chunks[ind], chunks[i]))
elif isinstance(adjust_chunks[ind], int):
chunks[i] = tuple(adjust_chunks[ind] for _ in chunks[i])
elif isinstance(adjust_chunks[ind], (tuple, list)):
chunks[i] = tuple(adjust_chunks[ind])
else:
raise NotImplementedError(
"adjust_chunks values must be callable, int, or tuple")
chunks = tuple(chunks)
return Array(sharedict.merge((out, dsk), *dsks), out, chunks, dtype=dtype)
def run(n, x, *goals):
""" Run a logic program. Obtain n solutions to satisfy goals.
n - number of desired solutions. See ``take``
0 for all
None for a lazy sequence
x - Output variable
goals - a sequence of goals. All must be true
>>> from logpy import run, var, eq
>>> x = var()
>>> run(1, x, eq(x, 1))
(1,)
"""
results = map(partial(reify, x), goaleval(lall(*goals))({}))
return take(n, unique(results, key=multihash))
def numblocks(self):
return tuple(map(len, self.chunks))
{'i': 'Hello', 'j': (2, 3)}
"""
# List like [('i', 2), ('j', 1), ('i', 1), ('j', 2)]
argpairs2 = [(a, ind) for a, ind in argpairs if ind is not None]
L = concat([zip(inds, dims) for (x, inds), (x, dims)
in join(first, argpairs2, first, numblocks.items())])
g = groupby(0, L)
g = dict((k, set([d for i, d in v])) for k, v in g.items())
g2 = dict((k, v - set(sentinels) if len(v) > 1 else v) for k, v in g.items())
if consolidate:
return valmap(consolidate, g2)
if g2 and not set(map(len, g2.values())) == set([1]):
raise ValueError("Shapes do not align %s" % g)
return valmap(first, g2)
def _set_op(operator, *sets, labels=None):
new_ids = t.reduce(operator, t.map(lambda s: s.artifact_ids, sets))
return ArtifactSet(new_ids, labels)
>>> common_blockdim([(1, 2), (2, 1)])
(1, 1, 1)
>>> common_blockdim([(2, 2), (3, 1)]) # doctest: +SKIP
Traceback (most recent call last):
...
ValueError: Chunks do not align
"""
if not any(blockdims):
return ()
non_trivial_dims = set([d for d in blockdims if len(d) > 1])
if len(non_trivial_dims) == 1:
return first(non_trivial_dims)
if len(non_trivial_dims) == 0:
return max(blockdims, key=first)
if np.isnan(sum(map(sum, blockdims))):
raise ValueError("Arrays chunk sizes are unknown: %s", blockdims)
if len(set(map(sum, non_trivial_dims))) > 1:
raise ValueError("Chunks do not add up to same value", blockdims)
# We have multiple non-trivial chunks on this axis
# e.g. (5, 2) and (4, 3)
# We create a single chunk tuple with the same total length
# that evenly divides both, e.g. (4, 1, 2)
# To accomplish this we walk down all chunk tuples together, finding the
# smallest element, adding it to the output, and subtracting it from all
# other elements and remove the element itself. We stop once we have
# burned through all of the chunk tuples.
# For efficiency's sake we reverse the lists so that we can pop off the end