How to use the cytoolz.concat function in cytoolz

To help you get started, we’ve selected a few cytoolz examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dask / dask / dask / array / core.py View on Github external
-------
    chunkss : dict
        Map like {index: chunks}.
    arrays : list
        List of rechunked arrays.

    See Also
    --------
    common_blockdim
    """
    if not args:
        return {}, []

    arginds = [(asarray(a) if ind is not None else a, ind)
               for a, ind in partition(2, args)]  # [x, ij, y, jk]
    args = list(concat(arginds))  # [(x, ij), (y, jk)]
    warn = kwargs.get('warn', True)

    arrays, inds = zip(*arginds)
    if all(ind == inds[0] for ind in inds) and all(a.chunks == arrays[0].chunks for a in arrays):
        return dict(zip(inds[0], arrays[0].chunks)), arrays

    nameinds = [(a.name if i is not None else a, i) for a, i in arginds]
    blockdim_dict = {a.name: a.chunks
                     for a, ind in arginds
                     if ind is not None}

    chunkss = broadcast_dimensions(nameinds, blockdim_dict,
                                   consolidate=common_blockdim)
    max_parts = max(arg.npartitions for arg, ind in arginds if ind is not None)
    nparts = np.prod(list(map(len, chunkss.values())))
github dask / dask / dask / array / core.py View on Github external
>>> argpairs = [('x', 'ij'), ('y', 'ij')]
    >>> numblocks = {'x': (2, 1), 'y': (1, 3)}
    >>> broadcast_dimensions(argpairs, numblocks)
    {'i': 2, 'j': 3}

    Works in other contexts too

    >>> argpairs = [('x', 'ij'), ('y', 'ij')]
    >>> d = {'x': ('Hello', 1), 'y': (1, (2, 3))}
    >>> broadcast_dimensions(argpairs, d)
    {'i': 'Hello', 'j': (2, 3)}
    """
    # List like [('i', 2), ('j', 1), ('i', 1), ('j', 2)]
    argpairs2 = [(a, ind) for a, ind in argpairs if ind is not None]
    L = concat([zip(inds, dims) for (x, inds), (x, dims)
                in join(first, argpairs2, first, numblocks.items())])

    g = groupby(0, L)
    g = dict((k, set([d for i, d in v])) for k, v in g.items())

    g2 = dict((k, v - set(sentinels) if len(v) > 1 else v) for k, v in g.items())

    if consolidate:
        return valmap(consolidate, g2)

    if g2 and not set(map(len, g2.values())) == set([1]):
        raise ValueError("Shapes do not align %s" % g)

    return valmap(first, g2)
github jcmgray / quimb / quimb / tensor / tensor_core.py View on Github external
``opt_einsum`` backend with the relevant library installed.
    contract_opts
        Passed to ``opt_einsum.contract_expression`` or
        ``opt_einsum.contract_path``.

    Returns
    -------
    scalar or Tensor
    """
    check_opt('get', get, _VALID_CONTRACT_GET)

    if backend is None:
        backend = _CONTRACT_BACKEND

    i_ix = tuple(t.inds for t in tensors)  # input indices per tensor
    total_ix = tuple(concat(i_ix))  # list of all input indices
    all_ix = tuple(unique(total_ix))

    if output_inds is None:
        # sort output indices  by input order for efficiency and consistency
        o_ix = tuple(_gen_output_inds(total_ix))
    else:
        o_ix = output_inds

    # possibly map indices into the range needed by opt- einsum
    eq = _maybe_map_indices_to_alphabet(all_ix, i_ix, o_ix)

    if get == 'symbol-map':
        return {oe.get_symbol(i): ix for i, ix in enumerate(all_ix)}

    if get == 'path-info':
        ops = (t.data for t in tensors)
github dask / dask / pbag / core.py View on Github external
>>> seq = iter([1, 2, 3])
    >>> empty, seq = isempty(seq)
    >>> empty
    False

    >>> list(seq)  # seq is preserved
    [1, 2, 3]

    >>> seq = iter([])
    >>> empty, seq = isempty(seq)
    >>> empty
    True
    """
    try:
        first = next(seq)
        return False, concat([[first], seq])
    except StopIteration:
        return True, False
github jcmgray / quimb / quimb / tensor / tensor_core.py View on Github external
``opt_einsum`` backend with the relevant library installed.
    contract_opts
        Passed to ``opt_einsum.contract_expression`` or
        ``opt_einsum.contract_path``.

    Returns
    -------
    scalar or Tensor
    """
    check_opt('get', get, _VALID_CONTRACT_GET)

    if backend is None:
        backend = _CONTRACT_BACKEND

    i_ix = tuple(t.inds for t in tensors)  # input indices per tensor
    total_ix = tuple(concat(i_ix))  # list of all input indices
    all_ix = tuple(unique(total_ix))

    if output_inds is None:
        # sort output indices  by input order for efficiency and consistency
        o_ix = tuple(_gen_output_inds(total_ix))
    else:
        o_ix = output_inds

    # possibly map indices into the range needed by opt- einsum
    eq = _maybe_map_indices_to_alphabet(all_ix, i_ix, o_ix)

    if get == 'symbol-map':
        return {oe.get_symbol(i): ix for i, ix in enumerate(all_ix)}

    if get == 'path-info':
        ops = (t.data for t in tensors)
github CoinAlpha / hummingbot / hummingbot / wallet / ethereum / watcher / contract_event_logs.py View on Github external
event_name: str,
                                        block_hashes: List[HexBytes]) -> List[AttributeDict]:
        event_abi: Dict[str, any] = self._event_abi_map.get(event_name, None)
        if event_abi is None:
            event_abi = find_matching_event_abi(self._contract_abi, event_name=event_name)
            self._event_abi_map[event_name] = event_abi

        _, event_filter_params = construct_event_filter_params(event_abi,
                                                               contract_address=self._address)
        tasks = []
        for block_hash in block_hashes:
            event_filter_params["blockHash"] = block_hash.hex()
            tasks.append(self._get_logs(event_filter_params))

        raw_logs = await asyncio.gather(*tasks, return_exceptions=True)
        logs: List[any] = list(cytoolz.concat(raw_logs))
        new_entries = []
        for log in logs:
            event_data: AttributeDict = get_event_data(event_abi, log)
            event_data_block_number: int = event_data["blockNumber"]
            event_data_tx_hash: HexBytes = event_data["transactionHash"]
            if event_data_tx_hash not in self._event_cache:
                if event_data_block_number not in self._block_events:
                    self._block_events[event_data_block_number] = [event_data_tx_hash]
                else:
                    self._block_events[event_data_block_number].append(event_data_tx_hash)
                self._event_cache.add(event_data_tx_hash)
                new_entries.append(event_data)
            else:
                self.logger().debug(
                    f"Duplicate event transaction hash found - '{event_data_tx_hash.hex()}'."
                )
github explosion / prodigy-recipes / nlp / textcat.py View on Github external
# Find 'seedy' examples
        examples_with_seeds = list(find_with_terms(stream, seeds,
                                   at_least=10, at_most=1000,
                                   give_up_after=10000))
        for eg in examples_with_seeds:
            eg.setdefault('meta', {})
            eg['meta']['via_seed'] = True
        print("Found {} examples with seeds".format(len(examples_with_seeds)))
        examples_with_seeds = [task for _, task in model(examples_with_seeds)]
    # Rank the stream. Note this is continuous, as model() is a generator.
    # As we call model.update(), the ranking of examples changes.
    stream = prefer_uncertain(model(stream))
    # Prepend 'seedy' examples, if present
    if seeds:
        log("RECIPE: Prepending examples with seeds to the stream")
        stream = cytoolz.concat((examples_with_seeds, stream))
    return {
        'view_id': 'classification',
        'dataset': dataset,
        'stream': stream,
        'exclude': exclude,
        'update': model.update,
        'config': {'lang': nlp.lang, 'labels': model.labels}
    }
github dask / dask / dask / array / core.py View on Github external
concatenate
    """
    n = len(seq)
    ndim = len(seq[0].shape)
    if axis < 0:
        axis = ndim + axis + 1
    if axis > ndim:
        raise ValueError("Axis must not be greater than number of dimensions"
                         "\nData has %d dimensions, but got axis=%d" %
                         (ndim, axis))
    if not all(x.shape == seq[0].shape for x in seq):
        raise ValueError("Stacked arrays must have the same shape. Got %s",
                         [x.shape for x in seq])

    ind = list(range(ndim))
    uc_args = list(concat((x, ind) for x in seq))
    _, seq = unify_chunks(*uc_args)

    dt = reduce(np.promote_types, [a.dtype for a in seq])
    seq = [x.astype(dt) for x in seq]

    assert len(set(a.chunks for a in seq)) == 1  # same chunks
    chunks = (seq[0].chunks[:axis] + ((1,) * n,) + seq[0].chunks[axis:])

    names = [a.name for a in seq]
    name = 'stack-' + tokenize(names, axis)
    keys = list(product([name], *[range(len(bd)) for bd in chunks]))

    inputs = [(names[key[axis + 1]], ) + key[1:axis + 1] + key[axis + 2:]
              for key in keys]
    values = [(getitem, inp, (slice(None, None, None),) * axis +
              (None, ) + (slice(None, None, None), ) * (ndim - axis))
github dask / dask / dask / delayed.py View on Github external
def call_function(func, func_token, args, kwargs, pure=None, nout=None):
    dask_key_name = kwargs.pop("dask_key_name", None)
    pure = kwargs.pop("pure", pure)

    if dask_key_name is None:
        name = "%s-%s" % (
            funcname(func),
            tokenize(func_token, *args, pure=pure, **kwargs),
        )
    else:
        name = dask_key_name

    args2, collections = unzip(map(unpack_collections, args), 2)
    collections = list(concat(collections))

    if kwargs:
        dask_kwargs, collections2 = unpack_collections(kwargs)
        collections.extend(collections2)
        task = (apply, func, list(args2), dask_kwargs)
    else:
        task = (func,) + args2

    graph = HighLevelGraph.from_collections(
        name, {name: task}, dependencies=collections
    )
    nout = nout if nout is not None else None
    return Delayed(name, graph, length=nout)