How to use the toolz.merge function in toolz

To help you get started, we’ve selected a few toolz examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github enigmampc / catalyst / tests / pipeline / test_engine.py View on Github external
decay_rate=decay_rate,
            )
            for decay_rate in decay_rates
        }

        ewmstds = {
            ewmstd_name(decay_rate): EWMSTD(
                inputs=(USEquityPricing.close,),
                window_length=window_length,
                decay_rate=decay_rate,
            )
            for decay_rate in decay_rates
        }

        all_results = self.engine.run_pipeline(
            Pipeline(columns=merge(ewmas, ewmstds)),
            self.dates[window_length],
            self.dates[-1],
        )

        for decay_rate in decay_rates:
            ewma_result = all_results[ewma_name(decay_rate)].unstack()
            ewma_expected = self.expected_ewma(window_length, decay_rate)
            assert_frame_equal(ewma_result, ewma_expected)

            ewmstd_result = all_results[ewmstd_name(decay_rate)].unstack()
            ewmstd_expected = self.expected_ewmstd(window_length, decay_rate)
            assert_frame_equal(ewmstd_result, ewmstd_expected)
github bmabey / provenance / provenance / utils.py View on Github external
def _args_dict(args, kargs):
            unnamed_args = dict(zip(spec.args, args[0:num_named_args]))
            varargs = args[num_named_args:]
            kargs = t.merge(kargs, unnamed_args)
            for k, d in default_dicts.items():
                kargs[k] = t.merge(d, kargs.get(k) or {})
            return varargs, kargs
github dask / dask / dask / base.py View on Github external
def _extract_graph_and_keys(vals):
    """Given a list of dask vals, return a single graph and a list of keys such
    that ``get(dsk, keys)`` is equivalent to ``[v.compute() for v in vals]``."""
    from .highlevelgraph import HighLevelGraph

    graphs = [v.__dask_graph__() for v in vals]
    keys = [v.__dask_keys__() for v in vals]

    if any(isinstance(graph, HighLevelGraph) for graph in graphs):
        graph = HighLevelGraph.merge(*graphs)
    else:
        graph = merge(*map(ensure_dict, graphs))

    return graph, keys
github zylo117 / tensorflow-gpu-macosx / tensorflow / contrib / learn / python / learn / learn_io / dask_io.py View on Github external
def _construct_dask_df_with_divisions(df):
  """Construct the new task graph and make a new dask.dataframe around it."""
  divisions = _get_divisions(df)
  # pylint: disable=protected-access
  name = 'csv-index' + df._name
  dsk = {(name, i): (_add_to_index, (df._name, i), divisions[i])
         for i in range(df.npartitions)}
  # pylint: enable=protected-access
  from toolz import merge  # pylint: disable=g-import-not-at-top
  if isinstance(df, dd.DataFrame):
    return dd.DataFrame(merge(dsk, df.dask), name, df.columns, divisions)
  elif isinstance(df, dd.Series):
    return dd.Series(merge(dsk, df.dask), name, df.name, divisions)
github dask / dask / dask / dataframe / shuffle.py View on Github external
)
                    for j in range(k)
                ],
            )
            for inp in inputs
        }
        groups.append(group)
        splits.append(split)
        joins.append(join)

    end = {
        ("shuffle-" + token, i): ("shuffle-join-" + token, stages, inp)
        for i, inp in enumerate(inputs)
    }

    dsk = toolz.merge(start, end, *(groups + splits + joins))
    graph = HighLevelGraph.from_collections("shuffle-" + token, dsk, dependencies=[df])
    df2 = DataFrame(graph, "shuffle-" + token, df, df.divisions)

    if npartitions is not None and npartitions != df.npartitions:
        parts = [i % df.npartitions for i in range(npartitions)]
        token = tokenize(df2, npartitions)

        dsk = {
            ("repartition-group-" + token, i): (shuffle_group_2, k, column)
            for i, k in enumerate(df2.__dask_keys__())
        }
        for p in range(npartitions):
            dsk[("repartition-get-" + token, p)] = (
                shuffle_group_get,
                ("repartition-group-" + token, parts[p]),
                p,
github enigmampc / catalyst / catalyst / finance / commission.py View on Github external
self._cost_per_contract = DummyMapping(float(cost))
        else:
            # Cost per contract is a dictionary. If the user's dictionary does
            # not provide a commission cost for a certain contract, fall back
            # on the pre-defined cost values per root symbol.
            self._cost_per_contract = defaultdict(
                lambda: DEFAULT_PER_CONTRACT_COST, **cost
            )

        if isinstance(exchange_fee, (int, float)):
            self._exchange_fee = DummyMapping(float(exchange_fee))
        else:
            # Exchange fee is a dictionary. If the user's dictionary does not
            # provide an exchange fee for a certain contract, fall back on the
            # pre-defined exchange fees per root symbol.
            self._exchange_fee = merge(
                FUTURE_EXCHANGE_FEES_BY_SYMBOL, exchange_fee,
            )

        self.min_trade_cost = min_trade_cost or 0
github dask / dask / dask / array / linalg.py View on Github external
# dsk.dependencies[name_r_stacked] = {data.name}
        r_stacked_meta = meta_from_array(
            data, len((sum(vchunks_rstacked), n)), dtype=rr.dtype
        )
        r_stacked = Array(
            graph,
            name_r_stacked,
            shape=(sum(vchunks_rstacked), n),
            chunks=(vchunks_rstacked, n),
            meta=r_stacked_meta,
        )

        # recurse
        q_inner, r_inner = tsqr(r_stacked, _max_vchunk_size=cr_max)
        layers = toolz.merge(q_inner.dask.layers, r_inner.dask.layers)
        dependencies = toolz.merge(q_inner.dask.dependencies, r_inner.dask.dependencies)

        # Q_inner: "unstack"
        name_q_st2 = "getitem" + token + "-q2"
        dsk_q_st2 = dict(
            (
                (name_q_st2, j, 0),
                (
                    operator.getitem,
                    (q_inner.name, i, 0),
                    ((slice(e[0], e[1])), (slice(0, n))),
                ),
            )
            for i, sub_block_info in enumerate(all_blocks)
            for j, e in zip(
                [x[0] for x in sub_block_info],
                _cumsum_blocks([x[1] for x in sub_block_info]),
github jrderuiter / geneviz / src / geneviz / tracks / annotation / gene.py View on Github external
def _data_to_transcript(self, grp):
        first = grp.iloc[0]

        obj_kws = dict(self._obj_kws)
        obj_kws['plot_kws'] = toolz.merge({'facecolor': first.color},
                                          obj_kws.get('plot_kws', {}))

        return Transcript(grp, name=first.transcript_id, **obj_kws)
github blaze / blaze / blaze / api / into.py View on Github external
def typehint(x, typedict):
    """Replace the dtypes in `x` keyed by `typedict` with the dtypes in
    `typedict`.
    """
    dtype = x.dtype
    lhs = dict(zip(dtype.fields.keys(), map(first, dtype.fields.values())))
    dtype_list = list(merge(lhs, typedict).items())
    return x.astype(np.dtype(sort_dtype_items(dtype_list, dtype.names)))
github blaze / odo / odo / backends / sql.py View on Github external
def append_table_to_csv(csv, selectable, dshape=None, bind=None, **kwargs):
    kwargs = keyfilter(keywords(CopyToCSV).__contains__,
                       merge(csv.dialect, kwargs))
    stmt = CopyToCSV(
        selectable,
        os.path.abspath(csv.path) if csv.path is not None else None,
        bind=bind,
        **kwargs
    )

    bind = getbind(selectable, bind)
    if bind.dialect.name == 'postgresql':
        with csv.open('ab+') as f:
            with bind.begin() as conn:
                conn.connection.cursor().copy_expert(literal_compile(stmt), f)
    else:
        with bind.begin() as conn:
            conn.execute(stmt)
    return csv