How to use the toolz.pipe function in toolz

To help you get started, we’ve selected a few toolz examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Azure / DevOps-For-AI-Apps / tests / integration / classify.py View on Github external
def to_img(img_url):
    return toolz.pipe(img_url,
                      read_image_from,
                      to_rgb,
                      resize(new_size=(224,224)))
github Azure / DevOps-For-AI-Apps / tests / integration / classify.py View on Github external
def read_image_from(url):
    return toolz.pipe(url,
                      urllib.request.urlopen,
                      lambda x: x.read(),
                      BytesIO)
github SteemData / steemdata-mongo / src / methods.py View on Github external
def get_comment(identifier):
    with suppress(PostDoesNotExist):
        return pipe(
            Post(identifier).export(),
            strip_dot_from_keys,
            safe_json_metadata
        )
github blaze / blaze / blaze / data / csv.py View on Github external
def _iter(self, usecols=None, chunksize=None):
        chunksize = chunksize or self.chunksize
        dfs = self.pandas_read_csv(usecols=usecols,
                                   chunksize=chunksize,
                                   dtype='O',
                                   parse_dates=[])
        reorder = get(list(usecols)) if usecols and len(usecols) > 1 else identity
        return pipe(dfs, map(reorder),
                         map(partial(pd.DataFrame.fillna, value='')),
                         map(partial(into, list)),
                         concat)
github msalvaris / gpu_monitor / gpumon / file / gpu_logger.py View on Github external
def plot(self, gpu_property='sm', num_gpus=1, plot_width=600, plot_height=400, y_range=(0, 110)):
        df = pipe(self._log_file,
                  parse_log,
                  extract(gpu_property))
        return plot(df,
                    num_gpus=num_gpus,
                    plot_width=plot_width,
                    plot_height=plot_height,
                    y_range=y_range)
github msalvaris / gpu_monitor / gpumon / nvidia_xml.py View on Github external
def start(plot_handle):
        try:
            num_gen = count()
            while True:
                new_data = pipe(nvidia_run_xml(), extract, gpu_property_func)
                new_data['index'] = [next(num_gen)]
                data.stream(new_data, rollover=None, setter=None)
                push_notebook(handle=plot_handle)
        except KeyboardInterrupt:
            print('Exiting plotting')
github dask / dask / dask / array / core.py View on Github external
>>> top(f, 'z', 'i', 'x', 'i', numblocks={'x': (2,)}, b=10)  # doctest: +SKIP
    {('z', 0): (apply, f, [('x', 0)], {'b': 10}),
     ('z', 1): (apply, f, [('x', 1)], {'b': 10})}

    See Also
    --------
    atop
    """
    numblocks = kwargs.pop('numblocks')
    concatenate = kwargs.pop('concatenate', None)
    new_axes = kwargs.pop('new_axes', {})
    argpairs = list(partition(2, arrind_pairs))

    assert set(numblocks) == set(pluck(0, argpairs))

    all_indices = pipe(argpairs, pluck(1), concat, set)
    dummy_indices = all_indices - set(out_indices)

    # Dictionary mapping {i: 3, j: 4, ...} for i, j, ... the dimensions
    dims = broadcast_dimensions(argpairs, numblocks)
    for k in new_axes:
        dims[k] = 1

    # (0, 0), (0, 1), (0, 2), (1, 0), ...
    keytups = list(product(*[range(dims[i]) for i in out_indices]))
    # {i: 0, j: 0}, {i: 0, j: 1}, ...
    keydicts = [dict(zip(out_indices, tup)) for tup in keytups]

    # {j: [1, 2, 3], ...}  For j a dummy index of dimension 3
    dummies = dict((i, list(range(dims[i]))) for i in dummy_indices)

    # Create argument lists
github Clinical-Genomics / chanjo / chanjo / annotator / cli.py View on Github external
with Samfile(bam_path) as bam:
    # user defined sample id or randomly generated
    sample = (sample or get_sample_id(bam.header) or id_generator())

  # step 1: metadata header
  metadata = dict(
    sample_id=sample,
    group_id=group,
    cutoff=cutoff,
    coverage_source=path(bam_path).abspath(),
    extension=extendby
  )
  click.echo("#%s" % json.dumps(metadata))

  # step 2: annotate list of intervals with coverage and completeness
  bed_lines = pipe(
    annotate_bed_stream(
      bed_stream=in_stream,
      bam_path=bam_path,
      cutoff=cutoff,
      extension=extendby,
      contig_prefix=prefix,
      bp_threshold=threshold
    ),
    map(serialize_interval)    # stringify/bedify
  )

  # reduce/write the BED lines
  for bed_line in bed_lines:
    click.echo(bed_line)
github dask / distributed / distributed / pool.py View on Github external
def choose_worker(needed, who_has, has_what, available_cores):
    """ Select worker to run computation

    Currently selects the worker with the most data local
    """
    workers = {w for w, c in available_cores.items() if c}
    counts = pipe(needed, map(who_has.__getitem__), concat,
            filter(workers.__contains__), frequencies)
    if not counts:
        return random.choice(list(workers))
    else:
        biggest = max(counts.values())
        best = {k: v for k, v in counts.items() if v == biggest}
        return random.choice(list(best))