How to use the pandarallel._pandarallel._Series function in pandarallel

To help you get started, we’ve selected a few pandarallel examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github nalepae / pandarallel / pandarallel / _pandarallel.py View on Github external
def closure(series, func, *args, **kwargs):
            chunks = _chunk(series.size, nb_workers)
            object_id = plasma_client.put(series)

            with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
                futures = [
                            executor.submit(_Series.worker_apply,
                                            plasma_store_name, object_id,
                                            chunk, func, progress_bar,
                                            *args, **kwargs)
                            for chunk in chunks
                        ]

            result = _pd.concat([
                                plasma_client.get(future.result())
                                for future in futures
                            ], copy=False)

            return result
        return closure
github nalepae / pandarallel / pandarallel / _pandarallel.py View on Github external
def closure(data, arg, **kwargs):
            chunks = _chunk(data.size, nb_workers)
            object_id = plasma_client.put(data)

            with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
                futures = [
                            executor.submit(_Series.worker_map,
                                            plasma_store_name, object_id,
                                            _chunk, arg, progress_bar,
                                            **kwargs)
                            for _chunk in chunks
                        ]

            result = _pd.concat([
                                plasma_client.get(future.result())
                                for future in futures
                            ], copy=False)

            return result
        return closure
github nalepae / pandarallel / pandarallel / _pandarallel.py View on Github external
print("WARNING: Progress bar is an experimental feature. This \
can lead to a considerable performance loss.")
            tqdm_notebook().pandas()

        cls.__store_ctx = _plasma.start_plasma_store(int(shm_size_mb * 1e6))
        plasma_store_name, _ = cls.__store_ctx.__enter__()

        plasma_client = _plasma.connect(plasma_store_name)

        args = plasma_store_name, nb_workers, plasma_client

        _pd.DataFrame.parallel_apply = _DataFrame.apply(*args, progress_bar)
        _pd.DataFrame.parallel_applymap = _DataFrame.applymap(*args, progress_bar)

        _pd.Series.parallel_map = _Series.map(*args, progress_bar)
        _pd.Series.parallel_apply = _Series.apply(*args, progress_bar)

        _pd.core.window.Rolling.parallel_apply = _SeriesRolling.apply(*args, progress_bar)

        _pd.core.groupby.DataFrameGroupBy.parallel_apply = _DataFrameGroupBy.apply(*args)

        _pd.core.window.RollingGroupby.parallel_apply = _RollingGroupby.apply(*args)
github nalepae / pandarallel / pandarallel / _pandarallel.py View on Github external
if progress_bar:
            print("WARNING: Progress bar is an experimental feature. This \
can lead to a considerable performance loss.")
            tqdm_notebook().pandas()

        cls.__store_ctx = _plasma.start_plasma_store(int(shm_size_mb * 1e6))
        plasma_store_name, _ = cls.__store_ctx.__enter__()

        plasma_client = _plasma.connect(plasma_store_name)

        args = plasma_store_name, nb_workers, plasma_client

        _pd.DataFrame.parallel_apply = _DataFrame.apply(*args, progress_bar)
        _pd.DataFrame.parallel_applymap = _DataFrame.applymap(*args, progress_bar)

        _pd.Series.parallel_map = _Series.map(*args, progress_bar)
        _pd.Series.parallel_apply = _Series.apply(*args, progress_bar)

        _pd.core.window.Rolling.parallel_apply = _SeriesRolling.apply(*args, progress_bar)

        _pd.core.groupby.DataFrameGroupBy.parallel_apply = _DataFrameGroupBy.apply(*args)

        _pd.core.window.RollingGroupby.parallel_apply = _RollingGroupby.apply(*args)