How to use the awswrangler.pandas.concat function in awswrangler

To help you get started, we’ve selected a few awswrangler examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github awslabs / aws-data-wrangler / awswrangler / pandas.py View on Github external
filters,
                        1  # procs_cpu_bound
                    ),
                )
                proc.daemon = False
                proc.start()
                procs.append(proc)
                receive_pipes.append(receive_pipe)
            logger.debug(f"len(procs): {len(bounders)}")
            for i in range(len(procs)):
                logger.debug(f"Waiting pipe number: {i}")
                df_received = receive_pipes[i].recv()
                if df is None:
                    df = df_received
                else:
                    df = pd.concat(objs=[df, df_received], ignore_index=True)
                logger.debug(f"Waiting proc number: {i}")
                procs[i].join()
                logger.debug(f"Closing proc number: {i}")
                receive_pipes[i].close()
        return df
github awslabs / aws-data-wrangler / awswrangler / pandas.py View on Github external
columns=columns,
                filters=filters,
                procs_cpu_bound=procs_cpu_bound)
        else:
            df = Pandas._read_parquet_path(session_primitives=session_primitives,
                                           path=path[0],
                                           columns=columns,
                                           filters=filters,
                                           procs_cpu_bound=procs_cpu_bound)
            for p in path[1:]:
                df_aux = Pandas._read_parquet_path(session_primitives=session_primitives,
                                                   path=p,
                                                   columns=columns,
                                                   filters=filters,
                                                   procs_cpu_bound=procs_cpu_bound)
                df = pd.concat(objs=[df, df_aux], ignore_index=True)
        return df