How to use the pathos.multiprocessing.cpu_count function in pathos

To help you get started, we’ve selected a few pathos examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github PPPLDeepLearning / plasma-python / plasma / models / runner.py View on Github external
def make_predictions(conf, shot_list, loader):
    loader.set_inference_mode(True)

    use_cores = max(1, mp.cpu_count()-2)

    if backend == 'tf' or backend == 'tensorflow':
        first_time = "tensorflow" not in sys.modules
        if first_time:
            import tensorflow as tf
            os.environ['KERAS_BACKEND'] = 'tensorflow'
            from keras.backend.tensorflow_backend import set_session
            config = tf.ConfigProto(device_count={"CPU": use_cores})
            set_session(tf.Session(config=config))
    else:
        os.environ['THEANO_FLAGS'] = 'device=cpu'
        # import theano

    from plasma.models.builder import ModelBuilder
    specific_builder = ModelBuilder(conf)
github Rapid-Design-of-Systems-Laboratory / beluga / beluga / beluga.py View on Github external
else:
        pool = None

    if ocp is None:
        raise NotImplementedError('\"ocp\" must be defined.')
    """
    Main code
    """

    f_ocp = FuncOCP(ocp)
    # breakpoint()
    # s_ocp = SymOCP(ocp)
    # breakpoint()
    # bvp = FuncBVP(s_bvp)

    logging.debug('Using ' + str(n_cpus) + '/' + str(pathos.multiprocessing.cpu_count()) + ' CPUs. ')

    if bvp is None:
        bvp, ocp_map, ocp_map_inverse = ocp2bvp(ocp, method=method, optim_options=optim_options)
        logging.debug('Resulting BVP problem:')
        for key in bvp.raw.keys():
            logging.debug(str(key) + ': ' + str(bvp.raw[key]))

    else:
        if ocp_map is None or ocp_map_inverse is None:
            raise ValueError('BVP problem must have an associated \'ocp_map\' and \'ocp_map_inverse\'')



    solinit = Trajectory()
    solinit.const = np.array(bvp.raw['constants_values'])
github thouska / spotpy / spotpy / parallel / mproc.py View on Github external
def __init__(self,process):
        self.size = mp.cpu_count()
        self.process = process
        self.phase=None
        self.pool = mp.ProcessingPool(self.size)
github PPPLDeepLearning / plasma-python / plasma / preprocessor / normalize.py View on Github external
shot_list = ShotList()
        shot_list.load_from_shot_list_files_objects(shot_files, all_signals)
        shot_list_picked = shot_list.random_sublist(use_shots)

        previously_saved, machines_saved = self.previously_saved_stats()
        machines_to_compute = all_machines - machines_saved
        recompute = conf['data']['recompute_normalization']
        if recompute:
            machines_to_compute = all_machines
            previously_saved = False
        if not previously_saved or len(machines_to_compute) > 0:
            if previously_saved:
                self.load_stats(verbose=True)
            print('computing normalization for machines {}'.format(
                machines_to_compute))
            use_cores = max(1, mp.cpu_count()-2)
            pool = mp.Pool(use_cores)
            print('running in parallel on {} processes'.format(
                pool._processes))
            start_time = time.time()

            for (i, stats) in enumerate(pool.imap_unordered(
                    self.train_on_single_shot, shot_list_picked)):
                # for (i,stats) in
                # enumerate(map(self.train_on_single_shot,shot_list_picked)):
                if stats.machine in machines_to_compute:
                    self.incorporate_stats(stats)
                    self.machines.add(stats.machine)
                sys.stdout.write('\r'
                                 + '{}/{}'.format(i, len(shot_list_picked)))
            pool.close()
            pool.join()
github mantidproject / mantid / Framework / PythonInterface / plugins / algorithms / Abins.py View on Github external
def _check_threads(self, message_end=None):
        """
        Checks number of threads
        :param message_end: closing part of the error message.
        """
        if PATHOS_FOUND:
            threads = AbinsModules.AbinsParameters.performance['threads']
            if not (isinstance(threads, six.integer_types) and 1 <= threads <= mp.cpu_count()):
                raise RuntimeError("Invalid number of threads for parallelisation over atoms" + message_end)
github mantidproject / mantid / Framework / PythonInterface / plugins / algorithms / Abins.py View on Github external
def _check_threads(self, message_end=None):
        """
        Checks number of threads
        :param message_end: closing part of the error message.
        """
        if PATHOS_FOUND:
            atoms_threads = AbinsParameters.atoms_threads
            if not (isinstance(atoms_threads, (int, long)) and 1 <= atoms_threads <= mp.cpu_count()):
                raise RuntimeError("Invalid number of threads for parallelisation over atoms" + message_end)

            q_threads = AbinsParameters.q_threads
            if not (isinstance(q_threads, (int, long)) and 1 <= q_threads <= mp.cpu_count()):
                raise RuntimeError("Invalid number of threads for parallelisation over q" + message_end)

            if atoms_threads * q_threads > mp.cpu_count():
                raise RuntimeError("User asked for more threads than available.")
github morganics / bayesianpy / bayesianpy / model.py View on Github external
def _calc_num_threads(self, df_size: int, query_size: int, max_threads=None) -> int:
        num_queries = df_size * query_size

        if mp.cpu_count() == 1:
            max = 1
        else:
            max = mp.cpu_count() - 1

        calc = int(num_queries / 5000)
        if calc > max:
            r = max
        elif calc <= 1:
            if num_queries > 1000:
                r = 2
            else:
                r = 1
        else:
            r = calc

        if max_threads is not None and r > max_threads:
            return max_threads

        return r
github morganics / bayesianpy / bayesianpy / output.py View on Github external
def _calc_num_threads(self, df_size: int, query_size: int, max_threads=None) -> int:
        num_queries = df_size * query_size

        if mp.cpu_count() == 1:
            max = 1
        else:
            max = mp.cpu_count() - 1

        calc = int(num_queries / 5000)
        if calc > max:
            r = max
        elif calc <= 1:
            if num_queries > 1000:
                r = 2
            else:
                r = 1
        else:
            r = calc

        if max_threads is not None and r > max_threads:
            return max_threads

        return r