How to use the multiprocessing.util.debug function in multiprocessing

To help you get started, we’ve selected a few multiprocessing examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github IronLanguages / ironpython2 / External.LCA_RESTRICTED / Languages / IronPython / 27 / Lib / multiprocessing / managers.py View on Github external
except AttributeError:
                if methodname is None:
                    msg = ('#TRACEBACK', format_exc())
                else:
                    try:
                        fallback_func = self.fallback_mapping[methodname]
                        result = fallback_func(
                            self, conn, ident, obj, *args, **kwds
                            )
                        msg = ('#RETURN', result)
                    except Exception:
                        msg = ('#TRACEBACK', format_exc())

            except EOFError:
                util.debug('got EOF -- exiting thread serving %r',
                           threading.current_thread().name)
                sys.exit(0)

            except Exception:
                msg = ('#TRACEBACK', format_exc())

            try:
                try:
                    send(msg)
                except Exception, e:
                    send(('#UNSERIALIZABLE', repr(msg)))
            except Exception, e:
                util.info('exception in thread serving %r',
                        threading.current_thread().name)
                util.info(' ... message was %r', msg)
                util.info(' ... exception was %r', e)
github mozilla / zamboni-lib / lib / python / celery / concurrency / processes / pool.py View on Github external
def join(self):
        assert self._state in (CLOSE, TERMINATE)
        debug('joining worker handler')
        self._worker_handler.join()
        debug('joining task handler')
        self._task_handler.join()
        debug('joining result handler')
        self._result_handler.join()
        debug('result handler joined')
        for i, p in enumerate(self._pool):
            debug('joining worker %s/%s (%r)' % (i, len(self._pool), p, ))
            p.join()
github kbengine / kbengine / kbe / src / lib / python / Lib / multiprocessing / pool.py View on Github external
result_handler._state = TERMINATE
        outqueue.put(None)                  # sentinel

        # We must wait for the worker handler to exit before terminating
        # workers because we don't want workers to be restarted behind our back.
        debug('joining worker handler')
        worker_handler.join()

        # Terminate workers which haven't already finished.
        if pool and hasattr(pool[0], 'terminate'):
            debug('terminating workers')
            for p in pool:
                if p.exitcode is None:
                    p.terminate()

        debug('joining task handler')
        task_handler.join()

        debug('joining result handler')
        result_handler.join()

        if pool and hasattr(pool[0], 'terminate'):
            debug('joining pool workers')
            for p in pool:
                if p.is_alive():
                    # worker has not yet exited
                    debug('cleaning up worker %d' % p.pid)
                    p.join()
github python / cpython / Lib / concurrent / futures / process.py View on Github external
def weakref_cb(_,
                           thread_wakeup=self._queue_management_thread_wakeup):
                mp.util.debug('Executor collected: triggering callback for'
                              ' QueueManager wakeup')
                thread_wakeup.wakeup()
            # Start the processes so that their sentinels are known.
github akheron / cpython / Lib / multiprocessing / pool.py View on Github external
job, i, obj = task
            try:
                cache[job]._set(i, obj)
            except KeyError:
                pass

        while cache and thread._state != TERMINATE:
            try:
                task = get()
            except (IOError, EOFError):
                debug('result handler got EOFError/IOError -- exiting')
                return

            if task is None:
                debug('result handler ignoring extra sentinel')
                continue
            job, i, obj = task
            try:
                cache[job]._set(i, obj)
            except KeyError:
                pass

        if hasattr(outqueue, '_reader'):
            debug('ensuring that outqueue is not full')
            # If we don't make room available in outqueue then
            # attempts to add the sentinel (None) to outqueue may
            # block.  There is guaranteed to be no more than 2 sentinels.
            try:
                for i in range(10):
                    if not outqueue._reader.poll():
                        break
github OpenMDAO / OpenMDAO-Framework / openmdao.main / src / openmdao / main / mp_support.py View on Github external
if OpenMDAO_Proxy.manager_is_alive(token.address):
                # tell manager this process no longer cares about referent
                try:
                    util.debug('DECREF %r', token.id)
                    conn = _get_connection(_Client, token.address, authkey)
                    dispatch(conn, None, 'decref', (token.id,))
                # Hard to cause this to happen.
                except Exception as exc:  #pragma no cover
                    util.debug('... decref failed %s', exc)
        else:
            util.debug('DECREF %r -- manager already shutdown', token.id)

        # check whether we can close this thread's connection because
        # the process owns no more references to objects for this manager
        if not idset and hasattr(tls, 'connection'):
            util.debug('thread %r has no more %r proxies so closing conn',
                       threading.current_thread().name, token.typeid)
            tls.connection.close()
            del tls.connection
github mozilla / zamboni-lib / packages / celery / celery / concurrency / processes / pool.py View on Github external
try:
                task = get()
            except (IOError, EOFError), exc:
                debug('result handler got %s -- exiting',
                        exc.__class__.__name__)
                return

            if putlock is not None:
                try:
                    putlock.release()
                except ValueError:
                    pass

            if self._state:
                assert self._state == TERMINATE
                debug('result handler found thread._state=TERMINATE')
                break

            if task is None:
                debug('result handler got sentinel')
                break

            job, i, obj = task
            try:
                cache[job]._set(i, obj)
            except KeyError:
                pass

        if putlock is not None:
            try:
                putlock.release()
            except ValueError:
github StranikS-Scan / WorldOfTanks-Decompiled / source / res_bw / scripts / common / lib / multiprocessing / pool.py View on Github external
def _handle_results(outqueue, get, cache):
        thread = threading.current_thread()
        while 1:
            try:
                task = get()
            except (IOError, EOFError):
                debug('result handler got EOFError/IOError -- exiting')
                return

            if thread._state:
                assert thread._state == TERMINATE
                debug('result handler found thread._state=TERMINATE')
                break
            if task is None:
                debug('result handler got sentinel')
                break
            job, i, obj = task
            try:
                cache[job]._set(i, obj)
            except KeyError:
                pass

        while cache and thread._state != TERMINATE:
            try:
                task = get()
            except (IOError, EOFError):
                debug('result handler got EOFError/IOError -- exiting')
                return

            if task is None:
                debug('result handler ignoring extra sentinel')
github michalliu / MyCygwin / lib / python2.7 / multiprocessing / pool.py View on Github external
def _repopulate_pool(self):
        """Bring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        """
        for i in range(self._processes - len(self._pool)):
            w = self.Process(target=worker,
                             args=(self._inqueue, self._outqueue,
                                   self._initializer,
                                   self._initargs, self._maxtasksperchild)
                            )
            self._pool.append(w)
            w.name = w.name.replace('Process', 'PoolWorker')
            w.daemon = True
            w.start()
            debug('added worker')
github StranikS-Scan / WorldOfTanks-Decompiled / source / res_bw / scripts / common / lib / multiprocessing / connection.py View on Github external
def SocketClient(address):
    """
    Return a connection object connected to the socket given by `address`
    """
    family = getattr(socket, address_type(address))
    t = _init_timeout()
    while 1:
        s = socket.socket(family)
        s.setblocking(True)
        try:
            s.connect(address)
        except socket.error as e:
            s.close()
            if e.args[0] != errno.ECONNREFUSED or _check_timeout(t):
                debug('failed to connect to address %s', address)
                raise
            time.sleep(0.01)
        else:
            break

    else:
        raise

    fd = duplicate(s.fileno())
    conn = _multiprocessing.Connection(fd)
    s.close()
    return conn