Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
from time import sleep
from cl.g import Event
from scs.thread import gThread
SUP_ERROR_NOT_STARTED = """\
found thread not able to start?\
"""
SUP_ERROR_PING_TIMEOUT = """\
suspected thread crash or blocking: %r\
"""
class gSup(gThread):
def __init__(self, thread, signal, interval=5, timeout=600):
self.thread = thread
self.interval = interval
self.timeout = timeout
self.signal = signal
super(gSup, self).__init__()
def start_wait_child(self):
self._ready_event = Event()
self.signal.connect(self._on_thread_ready, sender=self.thread)
self.thread.start()
self._ready_event.wait()
assert self._ready_event.ready()
return self.thread
from celery import current_app as celery
from celery.utils import instantiate
from cl.g import Event
from kombu.utils import gen_unique_id
from . import signals
from . import supervisor
from .models import Broker, Node
from .intsup import gSup
from .state import state
from .thread import gThread
from .utils import setup_logging
class Agent(gThread):
controller_cls = "scs.controller.Controller"
httpd_cls = "scs.httpd.HttpServer"
supervisor_cls = "scs.supervisor.Supervisor"
httpd = None
controller = None
_components_ready = {}
_ready = False
def __init__(self, addrport="", id=None, loglevel=logging.INFO,
logfile=None, without_httpd=False, numc=2, sup_interval=None,
ready_event=None, **kwargs):
self.id = id or gen_unique_id()
if isinstance(addrport, basestring):
addr, _, port = addrport.partition(":")
addrport = (addr, int(port) if port else 8000)
return self.send_to_able("get", {"name": name}, to=name)
def add(self, name, nowait=False, **decl):
return self.throw("add", dict({"name": name}, **decl), nowait=nowait)
def delete(self, name, **kw):
nodes.remove_queue_from_all(name, nowait=True)
return self.send_to_able("delete", {"name": name}, to=name, **kw)
@property
def meta(self):
return {"queues": self.state.all()}
queues = Queue()
class Controller(ControllerBase, gThread):
actors = [App(), Node(), Queue()]
connect_max_retries = celery.conf.BROKER_CONNECTION_MAX_RETRIES
_ready_sent = False
def __init__(self, *args, **kwargs):
ControllerBase.__init__(self, *args, **kwargs)
gThread.__init__(self)
def on_awake(self):
# bind global actors to this agent,
# so presence can be used.
for actor in (apps, nodes, queues):
actor.agent = self
def on_connection_revived(self):
state.on_broker_revive()
from __future__ import with_statement
import sys
from contextlib import nested, contextmanager
from functools import partial
from celery import current_app as celery
from kombu import Consumer
from scs.state import state
from scs.thread import gThread
from scs.messaging import MessagingBase
class gConsumer(gThread, MessagingBase):
def get_consumers(self, Consumer, channel):
raise NotImplementedError("subclass responsibility")
def run(self):
while 1:
try:
self.consume_forever()
except self.connection_errors:
self.error("Connection to broker lost. "
"Trying to re-establish the connection...",
exc_info=sys.exc_info())
def consume_forever(self):
drain_events = self.drain_events
"""
from __future__ import absolute_import
from eventlet import listen
from eventlet import wsgi
from django.core.handlers import wsgi as djwsgi
from django.core.servers.basehttp import AdminMediaHandler
from requests import get
from .thread import gThread
from .signals import httpd_ready
class HttpServer(gThread):
joinable = False
def __init__(self, addrport=None):
self.addrport = addrport or ("", 8000)
super(HttpServer, self).__init__()
def server(self, sock, handler):
return wsgi.server(sock, handler,
log=self.create_log(),
protocol=self.create_http_protocol())
def run(self):
handler = AdminMediaHandler(djwsgi.WSGIHandler())
sock = listen(self.addrport)
g = self.spawn(self.server, sock, handler)
self.info("ready")
from cl.consumers import ConsumerMixin
from cl.models import ModelConsumer
from cl.pools import producers
from .. import metrics
from ..thread import gThread
from ..utils import rfc2822
from ..models import Node
from ..state import state
class NodeConsumer(ModelConsumer):
model = Node
class SRSAgent(ConsumerMixin, gThread):
"""The SRS Agent enables the SRS controller to communicate with this
agent via AMQP.
:param id: The agent id.
**Messaging basics**
The message body must be JSON encoded
(with a content type of ``application/json``),
**Creating instances**
An instance can be created by sending a message
to the ``srs.create.%(id)s`` fanout exchange, where
`id` is this agents id.
def errback(exc, interval):
supervisor.error(
"Error while trying to broadcast %r: %r\n" % (fun, exc))
supervisor.pause()
return _insured(node.broker.pool, fun, args, kwargs,
on_revive=state.on_broker_revive,
errback=errback)
def ib(fun, *args, **kwargs):
"""Shortcut to ``blocking(insured(fun.im_self, fun(*args, **kwargs)))``"""
return blocking(insured, fun.im_self, fun, *args, **kwargs)
class Supervisor(gThread):
"""The supervisor wakes up at intervals to monitor changes in the model.
It can also be requested to perform specific operations, and these
operations can be either async or sync.
:keyword interval: This is the interval (in seconds as an int/float),
between verifying all the registered nodes.
:keyword queue: Custom :class:`~Queue.Queue` instance used to send
and receive commands.
It is responsible for:
* Stopping removed instances.
* Starting new instances.
* Restarting unresponsive/killed instances.
* Making sure the instances consumes from the queues
specified in the model, sending ``add_consumer``/-
def __init__(self, *args, **kwargs):
ControllerBase.__init__(self, *args, **kwargs)
gThread.__init__(self)