How to use the pydra.cluster.module.Module function in pydra

To help you get started, we’ve selected a few pydra examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github kreneskyp / Pydra / pydra / cluster / master / scheduler.py View on Github external
from heapq import heappush, heappop, heapify
from twisted.internet import reactor, threads
from twisted.internet.defer import Deferred, DeferredList

from pydra.cluster.module import Module
from pydra.cluster.tasks import *
from pydra.cluster.tasks.task_manager import TaskManager
from pydra.cluster.constants import *
from pydra.models import TaskInstance, WorkUnit

# init logging
import logging
logger = logging.getLogger('root')


class TaskScheduler(Module):
    """
    Handles Scheduling tasks

    Methods:

        == worker availability ==
        remove_worker - called on worker disconnect
        #return_work_success
        #return_work_failed
        worker_connected
        worker_status_returned
        
        == scheduling ==
        queue_task - add task to queue
        cancel_task
        advance_queue - pick next task
github kreneskyp / Pydra / pydra / cluster / master / node_manager.py View on Github external
from django.core.paginator import Paginator, InvalidPage, EmptyPage

from pydra.cluster.module import Module
from pydra.models import Node

import logging
logger = logging.getLogger('root')

class NodeManager(Module):
    """
    Module for managing nodes.  Provides functionality for listing, creating, and editing nodes
    """

    _signals = [
        'NODE_CREATED'
        'NODE_DELETED',
        'NODE_EDITED'
    ]

    _shared = [
        'workers',
        '_idle_workers',
        '_active_workers',
        'nodes'
    ]
github kreneskyp / Pydra / pydra / cluster / master / worker_connection_manager.py View on Github external
from twisted.spread import pb

from pydra.cluster.auth.master_realm import MasterRealm
from pydra.cluster.auth.rsa_auth import load_crypto
from pydra.cluster.auth.worker_avatar import WorkerAvatar
from pydra.cluster.constants import *
from pydra.cluster.module import Module
import pydra_settings


# init logging
import logging
logger = logging.getLogger('root')


class WorkerConnectionManager(Module):

    _signals = [
        'WORKER_CONNECTED',
        'WORKER_DISCONNECTED',
    ]

    _shared = [
        'worker_checker',
        'workers'
    ]

    def __init__(self):
        self._services = [self.get_worker_service]
        
        #locks
        self._lock = Lock() #general lock, use when multiple shared resources are touched
github kreneskyp / Pydra / pydra / cluster / master / auto_discovery.py View on Github external
You should have received a copy of the GNU General Public License
    along with Pydra.  If not, see .
"""
import avahi
import dbus
from dbus.mainloop.glib import DBusGMainLoop

from pydra.cluster.module import Module
from pydra.models import Node
import pydra_settings

# init logging
import logging
logger = logging.getLogger('root')

class AutoDiscoveryModule(Module):

    _signals = [
        'NODE_CREATED',
    ]

    _shared = [
        'known_nodes'
    ]


    def __init__(self):
        self._interfaces = [self.list_known_nodes]
        self._listeners = {'MANAGER_INIT':self.autodiscovery}    

    def _register(self, manager):
        Module._register(self, manager)
github kreneskyp / Pydra / pydra / cluster / node / node_zero_conf_service.py View on Github external
(at your option) any later version.

    Pydra is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with Pydra.  If not, see .
"""

import platform
from pydra.cluster.module import Module
from pydra.util.zero_conf_service import ZeroConfService

class NodeZeroConfService(Module):
    """
    Module that publishes the node port using ZeroConfService (avahi)
    """

    _shared = [
        'port',
        'host'
    ]

    def __init__(self):

        self._listeners = {
            'MANAGER_INIT':self.start_service
        }