How to use the parsl.launchers.SingleNodeLauncher function in parsl

To help you get started, we’ve selected a few parsl examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github funcx-faas / funcX / funcx / executor / parsl / providers / grid_engine / grid_engine.py View on Github external
def __init__(self,
                 channel=LocalChannel(),
                 nodes_per_block=1,
                 init_blocks=1,
                 min_blocks=0,
                 max_blocks=10,
                 parallelism=1,
                 walltime="00:10:00",
                 scheduler_options='',
                 worker_init='',
                 launcher=SingleNodeLauncher()):
        label = 'grid_engine'
        super().__init__(label,
                         channel,
                         nodes_per_block,
                         init_blocks,
                         min_blocks,
                         max_blocks,
                         parallelism,
                         walltime,
                         launcher)
        self.scheduler_options = scheduler_options
        self.worker_init = worker_init

        if launcher in ['srun', 'srun_mpi']:
            logger.warning("Use of {} launcher is usually appropriate for Slurm providers. "
                           "Recommended options include 'single_node' or 'aprun'.".format(launcher))
github Parsl / parsl / parsl / dataflow / workspace / app_slurm_dag_zhuozhao.py View on Github external
label='htex_local',
          max_threads=5)
      ],
  )
elif args.executor == 'HighThroughput_Local':
  config = Config(
      executors=[
          HighThroughputExecutor(
            label="htex_local",
            cores_per_worker=1,
            provider=LocalProvider(
              channel=LocalChannel(),
              init_blocks=1,
              max_blocks=1,
              # tasks_perss_node=1,  # For HighThroughputExecutor, this option sho<
              launcher=SingleNodeLauncher(),
            ),
        )
    ],
    #strategy='htex_aggressive',
    #strategy='htex_totaltime',
    #strategy='simple',
    strategy=args.strategy,
  )
elif args.executor == 'HighThroughput_Slurm':
  config = Config(
    executors=[
        HighThroughputExecutor(
            label="midway_htex",
            cores_per_worker=1,
            address=address_by_hostname(),
            provider=SlurmProvider(
github Parsl / parsl / parsl / configs / cooley_ssh_il_single_node.py View on Github external
label='cooley_ssh_il_local_single_node',
            provider=CobaltProvider(
                channel=SSHInteractiveLoginChannel(
                    hostname='cooley.alcf.anl.gov',
                    username='USERNAME',     # Please replace USERNAME with your username
                    script_dir='/home/USERNAME/parsl_scripts/',    # Please replace USERNAME with your username
                ),
                nodes_per_block=1,
                init_blocks=1,
                max_blocks=1,
                walltime="00:05:00",
                scheduler_options='',     # Input your scheduler_options if needed
                worker_init='',     # Input your worker_init if needed
                queue='pubnet-debug',
                account='ALCF_ALLOCATION',    # Please replace ALCF_ALLOCATION with your ALCF allocation
                launcher=SingleNodeLauncher(),
            ),
            controller=Controller(public_ip='PUBLIC_IP'),    # Please replace PUBLIC_IP with your public ip
        )
github funcx-faas / funcX / funcx / executor / parsl / providers / googlecloud / googlecloud.py View on Github external
def __init__(self,
                 project_id,
                 key_file,
                 region,
                 os_project,
                 os_family,
                 google_version='v1',
                 instance_type='n1-standard-1',
                 init_blocks=1,
                 min_blocks=0,
                 max_blocks=10,
                 launcher=SingleNodeLauncher(),
                 parallelism=1):
        self.project_id = project_id
        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = key_file
        self.zone = self.get_zone(region)
        self.os_project = os_project
        self.os_family = os_family
        self.label = 'google_cloud'
        self.client = googleapiclient.discovery.build('compute', google_version)
        self.instance_type = instance_type
        self.init_blocks = init_blocks
        self.min_blocks = min_blocks
        self.max_blocks = max_blocks
        self.parallelism = parallelism
        self.num_instances = 0
        self.launcher = launcher
github Parsl / parsl / parsl / dataflow / workspace / app_dev2_dag_zhuozhao.py View on Github external
label='htex_local',
          max_threads=5)
      ],
  )
elif args.executor == 'HighThroughput':
  config = Config(
      executors=[
          HighThroughputExecutor(
            label="htex_local",
            cores_per_worker=1,
            provider=LocalProvider(
              channel=LocalChannel(),
              init_blocks=1,
              max_blocks=1,
              # tasks_per_node=1,  # For HighThroughputExecutor, this option sho<
              launcher=SingleNodeLauncher(),
            ),
        )
    ],
    #strategy='htex_aggressive',
    strategy='htex_totaltime',
    #strategy='simple',
  )
# TODO: 
#try:
#except:
#  raise NameError("Invalid parsed argument")  

# Load config
print(config)
dfk = parsl.load(config)
github Parsl / parsl / parsl / dataflow / workspace / cpu_stress_midway.py View on Github external
return _logging

# config
config = Config(
    executors=[
        HighThroughputExecutor(
            label="local_threads",
            #label="htex_local",
            # worker_debug=True,
            cores_per_worker=1,
            provider=LocalProvider(
                channel=LocalChannel(),
                init_blocks=1,
                max_blocks=4,
                # tasks_per_node=1,  # For HighThroughputExecutor, this option sho<
                launcher=SingleNodeLauncher(),
            ),
        )
    ],
    #strategy='htex_aggressive',
    strategy='htex_totaltime',
    #strategy='simple',
)
# Load config 
parsl.load(config)


@python_app
def func(n=1000000, stime=0.00):
  # import necessary library?! otherwise get errors
  import os, sys, time
  import psutil
github Parsl / parsl / parsl / providers / local / local.py View on Github external
def __init__(self,
                 channel=LocalChannel(),
                 tasks_per_node=1,
                 nodes_per_block=1,
                 launcher=SingleNodeLauncher(),
                 init_blocks=4,
                 min_blocks=0,
                 max_blocks=10,
                 walltime="00:15:00",
                 parallelism=1):
        self.channel = channel
        self.label = 'local'
        self.provisioned_blocks = 0
        self.nodes_per_block = nodes_per_block
        self.tasks_per_node = tasks_per_node
        self.launcher = launcher
        self.init_blocks = init_blocks
        self.min_blocks = min_blocks
        self.max_blocks = max_blocks
        self.parallelism = parallelism
        self.walltime = walltime
github Parsl / parsl / parsl / providers / grid_engine / grid_engine.py View on Github external
def __init__(self,
                 channel=LocalChannel(),
                 nodes_per_block=1,
                 init_blocks=1,
                 min_blocks=0,
                 max_blocks=10,
                 parallelism=1,
                 walltime="00:10:00",
                 scheduler_options='',
                 worker_init='',
                 launcher=SingleNodeLauncher()):
        label = 'grid_engine'
        super().__init__(label,
                         channel,
                         nodes_per_block,
                         init_blocks,
                         min_blocks,
                         max_blocks,
                         parallelism,
                         walltime,
                         launcher)
        self.scheduler_options = scheduler_options
        self.worker_init = worker_init

        if launcher in ['srun', 'srun_mpi']:
            logger.warning("Use of {} launcher is usually appropriate for Slurm providers. "
                           "Recommended options include 'single_node' or 'aprun'.".format(launcher))
github funcx-faas / funcX / funcx / executor / parsl / providers / local / local.py View on Github external
def __init__(self,
                 channel=LocalChannel(),
                 nodes_per_block=1,
                 launcher=SingleNodeLauncher(),
                 init_blocks=4,
                 min_blocks=0,
                 max_blocks=10,
                 walltime="00:15:00",
                 worker_init='',
                 cmd_timeout=30,
                 parallelism=1,
                 move_files=None):
        self.channel = channel
        self._label = 'local'
        self.provisioned_blocks = 0
        self.nodes_per_block = nodes_per_block
        self.launcher = launcher
        self.worker_init = worker_init
        self.init_blocks = init_blocks
        self.min_blocks = min_blocks
github Parsl / parsl / parsl / providers / local / local.py View on Github external
def __init__(self,
                 channel=LocalChannel(),
                 nodes_per_block=1,
                 launcher=SingleNodeLauncher(),
                 init_blocks=4,
                 min_blocks=0,
                 max_blocks=10,
                 walltime="00:15:00",
                 worker_init='',
                 cmd_timeout=30,
                 parallelism=1,
                 move_files=None):
        self.channel = channel
        self._label = 'local'
        self.provisioned_blocks = 0
        self.nodes_per_block = nodes_per_block
        self.launcher = launcher
        self.worker_init = worker_init
        self.init_blocks = init_blocks
        self.min_blocks = min_blocks