How to use the parsl.providers.LocalProvider function in parsl

To help you get started, we’ve selected a few parsl examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github CoffeaTeam / coffea / tests / test_parsl.py View on Github external
import os
    import os.path as osp

    from parsl.providers import LocalProvider
    from parsl.channels import LocalChannel
    from parsl.executors import HighThroughputExecutor
    from parsl.addresses import address_by_hostname
    from parsl.config import Config
    parsl_config = Config(
        executors=[
            HighThroughputExecutor(
                label="coffea_parsl_default",
                address=address_by_hostname(),
                cores_per_worker=max(multiprocessing.cpu_count()//2, 1),
                max_workers=1,
                provider=LocalProvider(
                    channel=LocalChannel(),
                    init_blocks=1,
                    max_blocks=1,
                    nodes_per_block=1
                ),
            )
        ],
        strategy=None,
    )
    parsl.load(parsl_config)

    filelist = {
        'ZJets': [osp.join(os.getcwd(),'tests/samples/nano_dy.root')],
        'Data' : [osp.join(os.getcwd(),'tests/samples/nano_dimuon.root')]
    }
github Parsl / parsl / parsl / executors / extreme_scale / executor.py View on Github external
def __init__(self,
                 label='ExtremeScaleExecutor',
                 provider=LocalProvider(),
                 launch_cmd=None,
                 address="127.0.0.1",
                 worker_ports=None,
                 worker_port_range=(54000, 55000),
                 interchange_port_range=(55000, 56000),
                 storage_access=None,
                 working_dir=None,
                 worker_debug=False,
                 ranks_per_node=1,
                 heartbeat_threshold=120,
                 heartbeat_period=30,
                 managed=True):

        super().__init__(label=label,
                         provider=provider,
                         launch_cmd=launch_cmd,
github Parsl / parsl / parsl / configs / exex_local.py View on Github external
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.launchers import SimpleLauncher

from parsl.config import Config
from parsl.executors import ExtremeScaleExecutor

config = Config(
    executors=[
        ExtremeScaleExecutor(
            label="Extreme_Local",
            worker_debug=True,
            ranks_per_node=2,
            provider=LocalProvider(
                channel=LocalChannel(),
                init_blocks=1,
                max_blocks=1,
                launcher=SimpleLauncher(),
            )
        )
    ],
    strategy=None,
)
github funcx-faas / funcX / funcx / executor / parsl / executors / extreme_scale / executor.py View on Github external
def __init__(self,
                 label='ExtremeScaleExecutor',
                 provider=LocalProvider(),
                 launch_cmd=None,
                 address="127.0.0.1",
                 worker_ports=None,
                 worker_port_range=(54000, 55000),
                 interchange_port_range=(55000, 56000),
                 storage_access=None,
                 working_dir=None,
                 worker_debug=False,
                 ranks_per_node=1,
                 heartbeat_threshold=120,
                 heartbeat_period=30,
                 managed=True):

        super().__init__(label=label,
                         provider=provider,
                         launch_cmd=launch_cmd,
github funcx-faas / funcX / funcx / executor / parsl / configs / local_ipp_multisite.py View on Github external
"""The following config uses two IPP executors designed for python apps which may
not show any performance improvements on local threads. This also allows you to
send work to two separate remote executors, or to two separate partitions.
"""
from parsl.config import Config
from parsl.providers import LocalProvider
from parsl.executors.ipp import IPyParallelExecutor

config = Config(
    executors=[
        IPyParallelExecutor(
            label='local_ipp_1',
            engine_dir='engines',
            workers_per_node=1,
            provider=LocalProvider(
                nodes_per_block=1,
                walltime="00:15:00",
                init_blocks=4,
            )
        ),
        IPyParallelExecutor(
            label='local_ipp_2',
            engine_dir='engines',
            workers_per_node=1,
            provider=LocalProvider(
                nodes_per_block=1,
                walltime="00:15:00",
                init_blocks=2,
            )
github Parsl / parsl / parsl / executors / mpix / mpix_local.py View on Github external
# Set the worker ports explicitly. This is useful for re-using workers between workflows.
            # worker_ports=(50078, 50079),

            # Set a port range from which ports should be picked.
            worker_port_range=(40010, 40020),

            # The fabric_threaded.py script launches the MPI version
            # launch_cmd="./cleanup.sh ; mpiexec -np 4 python3 fabric_threaded.py {debug} --task_url={task_url} --result_url={result_url}",

            # launch_cmd="./cleanup.sh ; fabric_single_node.py {debug} --task_url={task_url} --result_url={result_url} ",
            # launch_cmd="./cleanup.sh ",
            # launch_cmd="sleep 600",

            # Enable engine debug logging
            engine_debug=True,
            provider=LocalProvider(
                channel=LocalChannel(),
                init_blocks=1,
                max_blocks=1,
                tasks_per_node=1,
            )
        )
    ],
    strategy=None,
)
github funcx-faas / funcX / funcx / executor / parsl / configs / local_threads_ipp.py View on Github external
The app decorator has a parameter `executors=[]` to specify the executor to which
apps should be directed.
"""
from parsl.providers import LocalProvider
from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor
from parsl.executors.threads import ThreadPoolExecutor

config = Config(
    executors=[
        ThreadPoolExecutor(max_threads=4, label='local_threads'),
        IPyParallelExecutor(
            label='local_ipp',
            engine_dir='engines',
            provider=LocalProvider(
                walltime="00:05:00",
                nodes_per_block=1,
                init_blocks=4
            )
github funcx-faas / funcX / funcx / endpoint / config.py View on Github external
Returns
    -------
    parsl.config.Config
        Parsl config to execute tasks.
    """

    config = Config(
        executors=[
            HighThroughputExecutor(
                label="htex_local",
                worker_debug=False,
                poll_period=1,
                cores_per_worker=1,
                max_workers=1,
                provider=LocalProvider(
                    channel=LocalChannel(),
                    init_blocks=1,
                    max_blocks=1,
                    min_blocks=1,
                ),
            )
        ],
        strategy=None
    )
    return config
github Parsl / parsl / parsl / dataflow / workspace / app_slurm_dag2.py View on Github external
# config
if args.executor == 'ThreadPool':
  config = Config(
      executors=[ThreadPoolExecutor(
          #label='threads',
          label='htex_local',
          max_threads=5)
      ],
  )
elif args.executor == 'HighThroughput_Local':
  config = Config(
      executors=[
          HighThroughputExecutor(
            label="htex_local",
            cores_per_worker=1,
            provider=LocalProvider(
              channel=LocalChannel(),
              init_blocks=1,
              max_blocks=1,
              # tasks_perss_node=1,  # For HighThroughputExecutor, this option sho<
              launcher=SingleNodeLauncher(),
            ),
        )
    ],
    #strategy='htex_aggressive',
    #strategy='htex_totaltime',
    #strategy='simple',
    strategy=args.strategy,
  )
elif args.executor == 'HighThroughput_Slurm':
  config = Config(
    executors=[
github funcx-faas / funcX / funcx / executor / parsl / configs / exex_local.py View on Github external
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.launchers import SimpleLauncher

from parsl.config import Config
from parsl.executors import ExtremeScaleExecutor

config = Config(
    executors=[
        ExtremeScaleExecutor(
            label="Extreme_Local",
            worker_debug=True,
            ranks_per_node=2,
            provider=LocalProvider(
                channel=LocalChannel(),
                init_blocks=1,
                max_blocks=1,
                launcher=SimpleLauncher(),
            )
        )
    ],
    strategy=None,
)