Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
import os
import os.path as osp
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_hostname
from parsl.config import Config
parsl_config = Config(
executors=[
HighThroughputExecutor(
label="coffea_parsl_default",
address=address_by_hostname(),
cores_per_worker=max(multiprocessing.cpu_count()//2, 1),
max_workers=1,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
nodes_per_block=1
),
)
],
strategy=None,
)
parsl.load(parsl_config)
filelist = {
'ZJets': [osp.join(os.getcwd(),'tests/samples/nano_dy.root')],
'Data' : [osp.join(os.getcwd(),'tests/samples/nano_dimuon.root')]
}
def __init__(self,
label='ExtremeScaleExecutor',
provider=LocalProvider(),
launch_cmd=None,
address="127.0.0.1",
worker_ports=None,
worker_port_range=(54000, 55000),
interchange_port_range=(55000, 56000),
storage_access=None,
working_dir=None,
worker_debug=False,
ranks_per_node=1,
heartbeat_threshold=120,
heartbeat_period=30,
managed=True):
super().__init__(label=label,
provider=provider,
launch_cmd=launch_cmd,
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.launchers import SimpleLauncher
from parsl.config import Config
from parsl.executors import ExtremeScaleExecutor
config = Config(
executors=[
ExtremeScaleExecutor(
label="Extreme_Local",
worker_debug=True,
ranks_per_node=2,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
launcher=SimpleLauncher(),
)
)
],
strategy=None,
)
def __init__(self,
label='ExtremeScaleExecutor',
provider=LocalProvider(),
launch_cmd=None,
address="127.0.0.1",
worker_ports=None,
worker_port_range=(54000, 55000),
interchange_port_range=(55000, 56000),
storage_access=None,
working_dir=None,
worker_debug=False,
ranks_per_node=1,
heartbeat_threshold=120,
heartbeat_period=30,
managed=True):
super().__init__(label=label,
provider=provider,
launch_cmd=launch_cmd,
"""The following config uses two IPP executors designed for python apps which may
not show any performance improvements on local threads. This also allows you to
send work to two separate remote executors, or to two separate partitions.
"""
from parsl.config import Config
from parsl.providers import LocalProvider
from parsl.executors.ipp import IPyParallelExecutor
config = Config(
executors=[
IPyParallelExecutor(
label='local_ipp_1',
engine_dir='engines',
workers_per_node=1,
provider=LocalProvider(
nodes_per_block=1,
walltime="00:15:00",
init_blocks=4,
)
),
IPyParallelExecutor(
label='local_ipp_2',
engine_dir='engines',
workers_per_node=1,
provider=LocalProvider(
nodes_per_block=1,
walltime="00:15:00",
init_blocks=2,
)
# Set the worker ports explicitly. This is useful for re-using workers between workflows.
# worker_ports=(50078, 50079),
# Set a port range from which ports should be picked.
worker_port_range=(40010, 40020),
# The fabric_threaded.py script launches the MPI version
# launch_cmd="./cleanup.sh ; mpiexec -np 4 python3 fabric_threaded.py {debug} --task_url={task_url} --result_url={result_url}",
# launch_cmd="./cleanup.sh ; fabric_single_node.py {debug} --task_url={task_url} --result_url={result_url} ",
# launch_cmd="./cleanup.sh ",
# launch_cmd="sleep 600",
# Enable engine debug logging
engine_debug=True,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
tasks_per_node=1,
)
)
],
strategy=None,
)
The app decorator has a parameter `executors=[]` to specify the executor to which
apps should be directed.
"""
from parsl.providers import LocalProvider
from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor
from parsl.executors.threads import ThreadPoolExecutor
config = Config(
executors=[
ThreadPoolExecutor(max_threads=4, label='local_threads'),
IPyParallelExecutor(
label='local_ipp',
engine_dir='engines',
provider=LocalProvider(
walltime="00:05:00",
nodes_per_block=1,
init_blocks=4
)
Returns
-------
parsl.config.Config
Parsl config to execute tasks.
"""
config = Config(
executors=[
HighThroughputExecutor(
label="htex_local",
worker_debug=False,
poll_period=1,
cores_per_worker=1,
max_workers=1,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
min_blocks=1,
),
)
],
strategy=None
)
return config
# config
if args.executor == 'ThreadPool':
config = Config(
executors=[ThreadPoolExecutor(
#label='threads',
label='htex_local',
max_threads=5)
],
)
elif args.executor == 'HighThroughput_Local':
config = Config(
executors=[
HighThroughputExecutor(
label="htex_local",
cores_per_worker=1,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
# tasks_perss_node=1, # For HighThroughputExecutor, this option sho<
launcher=SingleNodeLauncher(),
),
)
],
#strategy='htex_aggressive',
#strategy='htex_totaltime',
#strategy='simple',
strategy=args.strategy,
)
elif args.executor == 'HighThroughput_Slurm':
config = Config(
executors=[
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.launchers import SimpleLauncher
from parsl.config import Config
from parsl.executors import ExtremeScaleExecutor
config = Config(
executors=[
ExtremeScaleExecutor(
label="Extreme_Local",
worker_debug=True,
ranks_per_node=2,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
launcher=SimpleLauncher(),
)
)
],
strategy=None,
)