Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_create_jobgroup(job_empty_script):
"""create dummy jobgroup."""
jobgroup = pyani_jobs.JobGroup("empty", "")
assert jobgroup.script == job_empty_script
def test_1d_jobgroup(job_scripts):
"""create dummy 1-parameter sweep jobgroup."""
jobgroup = pyani_jobs.JobGroup("1d-sweep", "cat", arguments=job_scripts[0].params)
assert (jobgroup.script, 3) == (job_scripts[0].script, jobgroup.tasks)
def test_remove_group_dependency(job_scripts):
"""add and remove jobgroup dependency."""
jg1 = pyani_jobs.JobGroup("1d-sweep", "cat", arguments=job_scripts[0].params)
jg2 = pyani_jobs.JobGroup("2d-sweep", "myprog", arguments=job_scripts[1].params)
jg2.add_dependency(jg1)
dep = jg2.dependencies[0]
jg2.remove_dependency(dep)
assert 0 == len(jg2.dependencies)
def test_add_group_dependency(job_scripts):
"""add jobgroup dependency."""
jg1 = pyani_jobs.JobGroup("1d-sweep", "cat", arguments=job_scripts[0].params)
jg2 = pyani_jobs.JobGroup("2d-sweep", "myprog", arguments=job_scripts[1].params)
jg2.add_dependency(jg1)
dep = jg2.dependencies[0]
assert (1, 3, 4, "1d-sweep") == (
len(jg2.dependencies),
dep.tasks,
jg2.tasks,
dep.name,
)
def test_2d_jobgroup(job_scripts):
"""create dummy 2-parameter sweep jobgroup."""
jobgroup = pyani_jobs.JobGroup(
"2d-sweep", "myprog", arguments=job_scripts[1].params
)
assert (jobgroup.script, 4) == (job_scripts[1].script, jobgroup.tasks)
def test_remove_group_dependency(job_scripts):
"""add and remove jobgroup dependency."""
jg1 = pyani_jobs.JobGroup("1d-sweep", "cat", arguments=job_scripts[0].params)
jg2 = pyani_jobs.JobGroup("2d-sweep", "myprog", arguments=job_scripts[1].params)
jg2.add_dependency(jg1)
dep = jg2.dependencies[0]
jg2.remove_dependency(dep)
assert 0 == len(jg2.dependencies)
:param jgprefix: str, prefix for SGE jobgroup
:param sgegroupsize: int, number of jobs in each SGE jobgroup
"""
jobcmds = defaultdict(list) # type: Dict[str, List[str]]
for job in joblist:
jobcmds[job.command.split(" ", 1)[0]].append(job.command)
jobgroups = [] # type: List
for cmds in list(jobcmds.items()):
# Break arglist up into batches of sgegroupsize (default: 10,000)
sublists = split_seq(cmds[1], sgegroupsize)
count = 0
for sublist in sublists:
count += 1
sge_jobcmdlist = [f'"{jc}"' for jc in sublist]
jobgroups.append(
JobGroup(
f"{jgprefix}_{count}", "$cmds", arguments={"cmds": sge_jobcmdlist}
)
)
return jobgroups
job.out = root_dir / "stdout"
job.err = root_dir / "stderr"
# Add the job name, current working directory, and SGE stdout/stderr
# directories to the SGE command line
args = f" -N {job.name} " % (job.name)
args += " -cwd "
args += f" -o {job.out} -e {job.err} "
# If a queue is specified, add this to the SGE command line
# LP: This has an undeclared variable, not sure why - delete?
# if job.queue is not None and job.queue in local_queues:
# args += local_queues[job.queue]
# If the job is actually a JobGroup, add the task numbering argument
if isinstance(job, JobGroup):
args += f"-t 1:{job.tasks} "
# If there are dependencies for this job, hold the job until they are
# complete
if job.dependencies:
args += "-hold_jid "
for dep in job.dependencies:
args += dep.name + ","
args = args[:-1]
# Build the qsub SGE commandline (passing local environment)
qsubcmd = f"{pyani_config.QSUB_DEFAULT} -V {args} {job.scriptpath}"
if sgeargs is not None:
qsubcmd = f"{qsubcmd} {sgeargs}"
# We've considered Bandit warnings B404,B603 and silence
# subprocess.call(qsubcmd, shell=False) # nosec