Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Preserve the default shell if the step shell is different
adapter_config.update({"shell": default_shell})
# Preserve the default batch type if the step batch type is different
adapter_config.update({"batch_type": default_batch_type})
self.mstep.setup_workspace()
self.mstep.generate_script(adapter)
step_name = self.name()
step_dir = self.get_workspace()
# dry run: sets up a workspace without executing any tasks. Each step's
# workspace directory is created, and each step's command script is
# written to it. The command script is not run, so there is no
# 'MERLIN_FINISHED' file, nor '.out' nor '.err' log files.
if adapter_config["dry_run"] is True:
return ReturnCode.DRY_OK
LOG.info(f"Executing step '{step_name}' in '{step_dir}'...")
# TODO: once maestrowf is updated so that execute returns a
# submissionrecord, then we need to return the record.return_code here
# at that point, we can drop the use of MerlinScriptAdapter above, and
# go back to using the adapter specified by the adapter_config['type']
# above
# If the above is done, then merlin_step in tasks.py can be changed to
# calls to the step execute and restart functions.
if self.restart and self.get_restart_cmd():
return ReturnCode(self.mstep.restart(adapter))
else:
return ReturnCode(self.mstep.execute(adapter))
# 'MERLIN_FINISHED' file, nor '.out' nor '.err' log files.
if adapter_config["dry_run"] is True:
return ReturnCode.DRY_OK
LOG.info(f"Executing step '{step_name}' in '{step_dir}'...")
# TODO: once maestrowf is updated so that execute returns a
# submissionrecord, then we need to return the record.return_code here
# at that point, we can drop the use of MerlinScriptAdapter above, and
# go back to using the adapter specified by the adapter_config['type']
# above
# If the above is done, then merlin_step in tasks.py can be changed to
# calls to the step execute and restart functions.
if self.restart and self.get_restart_cmd():
return ReturnCode(self.mstep.restart(adapter))
else:
return ReturnCode(self.mstep.execute(adapter))
# workspace directory is created, and each step's command script is
# written to it. The command script is not run, so there is no
# 'MERLIN_FINISHED' file, nor '.out' nor '.err' log files.
if adapter_config["dry_run"] is True:
return ReturnCode.DRY_OK
LOG.info(f"Executing step '{step_name}' in '{step_dir}'...")
# TODO: once maestrowf is updated so that execute returns a
# submissionrecord, then we need to return the record.return_code here
# at that point, we can drop the use of MerlinScriptAdapter above, and
# go back to using the adapter specified by the adapter_config['type']
# above
# If the above is done, then merlin_step in tasks.py can be changed to
# calls to the step execute and restart functions.
if self.restart and self.get_restart_cmd():
return ReturnCode(self.mstep.restart(adapter))
else:
return ReturnCode(self.mstep.execute(adapter))
self.max_retries = step.max_retries
step_name = step.name()
step_dir = step.get_workspace()
LOG.debug(f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'")
finished_filename = os.path.join(step_dir, "MERLIN_FINISHED")
# if we've already finished this task, skip it
if os.path.exists(finished_filename):
LOG.info(f"Skipping step '{step_name}' in '{step_dir}'.")
result = ReturnCode.OK
else:
result = step.execute(config)
if result == ReturnCode.OK:
LOG.info(f"Step '{step_name}' in '{step_dir}' finished successfully.")
# touch a file indicating we're done with this step
open(finished_filename, "a").close()
elif result == ReturnCode.DRY_OK:
LOG.info(f"Dry-ran step '{step_name}' in '{step_dir}'.")
elif result == ReturnCode.RESTART:
LOG.info(f"** Restarting step '{step_name}' in '{step_dir}'.")
step.restart = True
raise RestartException
elif result == ReturnCode.RETRY:
LOG.warning(f"** Retrying step '{step_name}' in '{step_dir}'.")
step.restart = False
raise RetryException
elif result == ReturnCode.SOFT_FAIL:
LOG.warning(
f"*** Step '{step_name}' in '{step_dir}' soft failed. Continuing with workflow."
)
elif result == ReturnCode.HARD_FAIL:
# stop all workers attached to this queue
"""
:param glob_path: a glob that should yield the paths to all merlin samples
:param sample_paths: a delimited list of all of the samples
:return : list of pairs indicating what needs to be substituted for a
merlin cmd
"""
substitutions = []
substitutions.append(("$(MERLIN_GLOB_PATH)", glob_path))
substitutions.append(("$(MERLIN_PATHS_ALL)", sample_paths))
# Return codes
substitutions.append(("$(MERLIN_SUCCESS)", str(int(ReturnCode.OK))))
substitutions.append(("$(MERLIN_RESTART)", str(int(ReturnCode.RESTART))))
substitutions.append(("$(MERLIN_SOFT_FAIL)", str(int(ReturnCode.SOFT_FAIL))))
substitutions.append(("$(MERLIN_HARD_FAIL)", str(int(ReturnCode.HARD_FAIL))))
substitutions.append(("$(MERLIN_RETRY)", str(int(ReturnCode.RETRY))))
substitutions.append(("$(MERLIN_STOP_WORKERS)", str(int(ReturnCode.STOP_WORKERS))))
return substitutions
with open(o_path, "a") as out:
out.write(output)
if join_output:
out.write("\n####### stderr follows #######\n")
out.write(err)
if not join_output:
e_path = os.path.join(cwd, "{}.err".format(new_output_name))
with open(e_path, "a") as out:
out.write(err)
if retcode == 0:
LOG.info("Execution returned status OK.")
return SubmissionRecord(ReturnCode.OK, retcode, pid)
else:
_record = SubmissionRecord(ReturnCode.ERROR, retcode, pid)
_record.add_info("stderr", str(err))
return _record
if self.samples_file is not None:
self.spec.merlin["samples"]["file"] = self.samples_file
self.spec.merlin["samples"]["generate"]["cmd"] = ""
self.restart_dir = restart_dir
self.special_vars = {
"SPECROOT": self.spec.specroot,
"MERLIN_TIMESTAMP": self.timestamp,
"MERLIN_INFO": self.info,
"MERLIN_WORKSPACE": self.workspace,
"OUTPUT_PATH": self.output_path,
"MERLIN_SUCCESS": str(int(ReturnCode.OK)),
"MERLIN_RESTART": str(int(ReturnCode.RESTART)),
"MERLIN_SOFT_FAIL": str(int(ReturnCode.SOFT_FAIL)),
"MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)),
"MERLIN_RETRY": str(int(ReturnCode.RETRY)),
}
self.dag = None
self.load_dag()
# If we load from a file, record that in the object for provenance
# downstream
if self.samples_file is not None:
self.spec.merlin["samples"]["file"] = self.samples_file
self.spec.merlin["samples"]["generate"]["cmd"] = ""
self.restart_dir = restart_dir
self.special_vars = {
"SPECROOT": self.spec.specroot,
"MERLIN_TIMESTAMP": self.timestamp,
"MERLIN_INFO": self.info,
"MERLIN_WORKSPACE": self.workspace,
"OUTPUT_PATH": self.output_path,
"MERLIN_SUCCESS": str(int(ReturnCode.OK)),
"MERLIN_RESTART": str(int(ReturnCode.RESTART)),
"MERLIN_SOFT_FAIL": str(int(ReturnCode.SOFT_FAIL)),
"MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)),
"MERLIN_RETRY": str(int(ReturnCode.RETRY)),
}
self.dag = None
self.load_dag()
else:
LOG.debug(f"discard argument {a}")
config = kwargs.pop("adapter_config", {"type": "local"})
next_in_chain = kwargs.pop("next_in_chain", None)
if step:
self.max_retries = step.max_retries
step_name = step.name()
step_dir = step.get_workspace()
LOG.debug(f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'")
finished_filename = os.path.join(step_dir, "MERLIN_FINISHED")
# if we've already finished this task, skip it
if os.path.exists(finished_filename):
LOG.info(f"Skipping step '{step_name}' in '{step_dir}'.")
result = ReturnCode.OK
else:
result = step.execute(config)
if result == ReturnCode.OK:
LOG.info(f"Step '{step_name}' in '{step_dir}' finished successfully.")
# touch a file indicating we're done with this step
open(finished_filename, "a").close()
elif result == ReturnCode.DRY_OK:
LOG.info(f"Dry-ran step '{step_name}' in '{step_dir}'.")
elif result == ReturnCode.RESTART:
LOG.info(f"** Restarting step '{step_name}' in '{step_dir}'.")
step.restart = True
raise RestartException
elif result == ReturnCode.RETRY:
LOG.warning(f"** Retrying step '{step_name}' in '{step_dir}'.")
step.restart = False
raise RetryException
def parameter_substitutions_for_cmd(glob_path, sample_paths):
"""
:param glob_path: a glob that should yield the paths to all merlin samples
:param sample_paths: a delimited list of all of the samples
:return : list of pairs indicating what needs to be substituted for a
merlin cmd
"""
substitutions = []
substitutions.append(("$(MERLIN_GLOB_PATH)", glob_path))
substitutions.append(("$(MERLIN_PATHS_ALL)", sample_paths))
# Return codes
substitutions.append(("$(MERLIN_SUCCESS)", str(int(ReturnCode.OK))))
substitutions.append(("$(MERLIN_RESTART)", str(int(ReturnCode.RESTART))))
substitutions.append(("$(MERLIN_SOFT_FAIL)", str(int(ReturnCode.SOFT_FAIL))))
substitutions.append(("$(MERLIN_HARD_FAIL)", str(int(ReturnCode.HARD_FAIL))))
substitutions.append(("$(MERLIN_RETRY)", str(int(ReturnCode.RETRY))))
substitutions.append(("$(MERLIN_STOP_WORKERS)", str(int(ReturnCode.STOP_WORKERS))))
return substitutions