How to use the merlin.common.abstracts.enums.ReturnCode function in merlin

To help you get started, we’ve selected a few merlin examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github LLNL / merlin / merlin / study / step.py View on Github external
# Preserve the default shell if the step shell is different
        adapter_config.update({"shell": default_shell})
        # Preserve the default batch type if the step batch type is different
        adapter_config.update({"batch_type": default_batch_type})

        self.mstep.setup_workspace()
        self.mstep.generate_script(adapter)
        step_name = self.name()
        step_dir = self.get_workspace()

        # dry run: sets up a workspace without executing any tasks. Each step's
        # workspace directory is created, and each step's command script is
        # written to it. The command script is not run, so there is no
        # 'MERLIN_FINISHED' file, nor '.out' nor '.err' log files.
        if adapter_config["dry_run"] is True:
            return ReturnCode.DRY_OK

        LOG.info(f"Executing step '{step_name}' in '{step_dir}'...")
        # TODO: once maestrowf is updated so that execute returns a
        # submissionrecord, then we need to return the record.return_code here
        # at that point, we can drop the use of MerlinScriptAdapter above, and
        # go back to using the adapter specified by the adapter_config['type']
        # above
        # If the above is done, then merlin_step in tasks.py can be changed to
        # calls to the step execute and restart functions.
        if self.restart and self.get_restart_cmd():
            return ReturnCode(self.mstep.restart(adapter))
        else:
            return ReturnCode(self.mstep.execute(adapter))
github LLNL / merlin / merlin / study / step.py View on Github external
# 'MERLIN_FINISHED' file, nor '.out' nor '.err' log files.
        if adapter_config["dry_run"] is True:
            return ReturnCode.DRY_OK

        LOG.info(f"Executing step '{step_name}' in '{step_dir}'...")
        # TODO: once maestrowf is updated so that execute returns a
        # submissionrecord, then we need to return the record.return_code here
        # at that point, we can drop the use of MerlinScriptAdapter above, and
        # go back to using the adapter specified by the adapter_config['type']
        # above
        # If the above is done, then merlin_step in tasks.py can be changed to
        # calls to the step execute and restart functions.
        if self.restart and self.get_restart_cmd():
            return ReturnCode(self.mstep.restart(adapter))
        else:
            return ReturnCode(self.mstep.execute(adapter))
github LLNL / merlin / merlin / study / step.py View on Github external
# workspace directory is created, and each step's command script is
        # written to it. The command script is not run, so there is no
        # 'MERLIN_FINISHED' file, nor '.out' nor '.err' log files.
        if adapter_config["dry_run"] is True:
            return ReturnCode.DRY_OK

        LOG.info(f"Executing step '{step_name}' in '{step_dir}'...")
        # TODO: once maestrowf is updated so that execute returns a
        # submissionrecord, then we need to return the record.return_code here
        # at that point, we can drop the use of MerlinScriptAdapter above, and
        # go back to using the adapter specified by the adapter_config['type']
        # above
        # If the above is done, then merlin_step in tasks.py can be changed to
        # calls to the step execute and restart functions.
        if self.restart and self.get_restart_cmd():
            return ReturnCode(self.mstep.restart(adapter))
        else:
            return ReturnCode(self.mstep.execute(adapter))
github LLNL / merlin / merlin / common / tasks.py View on Github external
self.max_retries = step.max_retries
        step_name = step.name()
        step_dir = step.get_workspace()
        LOG.debug(f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'")
        finished_filename = os.path.join(step_dir, "MERLIN_FINISHED")
        # if we've already finished this task, skip it
        if os.path.exists(finished_filename):
            LOG.info(f"Skipping step '{step_name}' in '{step_dir}'.")
            result = ReturnCode.OK
        else:
            result = step.execute(config)
        if result == ReturnCode.OK:
            LOG.info(f"Step '{step_name}' in '{step_dir}' finished successfully.")
            # touch a file indicating we're done with this step
            open(finished_filename, "a").close()
        elif result == ReturnCode.DRY_OK:
            LOG.info(f"Dry-ran step '{step_name}' in '{step_dir}'.")
        elif result == ReturnCode.RESTART:
            LOG.info(f"** Restarting step '{step_name}' in '{step_dir}'.")
            step.restart = True
            raise RestartException
        elif result == ReturnCode.RETRY:
            LOG.warning(f"** Retrying step '{step_name}' in '{step_dir}'.")
            step.restart = False
            raise RetryException
        elif result == ReturnCode.SOFT_FAIL:
            LOG.warning(
                f"*** Step '{step_name}' in '{step_dir}' soft failed. Continuing with workflow."
            )
        elif result == ReturnCode.HARD_FAIL:

            # stop all workers attached to this queue
github LLNL / merlin / merlin / spec / expansion.py View on Github external
"""
    :param glob_path: a glob that should yield the paths to all merlin samples
    :param sample_paths: a delimited list of all of the samples

    :return : list of pairs indicating what needs to be substituted for a
        merlin cmd
    """
    substitutions = []
    substitutions.append(("$(MERLIN_GLOB_PATH)", glob_path))
    substitutions.append(("$(MERLIN_PATHS_ALL)", sample_paths))
    # Return codes
    substitutions.append(("$(MERLIN_SUCCESS)", str(int(ReturnCode.OK))))
    substitutions.append(("$(MERLIN_RESTART)", str(int(ReturnCode.RESTART))))
    substitutions.append(("$(MERLIN_SOFT_FAIL)", str(int(ReturnCode.SOFT_FAIL))))
    substitutions.append(("$(MERLIN_HARD_FAIL)", str(int(ReturnCode.HARD_FAIL))))
    substitutions.append(("$(MERLIN_RETRY)", str(int(ReturnCode.RETRY))))
    substitutions.append(("$(MERLIN_STOP_WORKERS)", str(int(ReturnCode.STOP_WORKERS))))
    return substitutions
github LLNL / merlin / merlin / study / script_adapter.py View on Github external
with open(o_path, "a") as out:
                out.write(output)

                if join_output:
                    out.write("\n####### stderr follows #######\n")
                    out.write(err)

            if not join_output:
                e_path = os.path.join(cwd, "{}.err".format(new_output_name))
                with open(e_path, "a") as out:
                    out.write(err)

        if retcode == 0:
            LOG.info("Execution returned status OK.")
            return SubmissionRecord(ReturnCode.OK, retcode, pid)
        else:
            _record = SubmissionRecord(ReturnCode.ERROR, retcode, pid)
            _record.add_info("stderr", str(err))
            return _record
github LLNL / merlin / merlin / study / study.py View on Github external
if self.samples_file is not None:
            self.spec.merlin["samples"]["file"] = self.samples_file
            self.spec.merlin["samples"]["generate"]["cmd"] = ""

        self.restart_dir = restart_dir

        self.special_vars = {
            "SPECROOT": self.spec.specroot,
            "MERLIN_TIMESTAMP": self.timestamp,
            "MERLIN_INFO": self.info,
            "MERLIN_WORKSPACE": self.workspace,
            "OUTPUT_PATH": self.output_path,
            "MERLIN_SUCCESS": str(int(ReturnCode.OK)),
            "MERLIN_RESTART": str(int(ReturnCode.RESTART)),
            "MERLIN_SOFT_FAIL": str(int(ReturnCode.SOFT_FAIL)),
            "MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)),
            "MERLIN_RETRY": str(int(ReturnCode.RETRY)),
        }
        self.dag = None
        self.load_dag()
github LLNL / merlin / merlin / study / study.py View on Github external
# If we load from a file, record that in the object for provenance
        # downstream
        if self.samples_file is not None:
            self.spec.merlin["samples"]["file"] = self.samples_file
            self.spec.merlin["samples"]["generate"]["cmd"] = ""

        self.restart_dir = restart_dir

        self.special_vars = {
            "SPECROOT": self.spec.specroot,
            "MERLIN_TIMESTAMP": self.timestamp,
            "MERLIN_INFO": self.info,
            "MERLIN_WORKSPACE": self.workspace,
            "OUTPUT_PATH": self.output_path,
            "MERLIN_SUCCESS": str(int(ReturnCode.OK)),
            "MERLIN_RESTART": str(int(ReturnCode.RESTART)),
            "MERLIN_SOFT_FAIL": str(int(ReturnCode.SOFT_FAIL)),
            "MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)),
            "MERLIN_RETRY": str(int(ReturnCode.RETRY)),
        }
        self.dag = None
        self.load_dag()
github LLNL / merlin / merlin / common / tasks.py View on Github external
else:
            LOG.debug(f"discard argument {a}")

    config = kwargs.pop("adapter_config", {"type": "local"})
    next_in_chain = kwargs.pop("next_in_chain", None)

    if step:
        self.max_retries = step.max_retries
        step_name = step.name()
        step_dir = step.get_workspace()
        LOG.debug(f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'")
        finished_filename = os.path.join(step_dir, "MERLIN_FINISHED")
        # if we've already finished this task, skip it
        if os.path.exists(finished_filename):
            LOG.info(f"Skipping step '{step_name}' in '{step_dir}'.")
            result = ReturnCode.OK
        else:
            result = step.execute(config)
        if result == ReturnCode.OK:
            LOG.info(f"Step '{step_name}' in '{step_dir}' finished successfully.")
            # touch a file indicating we're done with this step
            open(finished_filename, "a").close()
        elif result == ReturnCode.DRY_OK:
            LOG.info(f"Dry-ran step '{step_name}' in '{step_dir}'.")
        elif result == ReturnCode.RESTART:
            LOG.info(f"** Restarting step '{step_name}' in '{step_dir}'.")
            step.restart = True
            raise RestartException
        elif result == ReturnCode.RETRY:
            LOG.warning(f"** Retrying step '{step_name}' in '{step_dir}'.")
            step.restart = False
            raise RetryException
github LLNL / merlin / merlin / spec / expansion.py View on Github external
def parameter_substitutions_for_cmd(glob_path, sample_paths):
    """
    :param glob_path: a glob that should yield the paths to all merlin samples
    :param sample_paths: a delimited list of all of the samples

    :return : list of pairs indicating what needs to be substituted for a
        merlin cmd
    """
    substitutions = []
    substitutions.append(("$(MERLIN_GLOB_PATH)", glob_path))
    substitutions.append(("$(MERLIN_PATHS_ALL)", sample_paths))
    # Return codes
    substitutions.append(("$(MERLIN_SUCCESS)", str(int(ReturnCode.OK))))
    substitutions.append(("$(MERLIN_RESTART)", str(int(ReturnCode.RESTART))))
    substitutions.append(("$(MERLIN_SOFT_FAIL)", str(int(ReturnCode.SOFT_FAIL))))
    substitutions.append(("$(MERLIN_HARD_FAIL)", str(int(ReturnCode.HARD_FAIL))))
    substitutions.append(("$(MERLIN_RETRY)", str(int(ReturnCode.RETRY))))
    substitutions.append(("$(MERLIN_STOP_WORKERS)", str(int(ReturnCode.STOP_WORKERS))))
    return substitutions