Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
script = DSC_Script(sys.argv[1])
script.print_help(print_version='-v' in sys.argv)
sys.exit(0)
except Exception as e:
if '--debug' in sys.argv:
raise
else:
env.logger.error(
f'No help information is available for script {sys.argv[1]}: ``{e}``'
)
sys.exit(1)
try:
args, unknown_args = p.parse_known_args()
except Exception as e:
env.logger.error(e)
env.logger.info("Please type ``{} -h`` to view available options".\
format(os.path.basename(sys.argv[0])))
sys.exit(1)
#
env.verbosity = args.verbosity
# keep `args.__recover__` to maintain backwards compatibility for `--touch` option.
if args.__recover__:
env.logger.warning(
f'Option ``--touch`` is deprecated. Please use ``-s existing`` next time.'
)
args.__construct__ = 'existing'
with Timer(verbose=True if (args.verbosity > 0) else False) as t:
try:
args.func(args, unknown_args)
except KeyboardInterrupt:
t.disable()
sys.exit('KeyboardInterrupt')
#pid = p.pid
ret = p.wait()
except Exception as e:
env.logger.error(e)
if ret != 0:
temp_file = os.path.join('.sos', f'{"Rmarkdown"}_{os.getpid()}.md')
shutil.copyfile(str(input), temp_file)
cmd = interpolate(f'Rscript -e "rmarkdown::render({args})"',
{'input': input, 'output': sos_targets(temp_file)})
raise RuntimeError(
f'Failed to execute script. Please use command \n"{cmd}"\nunder {os.getcwd()} to test it.')
if write_to_stdout:
with open(str(output[0])) as out:
sys.stdout.write(out.read())
else:
env.logger.info(f'Report saved to {output}')
env.logger.error(err_msg)
env.logger.error('Failed to get converted file.')
else:
with open(dest_file, 'rb') as tfile:
sys.stdout.buffer.write(tfile.read())
try:
os.remove(tmp)
except Exception:
pass
else:
ret = subprocess.call(['jupyter', 'nbconvert', os.path.abspath(notebook_file), '--to', to_format,
'--output', os.path.abspath(output_file)] + ([] if unknown_args is None else unknown_args))
if ret != 0:
env.logger.error('Failed to convert {} to {} format'.format(notebook_file, to_format))
else:
env.logger.info('Output saved to {}'.format(output_file))
# now, there should be no dangling targets, let us connect nodes
# this can be done more efficiently
runnable._depends_targets.append(target)
dag._all_dependent_files[target].append(runnable)
dag.build(self.workflow.auxiliary_sections)
#
cycle = dag.circular_dependencies()
if cycle:
raise RuntimeError('Circular dependency detected {}. It is likely a later step produces input of a previous step.'.format(cycle))
elif isinstance(res, RemovedTarget):
runnable._status = None
dag.regenerate_target(res.target)
elif isinstance(res, UnavailableLock):
runnable._status = 'pending'
runnable._signature = (res.output, res.sig_file)
env.logger.info('Waiting on another process for step {}'.format(section.step_name()))
# if the job is failed
elif isinstance(res, Exception):
runnable._status = 'failed'
exec_error.append(runnable._node_id, res)
prog.progress(1)
else:#
for k, v in res.items():
env.sos_dict.set(k, v)
#
# set context to the next logic step.
for edge in dag.out_edges(runnable):
node = edge[1]
# if node is the logical next step...
if node._node_index is not None and runnable._node_index is not None:
#and node._node_index == runnable._node_index + 1:
node._context.update(env.sos_dict.clone_selected_vars(
security_opt, # security option
volumes_opt, # volumes
volumes_from_opt, # volumes_from
name_opt, # name
stdin_opt, # stdin_optn
tty_opt, # tty
port_opt, # port
working_dir_opt, # working dir
user_opt, # user
env_opt, # environment
mem_limit_opt, # memory limit
extra_opt, # any extra parameters
image, # image
cmd_opt
)
env.logger.info(command)
ret = subprocess.call(command, shell=True)
if ret != 0:
msg = 'The script has been saved to .sos/{} so that you can execute it using the following command:\n{}'.format(
tempscript, command.replace(tempdir, os.path.abspath('./.sos')))
shutil.copy(os.path.join(tempdir, tempscript), '.sos')
if ret == 125:
raise RuntimeError('Docker daemon failed (exitcode=125). ' + msg)
elif ret == 126:
raise RuntimeError('Failed to invoke specified command (exitcode=126). ' + msg)
elif ret == 127:
raise RuntimeError('Failed to locate specified command (exitcode=127). ' + msg)
elif ret == 137:
if not hasattr(self, 'tot_mem'):
self.tot_mem = self.total_memory(image)
if self.tot_mem is None:
raise RuntimeError('Script killed by docker. ' + msg)
env.logger.error(str(e))
if ret != 0:
temp_file = os.path.join('.sos', f'{"Rmarkdown"}_{os.getpid()}.md')
shutil.copyfile(str(input), temp_file)
cmd = interpolate(f'Rscript -e "rmarkdown::render({args})"', {
'input': input,
'output': sos_targets(temp_file)
})
raise RuntimeError(
f'Failed to execute script. Please use command \n"{cmd}"\nunder {os.getcwd()} to test it.'
)
if write_to_stdout:
with open(str(output[0])) as out:
sys.stdout.write(out.read())
else:
env.logger.info(f'Report saved to {output}')
if to_html:
output_string.append(t.get_html_string(title='R libraries'))
else:
env.logger.info("``R LIBRARIES``")
print(t)
print('')
if print_version and len(self.runtime.pymodule):
from .utils import get_pymodule_versions
libs, versions = get_pymodule_versions(self.runtime.pymodule)
t = PrettyTable()
t.add_column('name', libs)
t.add_column('version', versions)
if to_html:
output_string.append(t.get_html_string(title='Python modules'))
else:
env.logger.info("``PYTHON MODULES``")
print(t)
print('')
return output_string
else:
print(
t.get_string(
title='Ungrouped' if len(rm_rows) else 'All modules'))
print('')
if not to_html:
env.logger.info("``PIPELINES``")
print(res['pipelines'] + '\n')
env.logger.info("``PIPELINES EXPANDED``")
print('\n'.join([
f'{i+1}: ' + ' * '.join(x)
for i, x in enumerate(self.runtime.sequence)
]) + '\n')
if print_version and len(
[x for x in self.runtime.rlib if not x.startswith('dscrutils')]):
env.logger.info("Scanning package versions ...")
from .utils import get_rlib_versions
libs, versions = get_rlib_versions(self.runtime.rlib)
t = PrettyTable()
t.add_column('name', libs)
t.add_column('version', versions)
if to_html:
output_string.append(t.get_html_string(title='R libraries'))
else:
env.logger.info("``R LIBRARIES``")
print(t)
print('')
if print_version and len(self.runtime.pymodule):
from .utils import get_pymodule_versions
libs, versions = get_pymodule_versions(self.runtime.pymodule)
t = PrettyTable()
t.add_column('name', libs)
'worker_procs': args.__max_jobs__,
}
if args.__construct__ == "none":
settings['sig_mode'] = "force"
# Get mapped IO database
settings['verbosity'] = args.verbosity if args.debug else 1
status = execute_workflow(script_prepare,
workflow='deploy',
options=settings)
env.verbosity = args.verbosity
if args.__construct__ == "existing":
settings['sig_mode'] = "build"
if args.__construct__ == "lenient":
settings['sig_mode'] = "skip"
# Get DSC meta database
env.logger.info("Building DSC database ...")
status = execute_workflow(script_prepare,
workflow='build',
options=settings)
if args.__construct__ == "all":
return
# Get the executed pipeline
pipeline.filter_execution(args.debug)
script_run = pipeline.get_pipeline("run", args.debug)
if args.debug:
if args.host:
import yaml
yaml.safe_dump(conf_tpl,
open(f'{DSC_CACHE}/{db}_remote_config.yml', 'w'),
default_flow_style=False)
return
env.logger.debug(f"Running command ``{' '.join(sys.argv)}``")