Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
}}
'''
# temporarily change the run mode to run to execute script
try:
with open(script_file, 'w') as sfile:
sfile.write(install_script)
#
p = subprocess.Popen(
['Rscript', '--default-packages=utils', script_file])
ret = p.wait()
if ret != 0:
env.logger.warning(
f'Failed to detect or install R library {name}')
return False
except Exception as e:
env.logger.error(f'Failed to execute script: {e}')
return False
finally:
os.remove(script_file)
ret_val = False
with open(output_file) as tmp:
for line in tmp:
lib, cur_version, status = line.split(' ', 2)
if status.strip() == "MISSING":
env.logger.error(
f'R library {lib} is not available and cannot be installed.'
)
elif status.strip() == "UNAVAILABLE":
env.logger.error(f'R library {lib} is not available.')
elif status.strip() == 'AVAILABLE':
env.logger.debug(
version_script += '''
if (cur_version {1} {0}) {{
quit("no")
}}
'''.format(repr(x), y)
version_script += 'write(paste(package, cur_version, "VERSION_MISMATCH"), file = {})'.\
format(repr(output_file))
# temporarily change the run mode to run to execute script
try:
with open(script_file, 'w') as sfile:
sfile.write(install_script + version_script)
#
p = subprocess.Popen(['Rscript', '--default-packages=utils', script_file])
ret = p.wait()
if ret != 0:
env.logger.warning('Failed to detect or install R library')
return False
except Exception as e:
env.logger.error('Failed to execute script: {}'.format(e))
return False
finally:
os.remove(script_file)
ret_val = False
with open(output_file) as tmp:
for line in tmp:
lib, version, status = line.split()
if status.strip() == "MISSING":
env.logger.warning('R Library {} is not available and cannot be installed.'.format(lib))
elif status.strip() == 'AVAILABLE':
env.logger.debug('R library {} ({}) is available'.format(lib, version))
ret_val = True
except Exception as e:
stdout_socket.send_multipart([b'ERROR', str(e).encode()])
informer_socket.send_pyobj({
'msg_type': 'workflow_status',
'data': {
'cell_id': env.config['slave_id'],
'status': 'failed',
'exception': str(e)
}
})
sys.exit(1)
finally:
try:
os.remove(filename)
except Exception as e:
env.logger.warning(
f'Failed to remove temp script {filename}: {e}')
stdout_socket.LINGER = 0
stdout_socket.close()
informer_socket.LINGER = 0
informer_socket.close()
context.term()
if runtime['run_mode'] == 'dryrun':
try:
cmd = 'bash ~/.sos/tasks/{}.sh'.format(task_id)
print(self.agent.check_output(cmd))
except Exception as e:
raise RuntimeError('Failed to submit task {}: {}'.format(task_id, e))
else:
#
# now we need to figure out a command to submit the task
try:
cmd = cfg_interpolate(self.submit_cmd, runtime)
except Exception as e:
raise ValueError('Failed to generate job submission command from template "{}": {}'.format(
self.submit_cmd, e))
env.logger.debug('submit {}: {}'.format(task_id, cmd))
try:
try:
cmd_output = self.agent.check_output(cmd).strip()
except Exception as e:
raise RuntimeError('Failed to submit task {}: {}'.format(task_id, e))
if 'submit_cmd_output' not in self.config:
submit_cmd_output = '{job_id}'
else:
submit_cmd_output = self.config['submit_cmd_output']
#
if not '{job_id}' in submit_cmd_output:
raise ValueError('Option submit_cmd_output should have at least a pattern for job_id, "{}" specified.'.format(submit_cmd_output))
#
# try to extract job_id from command output
# let us write an job_id file so that we can check status of tasks more easily
def _is_image_avail(self, image):
# the command will return ID of the image if it exists
try:
return bool(
subprocess.check_output(
f'''Singularity images {image} --no-trunc --format "{{{{.ID}}}}"''',
shell=True))
except Exception as e:
env.logger.warning(f'Failed to check image {image}: {e}')
return False
def cancel_workflow(cell_id, kernel):
global g_workflow_queue
env.logger.info(f'A queued or running workflow in this cell is canceled')
kernel.send_frontend_msg('workflow_status', {
'cell_id': cell_id,
'status': 'purged'
})
for idx, (cid, proc) in enumerate(g_workflow_queue):
if cid != cell_id or proc is None:
continue
if not isinstance(proc, tuple) and (proc.is_alive() and
psutil.pid_exists(proc.pid)):
from sos.executor_utils import kill_all_subprocesses
kill_all_subprocesses(proc.pid, include_self=True)
proc.terminate()
if psutil.pid_exists(proc.pid):
raise RuntimeError('Failed to kill workflow')
g_workflow_queue[idx][1] = None
Convert a ipython notebook to sos format.
'''
if unknown_args:
raise ValueError('Unrecognized parameter {}'.format(unknown_args))
if args:
exporter = SoS_Exporter(export_all=args.__all__)
else:
exporter = SoS_Exporter()
notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
output, _ = exporter.from_notebook_node(notebook, {})
if not sos_file:
sys.stdout.write(output)
elif isinstance(sos_file, str):
with open(sos_file, 'w') as sos:
sos.write(output)
env.logger.info('SoS script saved to {}'.format(sos_file))
else:
sos_file.write(output)
return
options, remaining_code = self.get_magic_and_code(code, True)
try:
self.sos_kernel.options = options
try:
if sys.platform == 'darwin':
try:
code = osx_clipboard_get()
except Exception:
code = tkinter_clipboard_get()
else:
code = tkinter_clipboard_get()
except ClipboardEmpty:
raise UsageError("The clipboard appears to be empty")
except Exception as e:
env.logger.warn(f'Failed to get text from the clipboard: {e}')
return
#
self.sos_kernel.send_response(
self.sos_kernel.iopub_socket, 'stream', {
'name': 'stdout',
'text': code.strip() + '\n## -- End pasted text --\n'
})
return self.sos_kernel._do_execute(code, silent, store_history,
user_expressions, allow_stdin)
finally:
self.sos_kernel.options = ''
def skip(self, section):
if section.global_def:
try:
SoS_exec(section.global_def, section.global_sigil)
except RuntimeError as e:
if env.verbosity > 2:
sys.stderr.write(get_traceback())
raise RuntimeError('Failed to execute statements\n"{}"\n{}'.format(
section.global_def, e))
#
if 'skip' in section.options:
val_skip = section.options['skip']
if val_skip is None or val_skip is True:
env.logger.info('Step ``{}`` is ``ignored`` due to skip option.'.format(section.step_name()))
return True
elif val_skip is not False:
raise RuntimeError('The value of section option skip can only be None, True or False, {} provided'.format(val_skip))
return False
# this is the result returned by the workflow, if the
# last stement is an expression.
last_res = None
# process step of the pipelinp
if isinstance(targets, str):
targets = [targets]
dag = self.initialize_dag(targets=targets)
#
# if targets are specified and there are only signatures for them, we need
# to remove the signature and really generate them
if targets:
for t in targets:
if not FileTarget(t).exists('target') and FileTarget(t).exists('signature'):
env.logger.debug('Re-generating {}'.format(t))
FileTarget(t).remove('signature')
else:
env.logger.debug('Target {} already exists'.format(t))
#
while True:
# find any step that can be executed and run it, and update the DAT
# with status.
runnable = dag.find_executable()
if runnable is None:
# no runnable
#dag.show_nodes()
break
# find the section from runnable
section = self.workflow.section_by_id(runnable._step_uuid)
#
# this is to keep compatibility of dag run with sequential run because