Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
3: logging.DEBUG,
4: logging.DEBUG,
None: logging.INFO
}
env.logger.addHandler(
NotebookLoggingHandler(
levels[env.verbosity], kernel, title=' '.join(sys.argv)))
else:
env.logger.handers[0].setTitle(' '.join(sys.argv))
global last_cell_id
# we retain step_input etc only when we step through a cell #256
if kernel and kernel.cell_id != last_cell_id:
# clear __step_input__, __step_output__ etc because there is
# no concept of passing input/outputs across cells.
env.sos_dict.set('__step_output__', sos_targets([]))
for k in [
'__step_input__', '__default_output__', 'step_input',
'step_output', 'step_depends', '_input', '_output', '_depends'
]:
env.sos_dict.pop(k, None)
last_cell_id = kernel.cell_id
config = {
'config_file':
args.__config__,
'default_queue':
args.__queue__,
'run_mode':
'dryrun' if args.dryrun else 'interactive',
# issue 230, ignore sig mode in interactive mode
os.getcwd(),
'workflow':
args.workflow,
'targets':
args.__targets__,
'workflow_args':
workflow_args,
'workflow_id':
textMD5(code),
# interactive work is also a slave of the controller
'slave_id':
kernel.cell_id,
}
env.sos_dict.set('workflow_id', config['workflow_id'])
env.config.update(config)
try:
if not any([
SOS_SECTION_HEADER.match(line) or line.startswith('%from') or
line.startswith('%include') for line in code.splitlines()
]):
code = '[scratch_0]\n' + code
script = SoS_Script(content=code)
else:
return
workflow = script.workflow(args.workflow)
section = workflow.sections[0]
res = analyze_section(section)
env.sos_dict.quick_update({
'__signature_vars__': res['signature_vars'],
raise RuntimeError('Circular dependency detected {}. It is likely a later step produces input of a previous step.'.format(cycle))
elif isinstance(res, RemovedTarget):
runnable._status = None
dag.regenerate_target(res.target)
elif isinstance(res, UnavailableLock):
runnable._status = 'pending'
runnable._signature = (res.output, res.sig_file)
env.logger.info('Waiting on another process for step {}'.format(section.step_name()))
# if the job is failed
elif isinstance(res, Exception):
runnable._status = 'failed'
exec_error.append(runnable._node_id, res)
prog.progress(1)
else:#
for k, v in res.items():
env.sos_dict.set(k, v)
#
# set context to the next logic step.
for edge in dag.out_edges(runnable):
node = edge[1]
# if node is the logical next step...
if node._node_index is not None and runnable._node_index is not None:
#and node._node_index == runnable._node_index + 1:
node._context.update(env.sos_dict.clone_selected_vars(
node._context['__signature_vars__'] | node._context['__environ_vars__'] \
| {'_input', '__step_output__', '__default_output__', '__args__'}))
runnable._status = 'completed'
prog.progress(1)
#env.logger.error('completed')
prog.done()
if exec_error.errors:
failed_steps, pending_steps = dag.pending()
if self.nested:
#
# if this is a nested workflow, we do not clear sos_dict because it contains all
# the symbols from the main workflow. _base_symbols need to be defined though.
self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
self._base_symbols -= {'dynamic'}
return
env.sos_dict = WorkflowDict()
env.parameter_vars.clear()
# inject a few things
env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))
env.sos_dict.set('__null_func__', __null_func__)
env.sos_dict.set('__args__', self.args)
env.sos_dict.set('__unknown_args__', self.args)
# initial values
env.sos_dict.set('SOS_VERSION', __version__)
env.sos_dict.set('__step_output__', [])
# load configuration files
cfg = {}
sos_config_file = os.path.join(os.path.expanduser('~'), '.sos', 'config.yaml')
if os.path.isfile(sos_config_file):
try:
with open(sos_config_file) as config:
cfg = yaml.safe_load(config)
except Exception as e:
raise RuntimeError('Failed to parse global sos config file {}, is it in YAML/JSON format? ({})'.format(sos_config_file, e))
# local config file
sos_config_file = 'config.yaml'
if os.path.isfile(sos_config_file):
def _reset(self):
env.sos_dict = WorkflowDict()
SoS_exec('from sos.runtime import *', None)
env.sos_dict.set('__interactive__', True)
self.original_keys = set(env.sos_dict._dict.keys())
self.original_keys.add('__builtins__')
self.options = ''
# the symbols from the main workflow. _base_symbols need to be defined though.
self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
self._base_symbols -= {'dynamic'}
return
env.sos_dict = WorkflowDict()
env.parameter_vars.clear()
# inject a few things
env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))
env.sos_dict.set('__null_func__', __null_func__)
env.sos_dict.set('__args__', self.args)
env.sos_dict.set('__unknown_args__', self.args)
# initial values
env.sos_dict.set('SOS_VERSION', __version__)
env.sos_dict.set('__step_output__', [])
# load configuration files
cfg = {}
sos_config_file = os.path.join(os.path.expanduser('~'), '.sos', 'config.yaml')
if os.path.isfile(sos_config_file):
try:
with open(sos_config_file) as config:
cfg = yaml.safe_load(config)
except Exception as e:
raise RuntimeError('Failed to parse global sos config file {}, is it in YAML/JSON format? ({})'.format(sos_config_file, e))
# local config file
sos_config_file = 'config.yaml'
if os.path.isfile(sos_config_file):
try:
with open(sos_config_file) as config:
dict_merge(cfg, yaml.safe_load(config))
def reset_dict(self):
env.sos_dict.set('__null_func__', __null_func__)
env.sos_dict.set('SOS_VERSION', __version__)
env.sos_dict.set('__args__', self.args)
if self.md5:
env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))
self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
self._base_symbols -= {'dynamic'}
# load configuration files
cfg = load_config_files(self.config['config_file'])
# if check_readonly is set to True, allow checking readonly vars
if cfg.get('sos', {}).get('change_all_cap_vars', None) is not None:
if cfg['sos']['change_all_cap_vars'] not in ('warning', 'error'):
env.logger.error('Configuration sos.change_all_cap_vars can only be warning or error: {} provided'.format(cfg['sos']['change_all_cap_vars']))
else:
env.sos_dict._change_all_cap_vars = cfg['sos']['change_all_cap_vars']
)
#
if args.__to__ and not args.__to__.isidentifier():
self.sos_kernel.warn(f'Invalid variable name {args.__to__}')
self.sos_kernel._meta['capture_result'] = None
return
if args.__append__ and not args.__append__.isidentifier():
self.sos_kernel.warn(f'Invalid variable name {args.__append__}')
self.sos_kernel._meta['capture_result'] = None
return
if args.__to__:
env.sos_dict.set(args.__to__, content)
elif args.__append__:
if args.__append__ not in env.sos_dict:
env.sos_dict.set(args.__append__, content)
elif isinstance(env.sos_dict[args.__append__], str):
if isinstance(content, str):
env.sos_dict[args.__append__] += content
else:
self.sos_kernel.warn(
f'Cannot append new content of type {type(content).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}'
)
elif isinstance(env.sos_dict[args.__append__], dict):
if isinstance(content, dict):
env.sos_dict[args.__append__].update(content)
else:
self.sos_kernel.warn(
f'Cannot append new content of type {type(content).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}'
)
elif isinstance(env.sos_dict[args.__append__], pd.DataFrame):
if isinstance(content, pd.DataFrame):
# and functions
if self.nested:
#
# if this is a nested workflow, we do not clear sos_dict because it contains all
# the symbols from the main workflow. _base_symbols need to be defined though.
self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
self._base_symbols -= {'dynamic'}
return
env.sos_dict = WorkflowDict()
env.parameter_vars.clear()
# inject a few things
env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))
env.sos_dict.set('__null_func__', __null_func__)
env.sos_dict.set('__args__', self.args)
env.sos_dict.set('__unknown_args__', self.args)
# initial values
env.sos_dict.set('SOS_VERSION', __version__)
env.sos_dict.set('__step_output__', [])
# load configuration files
cfg = {}
sos_config_file = os.path.join(os.path.expanduser('~'), '.sos', 'config.yaml')
if os.path.isfile(sos_config_file):
try:
with open(sos_config_file) as config:
cfg = yaml.safe_load(config)
except Exception as e:
raise RuntimeError('Failed to parse global sos config file {}, is it in YAML/JSON format? ({})'.format(sos_config_file, e))
# local config file
sos_config_file = 'config.yaml'
def init_input_output_vars(self):
# we keep these variables (which can be result of stepping through previous statements)
# if no input and/or output statement is defined
for key in ('step_input', '_depends', 'step_output', 'step_depends',
'_depends'):
if key not in env.sos_dict:
env.sos_dict.set(key, sos_targets([]))
if '_output' not in env.sos_dict:
env.sos_dict.set('_output', sos_targets(_undetermined=True))
if any(x[0] == ':' and x[1] == 'input' for x in self.step.statements):
env.sos_dict.set('step_input', sos_targets([]))
env.sos_dict.set('_input', sos_targets([]))
if any(x[0] == ':' and x[1] == 'output' for x in self.step.statements):
env.sos_dict.set('step_output', sos_targets([]))
env.sos_dict.set('_output', sos_targets([]))
env.sos_dict.pop('__default_output__', None)