How to use the sos.utils.env.sos_dict.set function in sos

To help you get started, we’ve selected a few sos examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github vatlab / sos-notebook / src / sos_notebook / workflow_executor.py View on Github external
3: logging.DEBUG,
            4: logging.DEBUG,
            None: logging.INFO
        }
        env.logger.addHandler(
            NotebookLoggingHandler(
                levels[env.verbosity], kernel, title=' '.join(sys.argv)))
    else:
        env.logger.handers[0].setTitle(' '.join(sys.argv))

    global last_cell_id
    # we retain step_input etc only when we step through a cell  #256
    if kernel and kernel.cell_id != last_cell_id:
        # clear __step_input__, __step_output__ etc because there is
        # no concept of passing input/outputs across cells.
        env.sos_dict.set('__step_output__', sos_targets([]))
        for k in [
                '__step_input__', '__default_output__', 'step_input',
                'step_output', 'step_depends', '_input', '_output', '_depends'
        ]:
            env.sos_dict.pop(k, None)

    last_cell_id = kernel.cell_id

    config = {
        'config_file':
            args.__config__,
        'default_queue':
            args.__queue__,
        'run_mode':
            'dryrun' if args.dryrun else 'interactive',
        # issue 230, ignore sig mode in interactive mode
github vatlab / sos-notebook / src / sos_notebook / workflow_executor.py View on Github external
os.getcwd(),
        'workflow':
            args.workflow,
        'targets':
            args.__targets__,
        'workflow_args':
            workflow_args,
        'workflow_id':
            textMD5(code),

        # interactive work is also a slave of the controller
        'slave_id':
            kernel.cell_id,
    }

    env.sos_dict.set('workflow_id', config['workflow_id'])
    env.config.update(config)

    try:
        if not any([
                SOS_SECTION_HEADER.match(line) or line.startswith('%from') or
                line.startswith('%include') for line in code.splitlines()
        ]):
            code = '[scratch_0]\n' + code
            script = SoS_Script(content=code)
        else:
            return
        workflow = script.workflow(args.workflow)
        section = workflow.sections[0]
        res = analyze_section(section)
        env.sos_dict.quick_update({
            '__signature_vars__': res['signature_vars'],
github vatlab / sos / sos / sos_executor.py View on Github external
raise RuntimeError('Circular dependency detected {}. It is likely a later step produces input of a previous step.'.format(cycle))
            elif isinstance(res, RemovedTarget):
                runnable._status = None
                dag.regenerate_target(res.target)
            elif isinstance(res, UnavailableLock):
                runnable._status = 'pending'
                runnable._signature = (res.output, res.sig_file)
                env.logger.info('Waiting on another process for step {}'.format(section.step_name()))
            # if the job is failed
            elif isinstance(res, Exception):
                runnable._status = 'failed'
                exec_error.append(runnable._node_id, res)
                prog.progress(1)
            else:#
                for k, v in res.items():
                    env.sos_dict.set(k, v)
                #
                # set context to the next logic step.
                for edge in dag.out_edges(runnable):
                    node = edge[1]
                    # if node is the logical next step...
                    if node._node_index is not None and runnable._node_index is not None:
                        #and node._node_index == runnable._node_index + 1:
                        node._context.update(env.sos_dict.clone_selected_vars(
                            node._context['__signature_vars__'] | node._context['__environ_vars__'] \
                            | {'_input', '__step_output__', '__default_output__', '__args__'}))
                runnable._status = 'completed'
                prog.progress(1)
            #env.logger.error('completed')
        prog.done()
        if exec_error.errors:
            failed_steps, pending_steps = dag.pending()
github vatlab / sos / sos / sos_executor.py View on Github external
if self.nested:
            #
            # if this is a nested workflow, we do not clear sos_dict because it contains all
            # the symbols from the main workflow. _base_symbols need to be defined though.
            self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
            self._base_symbols -= {'dynamic'}
            return

        env.sos_dict = WorkflowDict()
        env.parameter_vars.clear()

        # inject a few things
        env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))
        env.sos_dict.set('__null_func__', __null_func__)
        env.sos_dict.set('__args__', self.args)
        env.sos_dict.set('__unknown_args__', self.args)
        # initial values
        env.sos_dict.set('SOS_VERSION', __version__)
        env.sos_dict.set('__step_output__', [])

        # load configuration files
        cfg = {}
        sos_config_file = os.path.join(os.path.expanduser('~'), '.sos', 'config.yaml')
        if os.path.isfile(sos_config_file):
            try:
                with open(sos_config_file) as config:
                    cfg = yaml.safe_load(config)
            except Exception as e:
                raise RuntimeError('Failed to parse global sos config file {}, is it in YAML/JSON format? ({})'.format(sos_config_file, e))
        # local config file
        sos_config_file = 'config.yaml'
        if os.path.isfile(sos_config_file):
github vatlab / sos / misc / ipython / ipython_magic.py View on Github external
def _reset(self):
        env.sos_dict = WorkflowDict()
        SoS_exec('from sos.runtime import *', None)
        env.sos_dict.set('__interactive__', True)
        self.original_keys = set(env.sos_dict._dict.keys())
        self.original_keys.add('__builtins__')
        self.options = ''
github vatlab / sos / sos / sos_executor.py View on Github external
# the symbols from the main workflow. _base_symbols need to be defined though.
            self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
            self._base_symbols -= {'dynamic'}
            return

        env.sos_dict = WorkflowDict()
        env.parameter_vars.clear()

        # inject a few things
        env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))
        env.sos_dict.set('__null_func__', __null_func__)
        env.sos_dict.set('__args__', self.args)
        env.sos_dict.set('__unknown_args__', self.args)
        # initial values
        env.sos_dict.set('SOS_VERSION', __version__)
        env.sos_dict.set('__step_output__', [])

        # load configuration files
        cfg = {}
        sos_config_file = os.path.join(os.path.expanduser('~'), '.sos', 'config.yaml')
        if os.path.isfile(sos_config_file):
            try:
                with open(sos_config_file) as config:
                    cfg = yaml.safe_load(config)
            except Exception as e:
                raise RuntimeError('Failed to parse global sos config file {}, is it in YAML/JSON format? ({})'.format(sos_config_file, e))
        # local config file
        sos_config_file = 'config.yaml'
        if os.path.isfile(sos_config_file):
            try:
                with open(sos_config_file) as config:
                    dict_merge(cfg, yaml.safe_load(config))
github vatlab / sos / src / sos / jupyter / sos_executor.py View on Github external
def reset_dict(self):
        env.sos_dict.set('__null_func__', __null_func__)
        env.sos_dict.set('SOS_VERSION', __version__)
        env.sos_dict.set('__args__', self.args)
        if self.md5:
            env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))

        self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
        self._base_symbols -= {'dynamic'}

        # load configuration files
        cfg = load_config_files(self.config['config_file'])
        # if check_readonly is set to True, allow checking readonly vars
        if cfg.get('sos', {}).get('change_all_cap_vars', None) is not None:
            if cfg['sos']['change_all_cap_vars'] not in ('warning', 'error'):
                env.logger.error('Configuration sos.change_all_cap_vars can only be warning or error: {} provided'.format(cfg['sos']['change_all_cap_vars']))
            else:
                env.sos_dict._change_all_cap_vars = cfg['sos']['change_all_cap_vars']
github vatlab / sos-notebook / src / sos_notebook / magics.py View on Github external
)
            #
            if args.__to__ and not args.__to__.isidentifier():
                self.sos_kernel.warn(f'Invalid variable name {args.__to__}')
                self.sos_kernel._meta['capture_result'] = None
                return
            if args.__append__ and not args.__append__.isidentifier():
                self.sos_kernel.warn(f'Invalid variable name {args.__append__}')
                self.sos_kernel._meta['capture_result'] = None
                return

            if args.__to__:
                env.sos_dict.set(args.__to__, content)
            elif args.__append__:
                if args.__append__ not in env.sos_dict:
                    env.sos_dict.set(args.__append__, content)
                elif isinstance(env.sos_dict[args.__append__], str):
                    if isinstance(content, str):
                        env.sos_dict[args.__append__] += content
                    else:
                        self.sos_kernel.warn(
                            f'Cannot append new content of type {type(content).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}'
                        )
                elif isinstance(env.sos_dict[args.__append__], dict):
                    if isinstance(content, dict):
                        env.sos_dict[args.__append__].update(content)
                    else:
                        self.sos_kernel.warn(
                            f'Cannot append new content of type {type(content).__name__} to {args.__append__} of type {type(env.sos_dict[args.__append__]).__name__}'
                        )
                elif isinstance(env.sos_dict[args.__append__], pd.DataFrame):
                    if isinstance(content, pd.DataFrame):
github vatlab / sos / sos / sos_executor.py View on Github external
# and functions
        if self.nested:
            #
            # if this is a nested workflow, we do not clear sos_dict because it contains all
            # the symbols from the main workflow. _base_symbols need to be defined though.
            self._base_symbols = set(dir(__builtins__)) | set(env.sos_dict['sos_symbols_']) | set(SOS_KEYWORDS) | set(keyword.kwlist)
            self._base_symbols -= {'dynamic'}
            return

        env.sos_dict = WorkflowDict()
        env.parameter_vars.clear()

        # inject a few things
        env.sos_dict.set('__workflow_sig__', os.path.join(env.exec_dir, '.sos', '{}.sig'.format(self.md5)))
        env.sos_dict.set('__null_func__', __null_func__)
        env.sos_dict.set('__args__', self.args)
        env.sos_dict.set('__unknown_args__', self.args)
        # initial values
        env.sos_dict.set('SOS_VERSION', __version__)
        env.sos_dict.set('__step_output__', [])

        # load configuration files
        cfg = {}
        sos_config_file = os.path.join(os.path.expanduser('~'), '.sos', 'config.yaml')
        if os.path.isfile(sos_config_file):
            try:
                with open(sos_config_file) as config:
                    cfg = yaml.safe_load(config)
            except Exception as e:
                raise RuntimeError('Failed to parse global sos config file {}, is it in YAML/JSON format? ({})'.format(sos_config_file, e))
        # local config file
        sos_config_file = 'config.yaml'
github vatlab / sos-notebook / src / sos_notebook / step_executor.py View on Github external
def init_input_output_vars(self):
        # we keep these variables (which can be result of stepping through previous statements)
        # if no input and/or output statement is defined
        for key in ('step_input', '_depends', 'step_output', 'step_depends',
                    '_depends'):
            if key not in env.sos_dict:
                env.sos_dict.set(key, sos_targets([]))
        if '_output' not in env.sos_dict:
            env.sos_dict.set('_output', sos_targets(_undetermined=True))
        if any(x[0] == ':' and x[1] == 'input' for x in self.step.statements):
            env.sos_dict.set('step_input', sos_targets([]))
            env.sos_dict.set('_input', sos_targets([]))
        if any(x[0] == ':' and x[1] == 'output' for x in self.step.statements):
            env.sos_dict.set('step_output', sos_targets([]))
            env.sos_dict.set('_output', sos_targets([]))
        env.sos_dict.pop('__default_output__', None)