How to use the sos.utils.env.logger function in sos

To help you get started, we’ve selected a few sos examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github vatlab / sos / src / sos / targets_r.py View on Github external
}}
            '''
        # temporarily change the run mode to run to execute script
        try:
            with open(script_file, 'w') as sfile:
                sfile.write(install_script)
            #
            p = subprocess.Popen(
                ['Rscript', '--default-packages=utils', script_file])
            ret = p.wait()
            if ret != 0:
                env.logger.warning(
                    f'Failed to detect or install R library {name}')
                return False
        except Exception as e:
            env.logger.error(f'Failed to execute script: {e}')
            return False
        finally:
            os.remove(script_file)

        ret_val = False
        with open(output_file) as tmp:
            for line in tmp:
                lib, cur_version, status = line.split(' ', 2)
                if status.strip() == "MISSING":
                    env.logger.error(
                        f'R library {lib} is not available and cannot be installed.'
                    )
                elif status.strip() == "UNAVAILABLE":
                    env.logger.error(f'R library {lib} is not available.')
                elif status.strip() == 'AVAILABLE':
                    env.logger.debug(
github vatlab / sos / src / sos / R / target.py View on Github external
version_script += '''
                if (cur_version {1} {0}) {{
                  quit("no")
                }}
                '''.format(repr(x), y)
            version_script += 'write(paste(package, cur_version, "VERSION_MISMATCH"), file = {})'.\
              format(repr(output_file))
        # temporarily change the run mode to run to execute script
        try:
            with open(script_file, 'w') as sfile:
                sfile.write(install_script + version_script)
            #
            p = subprocess.Popen(['Rscript', '--default-packages=utils', script_file])
            ret = p.wait()
            if ret != 0:
                env.logger.warning('Failed to detect or install R library')
                return False
        except Exception as e:
            env.logger.error('Failed to execute script: {}'.format(e))
            return False
        finally:
            os.remove(script_file)

        ret_val = False
        with open(output_file) as tmp:
            for line in tmp:
                lib, version, status = line.split()
                if status.strip() == "MISSING":
                    env.logger.warning('R Library {} is not available and cannot be installed.'.format(lib))
                elif status.strip() == 'AVAILABLE':
                    env.logger.debug('R library {} ({}) is available'.format(lib, version))
                    ret_val = True
github vatlab / sos-notebook / src / sos_notebook / workflow_executor.py View on Github external
except Exception as e:
            stdout_socket.send_multipart([b'ERROR', str(e).encode()])
            informer_socket.send_pyobj({
                'msg_type': 'workflow_status',
                'data': {
                    'cell_id': env.config['slave_id'],
                    'status': 'failed',
                    'exception': str(e)
                }
            })
            sys.exit(1)
        finally:
            try:
                os.remove(filename)
            except Exception as e:
                env.logger.warning(
                    f'Failed to remove temp script {filename}: {e}')
            stdout_socket.LINGER = 0
            stdout_socket.close()
            informer_socket.LINGER = 0
            informer_socket.close()
            context.term()
github vatlab / sos / src / sos / PBS / sos_task.py View on Github external
if runtime['run_mode'] == 'dryrun':
            try:
                cmd = 'bash ~/.sos/tasks/{}.sh'.format(task_id)
                print(self.agent.check_output(cmd))
            except Exception as e:
                raise RuntimeError('Failed to submit task {}: {}'.format(task_id, e))
        else:
            #
            # now we need to figure out a command to submit the task
            try:
                cmd = cfg_interpolate(self.submit_cmd, runtime)
            except Exception as e:
                raise ValueError('Failed to generate job submission command from template "{}": {}'.format(
                    self.submit_cmd, e))
            env.logger.debug('submit {}: {}'.format(task_id, cmd))
            try:
                try:
                    cmd_output = self.agent.check_output(cmd).strip()
                except Exception as e:
                    raise RuntimeError('Failed to submit task {}: {}'.format(task_id, e))

                if 'submit_cmd_output' not in self.config:
                    submit_cmd_output = '{job_id}'
                else:
                    submit_cmd_output = self.config['submit_cmd_output']
                #
                if not '{job_id}' in submit_cmd_output:
                    raise ValueError('Option submit_cmd_output should have at least a pattern for job_id, "{}" specified.'.format(submit_cmd_output))
                #
                # try to extract job_id from command output
                # let us write an job_id file so that we can check status of tasks more easily
github vatlab / sos / src / sos / singularity / client.py View on Github external
def _is_image_avail(self, image):
        # the command will return ID of the image if it exists
        try:
            return bool(
                subprocess.check_output(
                    f'''Singularity images {image} --no-trunc --format "{{{{.ID}}}}"''',
                    shell=True))
        except Exception as e:
            env.logger.warning(f'Failed to check image {image}: {e}')
            return False
github vatlab / sos-notebook / src / sos_notebook / workflow_executor.py View on Github external
def cancel_workflow(cell_id, kernel):
    global g_workflow_queue
    env.logger.info(f'A queued or running workflow in this cell is canceled')
    kernel.send_frontend_msg('workflow_status', {
        'cell_id': cell_id,
        'status': 'purged'
    })

    for idx, (cid, proc) in enumerate(g_workflow_queue):
        if cid != cell_id or proc is None:
            continue
        if not isinstance(proc, tuple) and (proc.is_alive() and
                                            psutil.pid_exists(proc.pid)):
            from sos.executor_utils import kill_all_subprocesses
            kill_all_subprocesses(proc.pid, include_self=True)
            proc.terminate()
            if psutil.pid_exists(proc.pid):
                raise RuntimeError('Failed to kill workflow')
        g_workflow_queue[idx][1] = None
github vatlab / sos / src / sos / jupyter / converter.py View on Github external
Convert a ipython notebook to sos format.
    '''
    if unknown_args:
        raise ValueError('Unrecognized parameter {}'.format(unknown_args))
    if args:
        exporter = SoS_Exporter(export_all=args.__all__)
    else:
        exporter = SoS_Exporter()
    notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
    output, _ = exporter.from_notebook_node(notebook, {})
    if not sos_file:
        sys.stdout.write(output)
    elif isinstance(sos_file, str):
        with open(sos_file, 'w') as sos:
            sos.write(output)
        env.logger.info('SoS script saved to {}'.format(sos_file))
    else:
        sos_file.write(output)
github vatlab / sos-notebook / src / sos_notebook / magics.py View on Github external
return
        options, remaining_code = self.get_magic_and_code(code, True)
        try:
            self.sos_kernel.options = options
            try:
                if sys.platform == 'darwin':
                    try:
                        code = osx_clipboard_get()
                    except Exception:
                        code = tkinter_clipboard_get()
                else:
                    code = tkinter_clipboard_get()
            except ClipboardEmpty:
                raise UsageError("The clipboard appears to be empty")
            except Exception as e:
                env.logger.warn(f'Failed to get text from the clipboard: {e}')
                return
            #
            self.sos_kernel.send_response(
                self.sos_kernel.iopub_socket, 'stream', {
                    'name': 'stdout',
                    'text': code.strip() + '\n## -- End pasted text --\n'
                })
            return self.sos_kernel._do_execute(code, silent, store_history,
                                               user_expressions, allow_stdin)
        finally:
            self.sos_kernel.options = ''
github vatlab / sos / sos / sos_executor.py View on Github external
def skip(self, section):
        if section.global_def:
            try:
                SoS_exec(section.global_def, section.global_sigil)
            except RuntimeError as e:
                if env.verbosity > 2:
                    sys.stderr.write(get_traceback())
                raise RuntimeError('Failed to execute statements\n"{}"\n{}'.format(
                    section.global_def, e))
        #
        if 'skip' in section.options:
            val_skip = section.options['skip']
            if val_skip is None or val_skip is True:
                env.logger.info('Step ``{}`` is ``ignored`` due to skip option.'.format(section.step_name()))
                return True
            elif val_skip is not False:
                raise RuntimeError('The value of section option skip can only be None, True or False, {} provided'.format(val_skip))
        return False
github vatlab / sos / src / sos / jupyter / sos_executor.py View on Github external
# this is the result returned by the workflow, if the
        # last stement is an expression.
        last_res = None

        # process step of the pipelinp
        if isinstance(targets, str):
            targets = [targets]
        dag = self.initialize_dag(targets=targets)
        #
        # if targets are specified and there are only signatures for them, we need
        # to remove the signature and really generate them
        if targets:
            for t in targets:
                if not FileTarget(t).exists('target') and FileTarget(t).exists('signature'):
                    env.logger.debug('Re-generating {}'.format(t))
                    FileTarget(t).remove('signature')
                else:
                    env.logger.debug('Target {} already exists'.format(t))
        #
        while True:
            # find any step that can be executed and run it, and update the DAT
            # with status.
            runnable = dag.find_executable()
            if runnable is None:
                # no runnable
                #dag.show_nodes()
                break
            # find the section from runnable
            section = self.workflow.section_by_id(runnable._step_uuid)
            #
            # this is to keep compatibility of dag run with sequential run because