Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
yaml_name = yaml_source.source_id
for key, val in list(user_inputs.items()):
LOGGER.debug('overriding:key={}, file={}'.format(key, yaml_name))
tags = key.split('.')
if key.startswith('inputs.default'):
# change value in inputs
if tags[-1] in list(self.user_overrides.keys()):
self.user_overrides[tags[-1]] = val
elif tags[-1] in list(self.extra_user_overrides.keys()):
self.extra_user_overrides[tags[-1]] = val
else:
msg = 'key not found in default inputs:key={}, file={}'
msg = msg.format(tags[-1], yaml_name)
LOGGER.error(msg)
raise AutoProcessorError(msg)
elif key.startswith('inputs.xnat'):
# change value in self.xnat_inputs
if tags[2] not in list(self.xnat_inputs.keys()):
msg = 'key not found in xnat inputs:key={}, file={}'
msg = msg.format(tags[3], yaml_name)
LOGGER.error(msg)
raise AutoProcessorError(msg)
# Match the scan number or assessor number (e.g: scan1)
sobj = None
for obj in self.xnat_inputs[tags[2]]:
if tags[3] == obj['name']:
sobj = obj
break
_val = ','.join(
[assr.parent().scan(r).attrs.get(_attr) for r in _refval])
elif _obj == 'assessor':
if 'ref' in attr_in:
_ref = attr_in['ref']
_refval = [a.rsplit('/', 1)[1] for a in assr_inputs[_ref]]
_val = ','.join(
[assr.parent().assessor(r).attrs.get(_attr) for r in _refval])
else:
print(_attr)
print((assr.label()))
_val = assr.attrs.get(_attr)
else:
LOGGER.error('invalid YAML')
err = 'YAML File:contains invalid attribute:{}'
raise AutoProcessorError(err.format(_attr))
if _val == '':
raise NeedInputsException('Missing ' + _attr)
else:
var2val[_var] = _val
# Handle edits
edit_res = assr.out_resource(task.EDITS_RESOURCE)
if edit_res.exists():
file_list = edit_res.files().get()
assr_path = '/projects/{}/subjects/{}/experiments/{}/assessors/{}'.format(
assr.parent().parent().parent().label(),
assr.parent().parent().label(),
assr.parent().label(),
assr.label()
)
def _raise_yaml_error_if_no_key(doc, source_id, key):
"""Method to raise an execption if the key is not in the dict
:param doc: dict to check
:param source_id: YAML source identifier string for logging
:param key: key to search
"""
if key not in list(doc.keys()):
err = 'YAML source {} does not have {} defined. See example.'
raise AutoProcessorError(err.format(source_id, key))
LOGGER.error(msg)
raise AutoProcessorError(msg)
if tags[4] == 'resources':
if tags[6] == 'fmatch':
# Match the resource name
robj = None
for obj in sobj['resources']:
if tags[5] == obj['varname']:
robj = obj
break
if robj is None:
msg = 'invalid override:key={}, file={}'
LOGGER.error(msg)
raise AutoProcessorError(msg)
msg = 'overriding fmatch:key={}, val={}'
msg = msg.format(key, val)
LOGGER.debug(msg)
robj['fmatch'] = val
else:
msg = 'invalid override:key={}, file={}'
msg = msg.format(key, yaml_name)
LOGGER.error(msg)
raise AutoProcessorError(msg)
else:
LOGGER.info('overriding:{}:{}'.format(tags[4], str(val)))
obj[tags[4]] = val
elif key.startswith('attrs'):
# change value in self.attrs
def __init__(self, xnat, yaml_source, user_inputs=None):
"""
Entry point for the auto processor
:param xnat: xnat context object (XnatUtils in production contexts)
:param yaml_source: dictionary containing source_type -> string,\
source_id -> string, document -> yaml document
:param user_inputs: a dictionary of user overrides to the yaml\
source document
:return: None
"""
if not xnat:
raise AutoProcessorError("Parameter 'xnat' must be provided")
if not yaml_source:
raise AutoProcessorError(
"Parameter 'yaml_source' must be provided")
self.xnat = xnat
self.user_overrides = dict()
self.extra_user_overrides = dict()
self._read_yaml(yaml_source)
# Edit the values from user inputs:
if user_inputs is not None:
self._edit_inputs(user_inputs, yaml_source)
self.parser = processor_parser.ProcessorParser(
yaml_source.contents, self.proctype)
# Set up attrs:
def _read_yaml(self, yaml_source):
"""
Method to read the processor arguments and there default value.
:param yaml_file: path to yaml file defining the processor
"""
if yaml_source.source_type is None:
raise AutoProcessorError('Empty yaml source provided')
doc = yaml_source.contents
# Set Inputs from Yaml
self._check_default_keys(yaml_source.source_id, doc)
self.attrs = doc.get('attrs')
self.command = doc.get('command')
# Set Inputs from Yaml
inputs = doc.get('inputs')
self.xnat_inputs = inputs.get('xnat')
for key, value in list(inputs.get('default').items()):
# If value is a key in command
k_str = '{{{}}}'.format(key)
if k_str in self.command:
self.user_overrides[key] = value
self.run_update(settings_path, log)
except AutoProcessorError as e:
err = 'error running update:project={}\n{}'.format(proj, e)
LOGGER.error(err)
run_errors.append(err)
# Launch - report to log if locked
LOGGER.info('launching')
for settings_path in self.settings_list:
try:
proj = self.project_from_settings(settings_path)
LOGGER.info('launching jobs:' + proj)
log = self.log_name('launch', proj, datetime.now())
self.run_launch(settings_path, log)
except AutoProcessorError as e:
err = 'error running launch:project={}\n{}'.format(proj, e)
LOGGER.error(err)
run_errors.append(err)
# Upload - report to log if locked
log = self.log_name('upload', 'upload', datetime.now())
upload_process = Process(
target=self.run_upload,
args=(log,))
LOGGER.info('starting upload')
upload_process.start()
LOGGER.info('waiting for upload')
upload_process.join()
LOGGER.info('upload complete')
if num_build_threads > 0:
def __init__(self, xnat, yaml_source, user_inputs=None):
"""
Entry point for the auto processor
:param xnat: xnat context object (XnatUtils in production contexts)
:param yaml_source: dictionary containing source_type -> string,\
source_id -> string, document -> yaml document
:param user_inputs: a dictionary of user overrides to the yaml\
source document
:return: None
"""
if not xnat:
raise AutoProcessorError("Parameter 'xnat' must be provided")
if not yaml_source:
raise AutoProcessorError(
"Parameter 'yaml_source' must be provided")
self.xnat = xnat
self.user_overrides = dict()
self.extra_user_overrides = dict()
self._read_yaml(yaml_source)
# Edit the values from user inputs:
if user_inputs is not None:
self._edit_inputs(user_inputs, yaml_source)
self.parser = processor_parser.ProcessorParser(
yaml_source.contents, self.proctype)
Check if a file exists and if it's a python file
:param filepath: path to the file to test
:return: True the file pass the test, False otherwise
"""
if not os.path.exists(filepath):
print(('[ERROR] %s does not exists.' % filepath))
return None
if filepath.endswith('yaml'):
doc = utilities.read_yaml(filepath)
if 'projects' in list(doc.keys()):
try:
return bin.read_yaml_settings(filepath, LOGGER)
except AutoProcessorError:
print('[ERROR]')
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stdout)
else:
# So far only auto processor:
try:
return processors.load_from_yaml(XnatUtils, filepath)
except AutoProcessorError:
print('[ERROR]')
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stdout)
elif filepath.endswith('.py') or is_python_file(filepath):
test = imp.load_source('test', filepath)
# Check if processor file