Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def __build_osn_sample_workflow(self):
workflow_json = None
with open(os.path.join(self.data_path, 'sample_osn_workflow.json')) as fp:
workflow_json = json.load(fp)
self.assertIsNotNone(workflow_json)
tasks = []
tasks_to_ports = {}
for task in workflow_json['tasks']:
inputs = {}
for input_entry in task['inputs']:
if 'source' in input_entry:
inputs[input_entry['name']] = 'source:' + input_entry['source']
elif 'value' in input_entry:
inputs[input_entry['name']] = input_entry['value']
gbdx_task = Task(task['taskType'], **inputs)
gbdx_task.name = task['name']
self.assertIsNotNone(gbdx_task)
tasks.append(gbdx_task)
return Workflow(tasks, name=workflow_json['name'])
def Task(self, __task_name, **kwargs):
return gbdxtools.simpleworkflows.Task(__task_name, **kwargs)
def ingest_vectors(self, output_port_value):
''' append two required tasks to the given output to ingest to VS
'''
# append two tasks to self['definition']['tasks']
ingest_task = Task('IngestItemJsonToVectorServices')
ingest_task.inputs.items = output_port_value
ingest_task.impersonation_allowed = True
stage_task = Task('StageDataToS3')
stage_task.inputs.destination = 's3://{vector_ingest_bucket}/{recipe_id}/{run_id}/{task_name}'
stage_task.inputs.data = ingest_task.outputs.result.value
self.definition['tasks'].append(ingest_task.generate_task_workflow_json())
self.definition['tasks'].append(stage_task.generate_task_workflow_json())
def ingest_vectors(self, output_port_value):
''' append two required tasks to the given output to ingest to VS
'''
# append two tasks to self['definition']['tasks']
ingest_task = Task('IngestItemJsonToVectorServices')
ingest_task.inputs.items = output_port_value
ingest_task.impersonation_allowed = True
stage_task = Task('StageDataToS3')
stage_task.inputs.destination = 's3://{vector_ingest_bucket}/{recipe_id}/{run_id}/{task_name}'
stage_task.inputs.data = ingest_task.outputs.result.value
self.definition['tasks'].append(ingest_task.generate_task_workflow_json())
self.definition['tasks'].append(stage_task.generate_task_workflow_json())
def Task(self, __task_name, **kwargs):
return gbdxtools.simpleworkflows.Task(__task_name, **kwargs)