Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
done()
except Exception as e:
done(e)
t = threading.Thread(target=monitor)
self.params['threads'].append(t)
t.start()
self.lre.run(TaskMock)
self.assertTrue(nonloc.local_task_check)
nonloc.should_fail = True
nonloc.remote_queue = 1
nonloc.task_limit_reached = False
with self.assertRaises(exceptions.TaskFailedError):
self.lre.run(TaskMock)
def process_local(self):
# First task should be 0000 or 0001
if not nonloc.local_task_check: nonloc.local_task_check = self.project_path.endswith("0000") or self.project_path.endswith("0001")
if nonloc.should_fail:
if self.project_path.endswith("0006"):
raise exceptions.TaskFailedError("FAIL #6")
time.sleep(1)
def process_remote(self, done):
time.sleep(0.05) # file upload
self.remote_task = OdmTaskMock(nonloc.remote_queue <= MAX_QUEUE, nonloc.remote_queue)
self.params['tasks'].append(self.remote_task)
if nonloc.should_fail:
if self.project_path.endswith("0006"):
raise exceptions.TaskFailedError("FAIL #6")
nonloc.remote_queue += 1
# Upload successful
done(error=None, partial=True)
# Async processing
def monitor():
try:
if nonloc.task_limit_reached and random.randint(0, 4) == 0:
nonloc.remote_queue -= 1
raise NodeTaskLimitReachedException("Random fail!")
if not nonloc.task_limit_reached and self.remote_task.queue_num > MAX_QUEUE:
nonloc.remote_queue -= 1
nonloc.task_limit_reached = True
log.ODM_INFO("LRE: Downloading assets for %s" % self)
task.download_assets(self.project_path, progress_callback=print_progress)
log.ODM_INFO("LRE: Downloaded and extracted assets for %s" % self)
done()
except exceptions.TaskFailedError as e:
# Try to get output
try:
output_lines = task.output()
# Save to file
error_log_path = self.path("error.log")
with open(error_log_path, 'w') as f:
f.write('\n'.join(output_lines) + '\n')
msg = "(%s) failed with task output: %s\nFull log saved at %s" % (task.uuid, "\n".join(output_lines[-10:]), error_log_path)
done(exceptions.TaskFailedError(msg))
except:
log.ODM_WARNING("LRE: Could not retrieve task output for %s (%s)" % (self, task.uuid))
done(e)
except Exception as e:
done(e)
# Retrieve results
task.download_assets("./results")
print("Assets saved in ./results (%s)" % os.listdir("./results"))
# Restart task and this time compute dtm
task.restart({'dtm': True})
task.wait_for_completion()
print("Task completed, downloading results...")
task.download_assets("./results_with_dtm")
print("Assets saved in ./results_with_dtm (%s)" % os.listdir("./results_with_dtm"))
except exceptions.TaskFailedError as e:
print("\n".join(task.output()))
except exceptions.NodeConnectionError as e:
print("Cannot connect: %s" % e)
except exceptions.NodeResponseError as e:
print("Error: %s" % e)
time.sleep(retry * retry_timeout)
continue
else:
raise e
retry = 0
if status_callback is not None:
status_callback(info)
if info.status in [TaskStatus.COMPLETED, TaskStatus.CANCELED, TaskStatus.FAILED]:
break
time.sleep(interval)
if info.status in [TaskStatus.FAILED, TaskStatus.CANCELED]:
raise TaskFailedError(info.status)
if nonloc.max_remote_tasks is None:
node_task_limit = 0
for t in self.params['tasks']:
try:
info = t.info(with_output=-3)
if info.status == TaskStatus.RUNNING and info.processing_time >= 0 and len(info.output) >= 3:
node_task_limit += 1
except exceptions.OdmError:
pass
nonloc.max_remote_tasks = max(1, node_task_limit)
log.ODM_INFO("LRE: Node task limit reached. Setting max remote tasks to %s" % node_task_limit)
# Retry, but only if the error is not related to a task failure
if task.retries < task.max_retries and not isinstance(error, exceptions.TaskFailedError):
# Put task back in queue
# Don't increment the retry counter if this task simply reached the task
# limit count.
if not task_limit_reached:
task.retries += 1
task.wait_until = datetime.datetime.now() + datetime.timedelta(seconds=task.retries * task.retry_timeout)
cleanup_remote()
q.task_done()
log.ODM_INFO("LRE: Re-queueing %s (retries: %s)" % (task, task.retries))
q.put(task)
if not local: remote_running_tasks.increment(-1)
return
else:
nonloc.error = error
finished_tasks.increment()