Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def send_mgr_worker_msg(comm, O):
"""Send message from worker to manager.
"""
D = {'calc_out': O,
'libE_info': {'persistent': True},
'calc_status': UNSET_TAG,
'calc_type': EVAL_GEN_TAG
}
comm.send(EVAL_GEN_TAG, D)
- If no points are left, call the gen_f
"""
Work = {}
gen_count = sum(W['persis_state'] == EVAL_GEN_TAG)
already_in_Work = np.zeros(len(H),dtype=bool) # To mark points as they are included in Work, but not yet marked as 'given' in H.
# If i is idle, but in persistent mode, and its calculated values have
# returned, give them back to i. Otherwise, give nothing to i
for i in W['worker_id'][np.logical_and(W['active']==0,W['persis_state']!=0)]:
gen_inds = H['gen_worker']==i
if np.all(H['returned'][gen_inds]):
last_ind = np.nonzero(gen_inds)[0][np.argmax(H['given_time'][gen_inds])]
Work[i] = {'persis_info': persis_info[i],
'H_fields': sim_specs['in'] + [name[0] for name in sim_specs['out']],
'tag':EVAL_GEN_TAG,
'libE_info': {'H_rows': np.atleast_1d(last_ind),
'persistent': True
}
}
for i in W['worker_id'][np.logical_and(W['active']==0,W['persis_state']==0)]:
# perform sim evaluations from existing runs (if they exist).
q_inds_logical = np.logical_and.reduce((~H['given'],~H['paused'],~already_in_Work))
if np.any(q_inds_logical):
sim_ids_to_send = np.nonzero(q_inds_logical)[0][0] # oldest point
Work[i] = {'H_fields': sim_specs['in'],
'persis_info': {}, # Our sims don't need information about how points were generatored
'tag':EVAL_SIM_TAG,
persis_info[i]['run_order'] = [ind]
gen_count += 1
elif np.any(task_avail):
# Perform sim evaluations from existing runs
q_inds_logical = np.logical_and(task_avail, H['local_pt'])
if not np.any(q_inds_logical):
q_inds_logical = task_avail
sim_ids_to_send = np.nonzero(q_inds_logical)[0][0] # oldest point
sim_work(Work, i, sim_specs['in'], np.atleast_1d(sim_ids_to_send), [])
task_avail[sim_ids_to_send] = False
elif (gen_count == 0
and not np.any(np.logical_and(W['active'] == EVAL_GEN_TAG,
W['persis_state'] == 0))):
# Finally, generate points since there is nothing else to do
gen_count += 1
gen_work(Work, i, gen_specs['in'], [], persis_info[i])
return Work, persis_info
def gen_work(Work, i, H_fields, H_rows, persis_info, **libE_info):
"Add gen work record to work array."
libE_info['H_rows'] = H_rows
Work[i] = {'H_fields': H_fields,
'persis_info': persis_info,
'tag': EVAL_GEN_TAG,
'libE_info': libE_info}
def _check_received_calc(D_recv):
"Check the type and status fields on a receive calculation."
calc_type = D_recv['calc_type']
calc_status = D_recv['calc_status']
assert calc_type in [EVAL_SIM_TAG, EVAL_GEN_TAG], \
"Aborting, Unknown calculation type received. " \
"Received type: {}".format(calc_type)
assert calc_status in [FINISHED_PERSISTENT_SIM_TAG,
FINISHED_PERSISTENT_GEN_TAG,
UNSET_TAG,
MAN_SIGNAL_FINISH,
MAN_SIGNAL_KILL,
WORKER_KILL_ON_ERR,
WORKER_KILL_ON_TIMEOUT,
WORKER_KILL,
JOB_FAILED,
WORKER_DONE], \
"Aborting: Unknown calculation status received. " \
"Received status: {}".format(calc_status)
sim_f = sim_specs['sim_f']
def run_sim(calc_in, persis_info, libE_info):
"Calls the sim func."
return sim_f(calc_in, persis_info, sim_specs, libE_info)
if gen_specs:
gen_f = gen_specs['gen_f']
def run_gen(calc_in, persis_info, libE_info):
"Calls the gen func."
return gen_f(calc_in, persis_info, gen_specs, libE_info)
else:
run_gen = []
return {EVAL_SIM_TAG: run_sim, EVAL_GEN_TAG: run_gen}
assert 'obj_func' in user or 'obj_and_grad_func' in user, "Must have some way to calculate objective values"
if 'obj_func' in user:
for i, row in enumerate(H_rows):
calc_in['f'][i] = user['obj_func'](local_H['x'][row])
if 'grad' in local_H.dtype.names:
for i, row in enumerate(H_rows):
calc_in['grad'][i] = user['grad_func'](local_H['x'][row])
else:
for i, row in enumerate(H_rows):
out = user['obj_and_grad_func'](local_H['x'][row])
calc_in['f'][i] = out[0]
calc_in['grad'][i] = out[1]
return EVAL_GEN_TAG, Work, calc_in
calc_status = D_recv['calc_status']
Manager._check_received_calc(D_recv)
if w not in self.persis_pending:
self.W[w-1]['active'] = 0
if calc_status in [FINISHED_PERSISTENT_SIM_TAG,
FINISHED_PERSISTENT_GEN_TAG]:
self.W[w-1]['persis_state'] = 0
if w in self.persis_pending:
self.persis_pending.remove(w)
self.W[w-1]['active'] = 0
else:
if calc_type == EVAL_SIM_TAG:
self.hist.update_history_f(D_recv)
if calc_type == EVAL_GEN_TAG:
self.hist.update_history_x_in(w, D_recv['calc_out'])
assert len(D_recv['calc_out']) or np.any(self.W['active']) or self.W[w-1]['persis_state'], \
"Gen must return work when is is the only thing active and not persistent."
if 'libE_info' in D_recv and 'persistent' in D_recv['libE_info']:
# Now a waiting, persistent worker
self.W[w-1]['persis_state'] = calc_type
if 'libE_info' in D_recv and 'blocking' in D_recv['libE_info']:
# Now done blocking these workers
for w_i in D_recv['libE_info']['blocking']:
self.W[w_i-1]['blocked'] = 0
self.W[w_i-1]['active'] = 0
if 'persis_info' in D_recv and len(D_recv['persis_info']):
persis_info[w].update(D_recv['persis_info'])
:ivar float time: Calculation run-time
:ivar string date_start: Calculation start date
:ivar string date_end: Calculation end date
:ivar int calc_type: Type flag:EVAL_SIM_TAG/EVAL_GEN_TAG
:ivar int id: Auto-generated ID for this calc (unique within Worker)
:ivar string status: "Description of the status of this calc"
"""
newid = itertools.count()
stat_file = 'libe_summary.txt'
worker_statfile = None
keep_worker_stat_files = False
calc_type_strings = {
EVAL_SIM_TAG: 'sim',
EVAL_GEN_TAG: 'gen',
None: 'No type set'
}
calc_status_strings = {
MAN_SIGNAL_FINISH: "Manager killed on finish",
MAN_SIGNAL_KILL: "Manager killed job",
WORKER_KILL_ON_ERR: " Worker killed job on Error",
WORKER_KILL_ON_TIMEOUT: "Worker killed job on Timeout",
WORKER_KILL: "Worker killed",
JOB_FAILED: "Job Failed",
WORKER_DONE: "Completed",
CALC_EXCEPTION: "Exception occurred",
None: "Unknown Status"
}
@staticmethod