Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def reset_experiments(db_collection_name, sacred_id, filter_states, batch_id, filter_dict):
collection = get_collection(db_collection_name)
if sacred_id is None:
if len({'PENDING', 'RUNNING', 'KILLED'} & set(filter_states)) > 0:
detect_killed(db_collection_name, print_detected=False)
filter_dict = build_filter_dict(filter_states, batch_id, filter_dict)
nreset = collection.count_documents(filter_dict)
exps = collection.find(filter_dict)
if nreset >= 10:
if input(f"Resetting the state of {nreset} experiment{s_if(nreset)}. "
f"Are you sure? (y/n) ").lower() != "y":
exit()
else:
logging.info(f"Resetting the state of {nreset} experiment{s_if(nreset)}.")
for exp in exps:
reset_single_experiment(collection, exp)
else:
exp = collection.find_one({'_id': sacred_id})
if exp is None:
logging.error(f"No experiment found with ID {sacred_id}.")
def delete_experiments(db_collection_name, sacred_id, filter_states, batch_id, filter_dict):
collection = get_collection(db_collection_name)
if sacred_id is None:
if len({'PENDING', 'RUNNING', 'KILLED'} & set(filter_states)) > 0:
detect_killed(db_collection_name, print_detected=False)
filter_dict = build_filter_dict(filter_states, batch_id, filter_dict)
ndelete = collection.count_documents(filter_dict)
batch_ids = collection.find(filter_dict, {'batch_id'})
batch_ids_in_del = set([x['batch_id'] for x in batch_ids])
if ndelete >= 10:
if input(f"Deleting {ndelete} configuration{s_if(ndelete)} from database collection. "
f"Are you sure? (y/n) ").lower() != "y":
exit()
else:
logging.info(f"Deleting {ndelete} configuration{s_if(ndelete)} from database collection.")
collection.delete_many(filter_dict)
else:
exp = collection.find_one({'_id': sacred_id})
if exp is None:
logging.error(f"No experiment found with ID {sacred_id}.")
sys.exit(1)
Returns
-------
None
"""
collection = get_collection(db_collection_name)
if sacred_id is None:
# no ID is provided: we check whether there are slurm jobs for which after this action no
# RUNNING experiment remains. These slurm jobs can be killed altogether.
# However, it is NOT possible right now to cancel a single experiment in a Slurm job with multiple
# running experiments.
try:
if len({'PENDING', 'RUNNING', 'KILLED'} & set(filter_states)) > 0:
detect_killed(db_collection_name, print_detected=False)
filter_dict = build_filter_dict(filter_states, batch_id, filter_dict)
ncancel = collection.count_documents(filter_dict)
if ncancel >= 10:
if input(f"Cancelling {ncancel} experiment{s_if(ncancel)}. "
f"Are you sure? (y/n) ").lower() != "y":
exit()
else:
logging.info(f"Cancelling {ncancel} experiment{s_if(ncancel)}.")
filter_dict_new = filter_dict.copy()
filter_dict_new.update({'slurm.array_id': {'$exists': True}})
exps = list(collection.find(filter_dict_new, {'_id': 1, 'status': 1, 'slurm.array_id': 1, 'slurm.task_id': 1}))
# set of slurm IDs in the database
slurm_ids = set([(e['slurm']['array_id'], e['slurm']['task_id']) for e in exps])
# set of experiment IDs to be cancelled.
exp_ids = set([e['_id'] for e in exps])
def start_experiments(db_collection_name, local, sacred_id, batch_id, filter_dict,
num_exps, unobserved, post_mortem, debug, dry_run,
output_to_console):
use_slurm = not local
output_to_file = not output_to_console
if debug:
num_exps = 1
use_slurm = False
unobserved = True
post_mortem = True
output_to_file = False
logging.root.setLevel(logging.VERBOSE)
if sacred_id is None:
filter_dict = build_filter_dict([], batch_id, filter_dict)
else:
filter_dict = {'_id': sacred_id}
if dry_run:
print_commands(db_collection_name, unobserved=unobserved, post_mortem=post_mortem,
num_exps=num_exps, filter_dict=filter_dict)
else:
start_jobs(db_collection_name, slurm=use_slurm,
unobserved=unobserved, post_mortem=post_mortem,
num_exps=num_exps, filter_dict=filter_dict, dry_run=dry_run,
output_to_file=output_to_file)