Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
if batch_id is None:
batch_id = 1
else:
batch_id = batch_id + 1
logging.info(f"Queueing {len(configs)} configs into the database (batch-ID {batch_id}).")
if source_files is not None:
seml_config['source_files'] = source_files
db_dicts = [{'_id': start_id + ix,
'batch_id': batch_id,
'status': 'QUEUED',
'seml': seml_config,
'slurm': slurm_config,
'config': c,
'config_hash': make_hash(c),
'git': git_info,
'queue_time': datetime.datetime.utcnow()}
for ix, c in enumerate(configs)]
collection.insert_many(db_dicts)
configurations: list of dicts
Contains the individual parameter configurations.
use_hash: bool (default: True)
Whether to use the hash of the config dictionary to perform a faster duplicate check.
Returns
-------
filtered_configs: list of dicts
No longer contains configurations that are already in the database collection.
"""
filtered_configs = []
for config in configurations:
if use_hash:
config_hash = make_hash(config)
lookup_result = collection.find_one({'config_hash': config_hash})
else:
lookup_dict = {
f'config.{key}': value for key, value in config.items()
}
lookup_result = collection.find_one(lookup_dict)
if lookup_result is None:
filtered_configs.append(config)
return filtered_configs