Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_join_paths():
assert join_paths() == ""
assert join_paths("foo") == "foo"
assert join_paths("foo", "bar") == "foo.bar"
assert join_paths("a", "b", "c", "d") == "a.b.c.d"
assert join_paths("", "b", "", "d") == "b.d"
assert join_paths("a.b", "c.d.e") == "a.b.c.d.e"
assert join_paths("a.b.", "c.d.e") == "a.b.c.d.e"
def test_join_paths():
assert join_paths() == ""
assert join_paths("foo") == "foo"
assert join_paths("foo", "bar") == "foo.bar"
assert join_paths("a", "b", "c", "d") == "a.b.c.d"
assert join_paths("", "b", "", "d") == "b.d"
assert join_paths("a.b", "c.d.e") == "a.b.c.d.e"
assert join_paths("a.b.", "c.d.e") == "a.b.c.d.e"
# --------- configuration process -------------------
# Phase 1: Config updates
config_updates = config_updates or {}
config_updates = convert_to_nested_dict(config_updates)
root_logger, run_logger = initialize_logging(experiment, scaffolding, log_level)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 2: Named Configs
for ncfg in named_configs:
scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
scaff.gather_fallbacks()
ncfg_updates = scaff.run_named_config(cfg_name)
distribute_presets(prefixes, scaffolding, ncfg_updates)
for ncfg_key, value in iterate_flattened(ncfg_updates):
set_by_dotted_path(config_updates, join_paths(scaff.path, ncfg_key), value)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 3: Normal config scopes
for scaffold in scaffolding.values():
scaffold.gather_fallbacks()
scaffold.set_up_config()
# update global config
config = get_configuration(scaffolding)
# run config hooks
config_hook_updates = scaffold.run_config_hooks(
config, command_name, run_logger
)
recursive_update(scaffold.config, config_hook_updates)
def gather_commands(self):
"""Collect all commands from this ingredient and its sub-ingredients.
Yields
------
cmd_name: str
The full (dotted) name of the command.
cmd: function
The corresponding captured function.
"""
for ingredient, _ in self.traverse_ingredients():
for command_name, command in ingredient.commands.items():
cmd_name = join_paths(ingredient.path, command_name)
cmd_name = self.post_process_name(cmd_name, ingredient)
yield cmd_name, command
def _log_blocked_setitem(self, key, value, fixed_value):
if type_changed(value, fixed_value):
self.typechanges[key] = (type(value), type(fixed_value))
if is_different(value, fixed_value):
self.modified.add(key)
# if both are dicts recursively collect modified and typechanges
if isinstance(fixed_value, DogmaticDict) and isinstance(value, dict):
for k, val in fixed_value.typechanges.items():
self.typechanges[join_paths(key, k)] = val
self.modified |= {join_paths(key, m) for m in fixed_value.modified}
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
if self.path:
add = join_paths(self.path, add)
raise ConfigAddedError(add, config=self.config)
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s'
% (key, type_old.__name__, type_new.__name__)
)
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
"""
# First try to decode to hyperparameter
if isinstance(search_space, dict):
try:
hparam = decode_param_or_op(search_space)
set_name(hparam, path)
return {hparam['uid']: hparam}
except ValueError:
pass
parameters = {}
# if the space is a dict (but not a hyperparameter) we parse it recursively
if isinstance(search_space, dict):
for k, v in search_space.items():
# add the current key and a '.' as prefix when recursing
sub_params = collect_hyperparameters(v, join_paths(path, k))
parameters = merge_parameters(parameters, sub_params)
return parameters
# if the space is a list we iterate it recursively
elif isinstance(search_space, (tuple, list)):
for i, v in enumerate(search_space):
# add '[N]' to the name when recursing
sub_params = collect_hyperparameters(v, path + '[{}]'.format(i))
parameters = merge_parameters(parameters, sub_params)
return parameters
else:
# if the space is anything else do nothing
return parameters
def _emit_started(self):
self.status = "RUNNING"
self.start_time = datetime.datetime.utcnow()
command = join_paths(
self.main_function.prefix, self.main_function.signature.name
)
self.run_logger.info("Running command '%s'", command)
for observer in self.observers:
_id = observer.started_event(
ex_info=self.experiment_info,
command=command,
host_info=self.host_info,
start_time=self.start_time,
config=self.config,
meta_info=self.meta_info,
_id=self._id,
)
if self._id is None:
self._id = _id
# do not catch any exceptions on startup:
def _log_blocked_setitem(self, key, value, fixed_value):
if type_changed(value, fixed_value):
self.typechanges[key] = (type(value), type(fixed_value))
if value != fixed_value:
self.modified.add(key)
# if both are dicts recursively collect modified and typechanges
if isinstance(fixed_value, DogmaticDict) and isinstance(value, dict):
for k, val in fixed_value.typechanges.items():
self.typechanges[join_paths(key, k)] = val
self.modified |= {join_paths(key, m) for m in fixed_value.modified}
def _emit_queued(self):
self.status = "QUEUED"
queue_time = datetime.datetime.utcnow()
self.meta_info["queue_time"] = queue_time
command = join_paths(
self.main_function.prefix, self.main_function.signature.name
)
self.run_logger.info("Queuing-up command '%s'", command)
for observer in self.observers:
_id = observer.queued_event(
ex_info=self.experiment_info,
command=command,
host_info=self.host_info,
queue_time=queue_time,
config=self.config,
meta_info=self.meta_info,
_id=self._id,
)
if self._id is None:
self._id = _id
# do not catch any exceptions on startup: