Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
'-l',
'--logging_level',
default='info',
help='the level of logging to use',
choices=['critical', 'error', 'warning', 'info', 'debug', 'notset']
)
args = parser.parse_args(sys_argv)
args.evaluate_performance = True
logging.getLogger('ludwig').setLevel(
logging_level_registry[args.logging_level]
)
set_on_master(args.use_horovod)
if is_on_master():
print_ludwig('Test', LUDWIG_VERSION)
full_predict(**vars(args))
feed_dict=self.feed_dict(
batch,
regularization_lambda=regularization_lambda,
dropout_rate=0.0,
is_training=is_training
)
)
output_stats, seq_set_size = self.update_output_stats_batch(
output_stats,
seq_set_size,
collect_predictions,
only_predictions,
result
)
if is_on_master():
progress_bar.update(1)
if is_on_master():
progress_bar.close()
if self.horovod:
output_stats, seq_set_size = self.merge_workers_outputs(
output_stats,
seq_set_size
)
output_stats = self.update_output_stats(
output_stats,
set_size,
seq_set_size,
collect_predictions,
def reduce_learning_rate(
self,
progress_tracker,
reduce_learning_rate_on_plateau,
reduce_learning_rate_on_plateau_patience,
reduce_learning_rate_on_plateau_rate
):
if (progress_tracker.last_improvement >=
reduce_learning_rate_on_plateau_patience):
if (progress_tracker.num_reductions_lr >=
reduce_learning_rate_on_plateau):
if is_on_master():
logger.info(
'It has been ' +
str(progress_tracker.last_improvement) +
' epochs since last validation accuracy improvement '
'and the learning rate was already reduced ' +
str(progress_tracker.num_reductions_lr) +
' times, not reducing it anymore'
)
else:
if is_on_master():
logger.info(
'PLATEAU REACHED, reducing learning rate '
'due to lack of validation improvement, it has been ' +
str(progress_tracker.last_improvement) +
' epochs since last validation accuracy improvement '
'or since the learning rate was reduced'
model = Model(
model_definition['input_features'],
model_definition['output_features'],
model_definition['combiner'],
model_definition['training'],
model_definition['preprocessing'],
use_horovod=use_horovod,
random_seed=random_seed,
debug=debug
)
contrib_command("train_model", model, model_definition, model_load_path)
# Train model
if is_on_master():
print_boxed('TRAINING')
return model, model.train(
training_set,
validation_set=validation_set,
test_set=test_set,
save_path=save_path,
resume=resume,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
skip_save_log=skip_save_log,
gpus=gpus, gpu_fraction=gpu_fraction,
random_seed=random_seed,
**model_definition['training']
)
def resume_training(self, save_path, model_weights_path):
if is_on_master():
logger.info('Resuming training of model: {0}'.format(save_path))
self.weights_save_path = model_weights_path
progress_tracker = ProgressTracker.load(
os.path.join(
save_path,
TRAINING_PROGRESS_FILE_NAME
)
)
return progress_tracker
self.save_hyperparameters(
self.hyperparameters,
model_hyperparameters_path
)
logger.info(
'Validation {} on {} improved, model saved'.format(
validation_measure,
validation_field
)
)
progress_tracker.last_improvement = (
progress_tracker.epoch - progress_tracker.last_improvement_epoch
)
if progress_tracker.last_improvement != 0:
if is_on_master():
logger.info(
'Last improvement of {} on {} happened '
'{} epoch{} ago'.format(
validation_measure,
validation_field,
progress_tracker.last_improvement,
'' if progress_tracker.last_improvement == 1 else 's'
)
)
# ========== Reduce Learning Rate Plateau logic ========
if reduce_learning_rate_on_plateau > 0:
self.reduce_learning_rate(
progress_tracker,
reduce_learning_rate_on_plateau,
reduce_learning_rate_on_plateau_patience,
if set_size == 0:
if is_on_master():
logger.warning('No datapoints to evaluate on.')
return output_stats
seq_set_size = {output_feature['name']: {} for output_feature in
self.hyperparameters['output_features'] if
output_feature['type'] in SEQUENCE_TYPES}
batcher = self.initialize_batcher(
dataset,
batch_size,
bucketing_field,
should_shuffle=False
)
if is_on_master():
progress_bar = tqdm(
desc='Evaluation' if name is None
else 'Evaluation {0: <5.5}'.format(name),
total=batcher.steps_per_epoch,
file=sys.stdout,
disable=is_progressbar_disabled()
)
while not batcher.last_batch():
batch = batcher.next_batch()
result = session.run(
output_nodes,
feed_dict=self.feed_dict(
batch,
regularization_lambda=regularization_lambda,
dropout_rate=0.0,
data_hdf5=data_hdf5,
data_train_hdf5=data_train_hdf5,
data_validation_hdf5=data_validation_hdf5,
data_test_hdf5=data_test_hdf5,
train_set_metadata_json=train_set_metadata_json,
skip_save_processed_input=skip_save_processed_input,
preprocessing_params=model_definition['preprocessing'],
random_seed=random_seed
)
(training_set,
validation_set,
test_set,
train_set_metadata) = preprocessed_data
if is_on_master():
logger.info('Training set: {0}'.format(training_set.size))
if validation_set is not None:
logger.info('Validation set: {0}'.format(validation_set.size))
if test_set is not None:
logger.info('Test set: {0}'.format(test_set.size))
# update model definition with metadata properties
update_model_definition_with_metadata(
model_definition,
train_set_metadata
)
if is_on_master():
if not skip_save_model:
# save train set metadata
os.makedirs(model_dir, exist_ok=True)
parser.add_argument(
'-l',
'--logging_level',
default='info',
help='the level of logging to use',
choices=['critical', 'error', 'warning', 'info', 'debug', 'notset']
)
args = parser.parse_args(sys_argv)
logging.getLogger('ludwig').setLevel(
logging_level_registry[args.logging_level]
)
set_on_master(args.use_horovod)
if is_on_master():
print_ludwig('Train', LUDWIG_VERSION)
full_train(**vars(args))
# there is no need to create a directory that will remain empty
should_create_exp_dir = not (
skip_save_unprocessed_output and
skip_save_test_predictions and
skip_save_test_statistics
)
if should_create_exp_dir:
os.makedirs(experiment_dir_name)
# postprocess
postprocessed_output = postprocess(
prediction_results,
model_definition['output_features'],
train_set_metadata,
experiment_dir_name,
skip_save_unprocessed_output or not is_on_master()
)
if not skip_save_test_predictions:
save_prediction_outputs(postprocessed_output, experiment_dir_name)
if evaluate_performance:
print_test_results(prediction_results)
if not skip_save_test_statistics:
save_test_statistics(prediction_results, experiment_dir_name)
logger.info('Saved to: {0}'.format(experiment_dir_name))