Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def build_lr_scheduler(optimizer: Optimizer, args: Namespace, total_epochs: List[int] = None) -> _LRScheduler:
"""
Builds a learning rate scheduler.
:param optimizer: The Optimizer whose learning rate will be scheduled.
:param args: Arguments.
:return: An initialized learning rate scheduler.
"""
# Learning rate scheduler
if args.scheduler == 'noam':
return NoamLR(
optimizer=optimizer,
warmup_epochs=args.warmup_epochs,
total_epochs=total_epochs or [args.epochs] * args.num_lrs,
steps_per_epoch=args.train_data_size // args.batch_size,
init_lr=args.init_lr,
max_lr=args.max_lr,
final_lr=args.final_lr
)
if args.scheduler == 'none':
return MockLR(optimizer=optimizer, lr=args.init_lr)
if args.scheduler == 'decay':
return ExponentialLR(optimizer, args.lr_decay_rate)
raise ValueError(f'Learning rate scheduler "{args.scheduler}" not supported.')
self.act_func,
nn.Linear(self.hidden_size, self.hidden_size),
self.act_func,
nn.Linear(self.hidden_size, self.hidden_size),
self.act_func,
nn.Linear(self.hidden_size, 1)
)
self.beta = args.wgan_beta
# the optimizers don't really belong here, but we put it here so that we don't clutter code for other opts
self.optimizerG = Adam(self.encoder.parameters(), lr=args.init_lr[0] * args.gan_lr_mult, betas=(0, 0.9))
self.optimizerD = Adam(self.netD.parameters(), lr=args.init_lr[0] * args.gan_lr_mult, betas=(0, 0.9))
self.use_scheduler = args.gan_use_scheduler
if self.use_scheduler:
self.schedulerG = NoamLR(
self.optimizerG,
warmup_epochs=args.warmup_epochs,
total_epochs=args.epochs,
steps_per_epoch=args.train_data_length // args.batch_size,
init_lr=args.init_lr[0] * args.gan_lr_mult,
max_lr=args.max_lr[0] * args.gan_lr_mult,
final_lr=args.final_lr[0] * args.gan_lr_mult
)
self.schedulerD = NoamLR(
self.optimizerD,
warmup_epochs=args.warmup_epochs,
total_epochs=args.epochs,
steps_per_epoch=(args.train_data_length // args.batch_size) * args.gan_d_per_g,
init_lr=args.init_lr[0] * args.gan_lr_mult,
max_lr=args.max_lr[0] * args.gan_lr_mult,
final_lr=args.final_lr[0] * args.gan_lr_mult
# the optimizers don't really belong here, but we put it here so that we don't clutter code for other opts
self.optimizerG = Adam(self.encoder.parameters(), lr=args.init_lr[0] * args.gan_lr_mult, betas=(0, 0.9))
self.optimizerD = Adam(self.netD.parameters(), lr=args.init_lr[0] * args.gan_lr_mult, betas=(0, 0.9))
self.use_scheduler = args.gan_use_scheduler
if self.use_scheduler:
self.schedulerG = NoamLR(
self.optimizerG,
warmup_epochs=args.warmup_epochs,
total_epochs=args.epochs,
steps_per_epoch=args.train_data_length // args.batch_size,
init_lr=args.init_lr[0] * args.gan_lr_mult,
max_lr=args.max_lr[0] * args.gan_lr_mult,
final_lr=args.final_lr[0] * args.gan_lr_mult
)
self.schedulerD = NoamLR(
self.optimizerD,
warmup_epochs=args.warmup_epochs,
total_epochs=args.epochs,
steps_per_epoch=(args.train_data_length // args.batch_size) * args.gan_d_per_g,
init_lr=args.init_lr[0] * args.gan_lr_mult,
max_lr=args.max_lr[0] * args.gan_lr_mult,
final_lr=args.final_lr[0] * args.gan_lr_mult
)
else:
loss.backward()
if args.max_grad_norm is not None:
clip_grad_norm_(model.parameters(), args.max_grad_norm)
optimizer.step()
if args.adjust_weight_decay:
current_pnorm = compute_pnorm(model)
if current_pnorm < args.pnorm_target:
for i in range(len(optimizer.param_groups)):
optimizer.param_groups[i]['weight_decay'] = max(0, optimizer.param_groups[i]['weight_decay'] - args.adjust_weight_decay_step)
else:
for i in range(len(optimizer.param_groups)):
optimizer.param_groups[i]['weight_decay'] += args.adjust_weight_decay_step
if isinstance(scheduler, NoamLR):
scheduler.step()
if args.adversarial:
for _ in range(args.gan_d_per_g):
train_val_smiles_batch = random.sample(train_val_smiles, args.batch_size)
test_smiles_batch = random.sample(test_smiles, args.batch_size)
d_loss, gp_norm = model.train_D(train_val_smiles_batch, test_smiles_batch)
train_val_smiles_batch = random.sample(train_val_smiles, args.batch_size)
test_smiles_batch = random.sample(test_smiles, args.batch_size)
g_loss = model.train_G(train_val_smiles_batch, test_smiles_batch)
# we probably only care about the g_loss honestly
d_loss_sum += d_loss * args.batch_size
gp_norm_sum += gp_norm * args.batch_size
g_loss_sum += g_loss * args.batch_size
self.warmup_epochs = np.array(warmup_epochs)
self.total_epochs = np.array(total_epochs)
self.steps_per_epoch = steps_per_epoch
self.init_lr = np.array(init_lr)
self.max_lr = np.array(max_lr)
self.final_lr = np.array(final_lr)
self.current_step = 0
self.lr = init_lr
self.warmup_steps = (self.warmup_epochs * self.steps_per_epoch).astype(int)
self.total_steps = self.total_epochs * self.steps_per_epoch
self.linear_increment = (self.max_lr - self.init_lr) / self.warmup_steps
self.exponential_gamma = (self.final_lr / self.max_lr) ** (1 / (self.total_steps - self.warmup_steps))
super(NoamLR, self).__init__(optimizer)