Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def load_cifar_hidden(args, c_idx):
path = 'data/cifar'
kwargs = {'num_workers': 2, 'pin_memory': True, 'drop_last': True}
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
def get_classes(target, labels):
label_indices = []
for i in range(len(target)):
if target[i][1] in labels:
label_indices.append(i)
return label_indices
trainset = torchvision.datasets.CIFAR10(root=path, train=True,
def get_transform(opt):
transform_list = []
if opt.resize_or_crop == 'resize_and_crop':
osize = [opt.loadSize, opt.loadSize]
transform_list.append(transforms.Scale(osize, Image.BICUBIC))
if opt.isTrain:
transform_list.append(transforms.RandomCrop(opt.fineSize))
else: # no random crop in testing time
transform_list.append(transforms.CenterCrop(opt.fineSize))
elif opt.resize_or_crop == 'crop':
transform_list.append(transforms.RandomCrop(opt.fineSize))
elif opt.resize_or_crop == 'scale_width':
transform_list.append(transforms.Lambda(
lambda img: __scale_width(img, opt.fineSize)))
elif opt.resize_or_crop == 'scale_width_and_crop':
transform_list.append(transforms.Lambda(
lambda img: __scale_width(img, opt.loadSize)))
if opt.isTrain:
transform_list.append(transforms.RandomCrop(opt.fineSize))
elif opt.resize_or_crop == 'disentangled':
# my version of tranform
transform_list.append(transforms.Lambda(
lambda img: __scale_width(img, opt.loadSize)))
if opt.isTrain:
transform_list.append(transforms.Lambda(
lambda img: __crop_width_random(img, opt.fineSize)))
else:
args = parser.parse_args()
args.dataset = 'cifar100'
torch.manual_seed(1)
np.random.seed(1)
state = {k: v for k, v in args._get_kwargs()}
state['tt'] = 0 # SGDR variable
state['init_learning_rate'] = args.learning_rate
# mean and standard deviation of channels of CIFAR-10 images
mean = [x / 255 for x in [125.3, 123.0, 113.9]]
std = [x / 255 for x in [63.0, 62.1, 66.7]]
train_transform = trn.Compose([trn.RandomHorizontalFlip(), trn.RandomCrop(32, padding=4),
trn.ToTensor(), trn.Normalize(mean, std)])
test_transform = trn.Compose([trn.ToTensor(), trn.Normalize(mean, std)])
train_data = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=True, transform=train_transform, download=False)
test_data = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=False, transform=test_transform, download=False)
test_data_out = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=False, transform=test_transform, download=False)
num_classes = 20
# d = dict.fromkeys([i for i in range(20)])
# for i in range(len(coarse)):
# if d[coarse[i]] is None: d[coarse[i]] = []
# if fine[i] not in d[coarse[i]]:
# d[coarse[i]].append(fine[i])
coarse_to_fine =\
def torch_loader(data_path, size, bs, val_bs=None, prefetcher=True):
val_bs = val_bs or bs
# Data loading code
tfms = [transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]
train_tfms = transforms.Compose([
pad, # TODO: use `padding` rather than assuming 4
transforms.RandomCrop(size),
transforms.RandomHorizontalFlip(),
] + tfms)
val_tfms = transforms.Compose(tfms)
train_dataset = datasets.CIFAR10(root=data_path, train=True, download=True, transform=train_tfms)
val_dataset = datasets.CIFAR10(root=data_path, train=False, download=True, transform=val_tfms)
aug_dataset = datasets.CIFAR10(root=data_path, train=False, download=True, transform=train_tfms)
train_loader = DataLoader(
train_dataset, batch_size=bs, shuffle=True,
num_workers=workers, pin_memory=True)
val_loader = DataLoader(
val_dataset, batch_size=val_bs, shuffle=False,
num_workers=workers, pin_memory=True)
else:
std = [1.0, 1.0, 1.0]
mean = [0, 0, 0]
normalize = transforms.Normalize(mean = mean, std = std)
if train_random_transform:
if normalize_input:
train = datasets.SVHN('./data', split='train', download=True,
transform=transforms.Compose([
transforms.RandomCrop(32, 4),
transforms.ToTensor(),
normalize,
]))
else:
train = datasets.SVHN('./data', split='train', download=True,
transform=transforms.Compose([
transforms.RandomCrop(32, 4),
transforms.ToTensor(),
]))
else:
train = datasets.SVHN('./data', split='train', download=True,
transform=transforms.Compose([transforms.ToTensor(),normalize]))
test = datasets.SVHN('./data', split='test', download=True,
transform=transforms.Compose([transforms.ToTensor(), normalize]))
if num_examples:
indices = list(range(num_examples))
train = data.Subset(train, indices)
test = data.Subset(test, indices)
train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size,
shuffle=shuffle_train, pin_memory=True, num_workers=min(multiprocessing.cpu_count(),6))
if test_batch_size:
curr_pat = 0
# Build data loader
data_loaders = {}
datasets = {}
data_dir = args.recipe1m_dir
for split in ['train', 'val']:
transforms_list = [transforms.Resize((args.image_size))]
if split == 'train':
# Image preprocessing, normalization for the pretrained resnet
transforms_list.append(transforms.RandomHorizontalFlip())
transforms_list.append(transforms.RandomAffine(degrees=10, translate=(0.1, 0.1)))
transforms_list.append(transforms.RandomCrop(args.crop_size))
else:
transforms_list.append(transforms.CenterCrop(args.crop_size))
transforms_list.append(transforms.ToTensor())
transforms_list.append(transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225)))
transform = transforms.Compose(transforms_list)
max_num_samples = max(args.max_eval, args.batch_size) if split == 'val' else -1
data_loaders[split], datasets[split] = get_loader(data_dir, args.aux_data_dir, split,
args.maxseqlen,
args.maxnuminstrs,
args.maxnumlabels,
args.maxnumims,
transform, args.batch_size,
shuffle=split == 'train', num_workers=args.num_workers,
print_log("Epoch prune: {}".format(args.epoch_prune), log)
# Init dataset
if not os.path.isdir(args.data_path):
os.makedirs(args.data_path)
if args.dataset == 'cifar10':
mean = [x / 255 for x in [125.3, 123.0, 113.9]]
std = [x / 255 for x in [63.0, 62.1, 66.7]]
elif args.dataset == 'cifar100':
mean = [x / 255 for x in [129.3, 124.1, 112.4]]
std = [x / 255 for x in [68.2, 65.4, 70.4]]
else:
assert False, "Unknow dataset : {}".format(args.dataset)
train_transform = transforms.Compose(
[transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
transforms.Normalize(mean, std)])
test_transform = transforms.Compose(
[transforms.ToTensor(), transforms.Normalize(mean, std)])
if args.dataset == 'cifar10':
train_data = dset.CIFAR10(args.data_path, train=True, transform=train_transform, download=True)
test_data = dset.CIFAR10(args.data_path, train=False, transform=test_transform, download=True)
num_classes = 10
elif args.dataset == 'cifar100':
train_data = dset.CIFAR100(args.data_path, train=True, transform=train_transform, download=True)
test_data = dset.CIFAR100(args.data_path, train=False, transform=test_transform, download=True)
num_classes = 100
elif args.dataset == 'svhn':
train_data = dset.SVHN(args.data_path, split='train', transform=train_transform, download=True)
test_data = dset.SVHN(args.data_path, split='test', transform=test_transform, download=True)
num_classes = 10
def get_transform(opt, params=None, grayscale=False, method=Image.BICUBIC, convert=True):
transform_list = []
if grayscale:
transform_list.append(transforms.Grayscale(1))
if 'resize' in opt.preprocess:
osize = [opt.load_size, opt.load_size]
transform_list.append(transforms.Resize(osize, method))
elif 'scale_width' in opt.preprocess:
transform_list.append(transforms.Lambda(lambda img: __scale_width(img, opt.load_size, method)))
if 'crop' in opt.preprocess:
if params is None:
transform_list.append(transforms.RandomCrop(opt.crop_size))
else:
transform_list.append(transforms.Lambda(lambda img: __crop(img, params['crop_pos'], opt.crop_size)))
if opt.preprocess == 'none':
transform_list.append(transforms.Lambda(lambda img: __make_power_2(img, base=4, method=method)))
if not opt.no_flip:
if params is None:
transform_list.append(transforms.RandomHorizontalFlip())
elif params['flip']:
transform_list.append(transforms.Lambda(lambda img: __flip(img, params['flip'])))
if convert:
transform_list += [transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5))]
def load_garbage_classification_data():
"""
load garbage classification data
:return:
"""
batch_size = cfg['batch_size']
print('loading Garbage Classification dataset...')
train_dataset = GarbageDataset(type='train',
transform=transforms.Compose([
transforms.Resize(224),
transforms.RandomCrop(224),
transforms.ColorJitter(),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.RandomRotation(30),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]))
trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=20, drop_last=True)
val_dataset = GarbageDataset(type='val',
transform=transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]))
def main():
global args, best_err1, best_err5
args = parser.parse_args()
if args.dataset.startswith('cifar'):
normalize = transforms.Normalize(mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
transform_test = transforms.Compose([
transforms.ToTensor(),
normalize
])
if args.dataset == 'cifar100':
train_loader = torch.utils.data.DataLoader(
datasets.CIFAR100('../data', train=True, download=True, transform=transform_train),
batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
datasets.CIFAR100('../data', train=False, transform=transform_test),