Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def run_full_test(USE_CUDA, CACHE_ENCODED_DATA, SELFAWARE, PLINEAR):
'''
Run full test example with home_rentals dataset
'''
lightwood.config.config.CONFIG.USE_CUDA = USE_CUDA
lightwood.config.config.CONFIG.PLINEAR = PLINEAR
config = {'input_features': [
{'name': 'number_of_bathrooms', 'type': 'numeric'}, {'name': 'sqft', 'type': 'numeric'},
{'name': 'days_on_market', 'type': 'numeric'},
{'name': 'neighborhood', 'type': 'categorical','dropout':0.4}],
'output_features': [{'name': 'number_of_rooms', 'type': 'categorical',
'weights':{
'0': 0.8,
'1': 0.6,
'2': 0.5,
'3': 0.7,
'4': 1,
}
},{'name': 'rental_price', 'type': 'numeric'},{'name': 'location', 'type': 'categorical'}],
'data_source': {'cache_transformed_data':CACHE_ENCODED_DATA},
def train(self):
lightwood.config.config.CONFIG.USE_CUDA = self.transaction.lmd['use_gpu']
lightwood.config.config.CONFIG.CACHE_ENCODED_DATA = not self.transaction.lmd['force_disable_cache']
lightwood.config.config.CONFIG.SELFAWARE = self.transaction.lmd['use_selfaware_model']
if self.transaction.lmd['model_order_by'] is not None and len(self.transaction.lmd['model_order_by']) > 0:
self.transaction.log.debug('Reshaping data into timeseries format, this may take a while !')
train_df = self._create_timeseries_df(self.transaction.input_data.train_df)
test_df = self._create_timeseries_df(self.transaction.input_data.test_df)
self.transaction.log.debug('Done reshaping data into timeseries format !')
else:
train_df = self.transaction.input_data.train_df
test_df = self.transaction.input_data.test_df
lightwood_config = self._create_lightwood_config()
if self.transaction.lmd['skip_model_training'] == True:
self.predictor = lightwood.Predictor(load_from_path=os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.transaction.lmd['name'] + '_lightwood_data'))
def get_devices():
device_str = "cuda" if CONFIG.USE_CUDA else "cpu"
if CONFIG.USE_DEVICE is not None:
device_str = CONFIG.USE_DEVICE
device = torch.device(device_str)
available_devices = 1
if device_str == 'cuda':
available_devices = torch.cuda.device_count()
return device, available_devices
def __init__(self, ds, dynamic_parameters, shape=None, selfaware=False, size_parameters={}, pretrained_net=None):
self.input_size = None
self.output_size = None
self.selfaware = selfaware
# How many devices we can train this network on
self.available_devices = 1
self.max_variance = None
if ds is not None:
self.out_indexes = ds.out_indexes
device_str = "cuda" if CONFIG.USE_CUDA else "cpu"
if CONFIG.USE_DEVICE is not None:
device_str = CONFIG.USE_DEVICE
self.device = torch.device(device_str)
if CONFIG.DETERMINISTIC:
'''
Seed that always has the same value on the same dataset plus setting the bellow CUDA options
In order to make sure pytorch randomly generate number will be the same every time
when training on the same dataset
'''
if ds is not None:
torch.manual_seed(len(ds))
else:
torch.manual_seed(2)
if device_str == 'cuda':