Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _upload_to_multi_tenant(self, file_to_upload):
file_path = 'workspace/projects/%s/files/%s-%s' % \
(self.parent_api.object_name, shortuuid.uuid(),
os.path.basename(file_to_upload))
res = self.rest_api.call('create_project_file_url', {
'project_id': self.parent_api.object_id,
'file_path': file_path})
if res is None:
raise AugerException(
'Error while uploading file to Auger Cloud...')
url = res['url']
with open(file_to_upload, 'rb') as f:
files = {'file': (file_path, f)}
res = requests.post(url, data=res['fields'], files=files)
if res.status_code == 201 or res.status_code == 200:
bucket = urllib.parse.urlparse(url).netloc.split('.')[0]
return 's3://%s/%s' % (bucket, file_path)
else:
raise AugerException(
'HTTP error [%s] while uploading file'
' to Auger Cloud...' % res.status_code)
def name(self):
if self.object_name is None:
properties = self.properties()
if properties is None:
raise AugerException(
'Can\'t find name for remote %s: %s...' % \
(self._get_readable_name(), self.object_id))
self.object_name = properties.get('name')
return self.object_name
def properties(self):
if self.object_id is not None:
return self.rest_api.call(
'get_%s' % self.api_request_path, {'id': self.object_id})
if self.object_name is None:
raise AugerException(
'No name or id was specified for %s' % \
self._get_readable_name())
alt_name = self.object_name.replace('_', '-')
for item in iter(self.list()):
if item['name'] in [self.object_name, alt_name]:
self.object_id = item.get('id')
return item
return None
'cluster/stack_version', default_stack)
settings = {
"kubernetes_stack": kubernetes_stack,
"autoterminate_minutes":
config.get('cluster/autoterminate_minutes', 30)
}
docker_image_tag = config.get('cluster/docker_image_tag', None)
if docker_image_tag is not None:
settings["docker_image_tag"] = docker_image_tag
cluster_type = config.get('cluster/type', None)
if cluster_type is not None:
if cluster_type not in ['standard', 'high_memory']:
raise AugerException(
'Cluster type \'%s\' is not supported' % cluster_type)
settings.update({
"worker_type_id": 1 if cluster_type == 'standard' else 2,
"workers_count": config.get('cluster/max_nodes', 2),
})
else: # single tenant settings
worker_nodes_count = config.get('cluster/worker_count', 2)
worker_nodes_count = config.get(
'cluster/worker_nodes_count', worker_nodes_count)
settings.update({
"worker_nodes_count": worker_nodes_count,
"instance_type": config.get('cluster/instance_type', 'c5.large')
})
workers_per_node_count = config.get(
'cluster/workers_per_node_count', None)
data_set_properties = data_set_api.properties()
stats = data_set_properties['statistics']
self._fill_data_options(options, stats, target, exclude)
if options['targetFeature'] is None:
raise AugerException('Please set target to build model.')
if model_type is not 'timeseries':
options['timeSeriesFeatures'] = []
else:
time_series = auger_config.get('experiment/time_series', None)
if time_series:
options['timeSeriesFeatures'] = [time_series]
if len(options['timeSeriesFeatures']) != 1:
raise AugerException('Please select time series feature'
' to build time series model'
' (experiment/time_series option).')
return {'evaluation_options': options}, model_type
def get_experiment_settings(self):
config = self.ctx.get_config('config')
auger_config = self.ctx.get_config('auger')
model_type = config.get('model_type', '')
if not model_type in MODEL_TYPES:
raise AugerException('Model type should be %s' % \
'|'.join(MODEL_TYPES))
target = config.get('target', '')
exclude = config.get('exclude', [])
options = {
'targetFeature': None,
'featureColumns': [],
'categoricalFeatures': [],
'timeSeriesFeatures': [],
'binaryClassification': False,
'labelEncodingFeatures':
auger_config.get('experiment/label_encoded', []),
# get these options from main config.yaml
'crossValidationFolds':
config.get('experiment/cross_validation_folds', 5),
'max_total_time_mins':
def create(self):
raise AugerException(
'You could\'t create organization using Auger Cloud API.'
' Please use Auger UI to do that...')
True if model_type == 'classification' else False,
'scoring':
auger_config.get('experiment/metric',
'f1_macro' if model_type == 'classification' else 'r2')
}
data_set_id = self.properties()['project_file_id']
data_set_api = AugerDataSetApi(
self.ctx, self.parent_api, None, data_set_id)
data_set_properties = data_set_api.properties()
stats = data_set_properties['statistics']
self._fill_data_options(options, stats, target, exclude)
if options['targetFeature'] is None:
raise AugerException('Please set target to build model.')
if model_type is not 'timeseries':
options['timeSeriesFeatures'] = []
else:
time_series = auger_config.get('experiment/time_series', None)
if time_series:
options['timeSeriesFeatures'] = [time_series]
if len(options['timeSeriesFeatures']) != 1:
raise AugerException('Please select time series feature'
' to build time series model'
' (experiment/time_series option).')
return {'evaluation_options': options}, model_type