Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
)
else:
url = res['url']
file_path = res['fields']['key']
with fsclient.open_file(file_to_upload, 'rb', encoding=None, auto_decompression=False) as f:
files = {'file': (file_path, f)}
res = requests.post(url, data=res['fields'], files=files)
if res.status_code == 201 or res.status_code == 200:
bucket = urllib.parse.urlparse(url).netloc.split('.')[0]
return 's3://%s/%s' % (bucket, file_path)
else:
if res.status_code == 400 and b'EntityTooLarge' in res.content:
max_size = ElementTree.fromstring(res.content).find('MaxSizeAllowed').text
max_size_mb = int(max_size) / 1024 / 1024
raise AugerException('Data set size is limited to %.1f MB' % max_size_mb)
else:
raise AugerException(
'HTTP error [%s] "%s" while uploading file'
' to Auger Cloud...' % (res.status_code, res.content))
def _docker_pull_image(self):
cluster_settings = AugerClusterApi.get_cluster_settings(self.ctx)
docker_tag = cluster_settings.get('kubernetes_stack')
try:
subprocess.check_call(
'docker pull deeplearninc/auger-ml-worker:%s' % \
docker_tag, shell=True)
except subprocess.CalledProcessError as e:
raise AugerException('Can\'t pull Docker container...')
return docker_tag
def _setup_op(self, name, verify_project=True):
old_name = self.ctx.config.get('name', None)
if name is None:
name = old_name
if name is None:
raise AugerException('Please specify Project name...')
project = Project(self.ctx, name)
if verify_project and not project.is_exists:
raise AugerException('Project %s doesn\'t exists...' % name)
return old_name, name, project
def leaderboard(self, dataset, run_id = None):
name = self.ctx.config.get('experiment/name', None)
if name is None:
raise AugerException('Please specify Experiment name...')
if run_id is None:
run_id = self.ctx.config.get(
'experiment/experiment_session_id', None)
leaderboard, status, run_id = Experiment(
self.ctx, dataset, name).leaderboard(run_id)
if leaderboard is None:
raise AugerException('No leaderboard was found...')
self.ctx.log('Leaderboard for Run %s' % run_id)
print_table(self.ctx.log, leaderboard[::-1])
messages = {
'preprocess': 'Search is preprocessing data for training...',
'started': 'Search is in progress...',
'completed': 'Search is completed.',
'interrupted': 'Search was interrupted.',
'error': 'Search was finished with error.'
}
message = messages.get(status, None)
if message:
self.ctx.log(message)
else:
self.ctx.log('Search status is %s' % status)
result = {
def wrapper(self, *args, **kwargs):
project = _get_project(self, False)
data_set_name = self.ctx.config.get('dataset', None)
if data_set_name is None:
raise AugerException(
'Please specify dataset name in auger.yaml/dataset...')
dataset = DataSet(self.ctx, project, data_set_name)
return decorated(self, dataset, *args, **kwargs)
return wrapper
def _setup_op(self, name, verify_project=True):
old_name = self.ctx.config.get('name', None)
if name is None:
name = old_name
if name is None:
raise AugerException('Please specify Project name...')
project = Project(self.ctx, name)
if verify_project and not project.is_exists:
raise AugerException('Project %s doesn\'t exists...' % name)
return old_name, name, project
def name(self):
if self.object_name is None:
properties = self.properties()
if properties is None:
raise AugerException(
'Can\'t find name for remote %s: %s...' % \
(self._get_readable_name(), self.object_id))
self.object_name = properties.get('name')
return self.object_name
def _get_project(self, autocreate):
from .project import Project
project_name = self.ctx.config.get('name', None)
if project_name is None:
raise AugerException(
'Please specify project name in auger.yaml/name...')
project = Project(self.ctx, project_name)
project_properties = project.properties()
if project_properties is None:
if autocreate:
self.ctx.log(
'Can\'t find project %s on the Auger Cloud. '
'Creating...' % project_name)
project.create()
else:
raise AugerException('Can\'t find project %s' % project_name)
return project
def _fill_data_options(self, model_type, stats):
config = self.ctx.config
target = config.get('target')
if not target:
raise AugerException('Please set target to build model.')
exclude = config.get_list('exclude', [])
label_encoded = config.get_list('experiment/label_encoded', [])
categoricals = config.get_list('experiment/categoricals', [])
date_time = config.get_list('experiment/date_time', [])
time_series = None
if model_type is 'timeseries':
time_series = config.get('experiment/time_series', None)
if not time_series:
raise AugerException('Please select time series feature'
' to build time series model'
' (experiment/time_series option).')
for item in stats.get('stat_data'):
column_name = item['column_name']
item['use'] = not column_name in exclude and column_name != target
def _upload_to_multi_tenant(self, file_to_upload):
file_path = 'workspace/projects/%s/files/%s-%s' % \
(self.parent_api.object_name, shortuuid.uuid(),
os.path.basename(file_to_upload))
res = self.rest_api.call('create_project_file_url', {
'project_id': self.parent_api.object_id,
'file_path': file_path,
'file_size': fsclient.get_file_size(file_to_upload)
})
if res is None:
raise AugerException(
'Error while uploading file to Auger Cloud...')
if 'multipart' in res:
upload_details = res['multipart']
config = upload_details['config']
uploader = FileUploader(
upload_details['bucket'],
config['endpoint'],
config['access_key'],
config['secret_key'],
config['security_token']
)
with fsclient.open_file(file_to_upload, 'rb', encoding=None, auto_decompression=False) as f:
return uploader.multipart_upload_obj(