Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_download_html(self):
# Note: In Python 2, pd.read_html returns .columns.inferred_type=mixed
# instead of unicde. So check column type only in PY3 not PY2
out = gramex.data.download(self.dummy, format='html')
result = pd.read_html(io.BytesIO(out), encoding='utf-8')[0]
afe(result, self.dummy, check_column_type=six.PY3)
out = gramex.data.download(AttrDict([
('dummy', self.dummy),
('sales', self.sales)
]), format='html')
result = pd.read_html(io.BytesIO(out), encoding='utf-8')
afe(result[0], self.dummy, check_column_type=six.PY3)
afe(result[1], self.sales, check_column_type=six.PY3)
def test_download_excel(self):
out = gramex.data.download(self.dummy, format='xlsx')
afe(pd.read_excel(io.BytesIO(out)), self.dummy)
out = gramex.data.download({'dummy': self.dummy, 'sales': self.sales}, format='xlsx')
result = pd.read_excel(io.BytesIO(out), sheet_name=None)
afe(result['dummy'], self.dummy)
afe(result['sales'], self.sales)
def test_download_html(self):
# Note: In Python 2, pd.read_html returns .columns.inferred_type=mixed
# instead of unicde. So check column type only in PY3 not PY2
out = gramex.data.download(self.dummy, format='html')
result = pd.read_html(io.BytesIO(out), encoding='utf-8')[0]
afe(result, self.dummy, check_column_type=six.PY3)
out = gramex.data.download(AttrDict([
('dummy', self.dummy),
('sales', self.sales)
]), format='html')
result = pd.read_html(io.BytesIO(out), encoding='utf-8')
afe(result[0], self.dummy, check_column_type=six.PY3)
afe(result[1], self.sales, check_column_type=six.PY3)
def test_download_excel(self):
out = gramex.data.download(self.dummy, format='xlsx')
afe(pd.read_excel(io.BytesIO(out)), self.dummy)
out = gramex.data.download({'dummy': self.dummy, 'sales': self.sales}, format='xlsx')
result = pd.read_excel(io.BytesIO(out), sheet_name=None)
afe(result['dummy'], self.dummy)
afe(result['sales'], self.sales)
def test_download_json(self):
out = gramex.data.download(self.dummy, format='json')
afe(pd.read_json(io.BytesIO(out)), self.dummy, check_like=True)
out = gramex.data.download({'dummy': self.dummy, 'sales': self.sales}, format='json')
result = json.loads(out, object_pairs_hook=AttrDict)
def from_json(key):
s = json.dumps(result[key])
# PY2 returns str (binary). PY3 returns str (unicode). Ensure it's binary
if isinstance(s, six.text_type):
s = s.encode('utf-8')
return pd.read_json(io.BytesIO(s))
afe(from_json('dummy'), self.dummy, check_like=True)
afe(from_json('sales'), self.sales, check_like=True)
def schedule(handler, service):
if handler.request.method == 'GET':
data = get_schedule(service)
data = gramex.data.filter(data, handler.args)
# TODO: handle _format, _meta, _download, etc just like FormHandler
raise Return(gramex.data.download(data))
elif handler.request.method == 'POST':
key = handler.get_argument('name')
schedule = gramex.service[service][key]
results, kwargs = [], {}
# If ?mock is set, and it's an alert, capture the alert mails in result
if handler.get_argument('mock', False) and service == 'alert':
kwargs = {'callback': lambda **kwargs: results.append(kwargs)}
if schedule.thread:
args = yield schedule.function(**kwargs)
else:
args = yield gramex.service.threadpool.submit(schedule.function, **kwargs)
if service == 'alert' and isinstance(args, dict):
for arg in args.get('fail', []):
raise arg['error']
for result in results:
if 'html' in result:
def renderdatas(self):
'''Render multiple datasets'''
fmt = self.getq('format', [None])[0]
kwargs = {}
if fmt == 'template':
kwargs.update(template=self.params.get('template', self._template), handler=self)
else:
self.result = {key: val['data'] for key, val in self.result.items()}
self.write(download(self.result, fmt, **kwargs))
# csv: 'QUERY: %s\n%s' % (key, result['data'].to_csv(index=False, encoding='utf-8'))
def renderdata(self):
# Set content and type based on format
if 'count' in self.result:
self.set_header('X-Count', self.result['count'])
fmt = self.getq('format', [None])[0]
kwargs = {}
if fmt == 'template':
kwargs.update(template=self.params.get('template', self._template), handler=self)
self.write(download(self.result['data'], fmt, **kwargs))
def get(self, *path_args):
'''
Request sent to model/name with no args returns model information,
(that can be changed via PUT/POST).
Request to model/name with args will accept model input and produce predictions.
Request to model/name/data will return the training data specified in model.url,
this should accept most formhandler flags and filters as well.
'''
model = gramex.cache.open(self.pickle_file_path, gramex.ml.load)
if self.get_data_flag():
file_kwargs = self.listify(['engine', 'url', 'ext', 'table', 'query', 'id'])
_format = file_kwargs.pop('_format', ['json'])[0]
# TODO: Add Support for formhandler filters/limit/sorting/groupby
data = gramex.data.filter(model.url, **file_kwargs)
self.write(gramex.data.download(data, format=_format, **file_kwargs))
return
# If no model columns are passed, return model info
if not vars(model).get('input', '') or not any(col in self.args for col in model.input):
model_info = {k: v for k, v in vars(model).items()
if k not in ('model', 'scaler')}
self.write(json.dumps(model_info, indent=4))
return
self._predict(model)