Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
@require_http_methods(["GET", "POST"])
@check_has_write_access_permission
def drop_partition(request, database, table):
source_type = request.POST.get('source_type', 'hive')
db = _get_db(user=request.user, source_type=source_type)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
if request.REQUEST.get("format", "html") == "json":
last_executed = json.loads(request.POST.get('start_time'), '-1')
sql = db.drop_partitions(database, table, partition_specs, design=None, generate_ddl_only=True)
job = make_notebook(
name=_('Drop partition %s') % ', '.join(partition_specs)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
def _get_sample_data(db, database, table, column, is_async=False, cluster=None, operation=None):
table_obj = db.get_table(database, table)
if table_obj.is_impala_only and db.client.query_server['server_name'] != 'impala':
query_server = get_query_server_config('impala', connector=cluster)
db = dbms.get(db.client.user, query_server, cluster=cluster)
sample_data = db.get_sample(database, table_obj, column, generate_sql_only=is_async, operation=operation)
response = {'status': -1}
if sample_data:
response['status'] = 0
if is_async:
notebook = make_notebook(
name=_('Table sample for `%(database)s`.`%(table)s`.`%(column)s`') % {'database': database, 'table': table, 'column': column},
editor_type=_get_servername(db),
statement=sample_data,
status='ready-execute',
skip_historify=True,
is_task=False,
compute=cluster if cluster else None
)
response['result'] = notebook.execute(request=MockedDjangoRequest(user=db.client.user), batch=False)
if table_obj.is_impala_only:
response['result']['type'] = 'impala'
else:
sample = escape_rows(sample_data.rows(), nulls_only=True)
if column:
sample = set([row[0] for row in sample])
sample = [[item] for item in sorted(list(sample))]
def _sync_execute(self, sql, database):
editor = make_notebook(
name='Execute and watch',
editor_type=self.engine,
statement=sql,
database=database,
status='ready-execute',
skip_historify=True
# async=False
)
request = MockRequest(self.user, self.cluster)
mock_notebook = {}
snippet = {'type': self.engine}
response = editor.execute(request)
if 'handle' in response:
# Send notifications: done/on failure
if type(user) is str:
lookup = {'email' if ENABLE_ORGANIZATIONS.get() else 'username': user}
user = User.objects.get(**lookup)
user = rewrite_user(user)
query_document = Document2.objects.get_by_uuid(user=user, uuid=doc_id)
notebook = Notebook(document=query_document).get_data()
snippet = notebook['snippets'][0]
editor_type = snippet['type']
sql = _get_statement(notebook)
request = MockedDjangoRequest(user=user)
last_executed=time.mktime(datetime.datetime.now().timetuple()) * 1000
notebook = make_notebook(
name='Scheduled query %s at %s' % (query_document.name, last_executed),
editor_type=editor_type,
statement=sql,
status='ready',
last_executed=last_executed,
is_task=True
)
task = notebook.execute(request, batch=True)
task['uuid'] = task['history_uuid']
status = check_status(task)
while status['status'] in ('waiting', 'running'):
status = check_status(task)
time.sleep(3)
else:
statement = """REGISTER hdfs://{}/piggybank.jar;
data = LOAD '{}/data/midsummer.txt' as (text:CHARARRAY);
upper_case = FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);
STORE upper_case INTO '$output';
""".format(REMOTE_SAMPLE_DIR.get(), REMOTE_SAMPLE_DIR.get())
snippet_properties = {
'hadoopProperties': [],
'parameters': [],
'resources': []
}
notebook = make_notebook(
name=name,
description=_('UpperText: Example Pig script'),
editor_type='pig',
statement=statement,
status='ready',
snippet_properties=snippet_properties,
is_saved=True
)
# Remove files, functions, settings from snippet properties
data = notebook.get_data()
data['snippets'][0]['properties'].pop('files')
data['snippets'][0]['properties'].pop('functions')
data['snippets'][0]['properties'].pop('settings')
try:
LOG.info("Sample Spark editor job already installed.")
doc2 = Document2.objects.get(owner=self.user, name=name, type='query-spark2', is_history=False)
else:
snippet_properties = {
'jars': ['/user/hue/oozie/workspaces/workflows/spark-scala/lib/oozie-examples.jar'],
'class': 'org.apache.oozie.example.SparkFileCopy',
'app_name': '',
'spark_opts': [],
'spark_arguments': [
"/user/hue/oozie/workspaces/data/sonnets.txt",
"sonnets"
],
'files': []
}
notebook = make_notebook(
name=name,
description=_('File Copy: Example Spark job'),
editor_type='spark2',
statement='',
status='ready',
snippet_properties=snippet_properties,
is_saved=True
)
# Remove files, functions, settings from snippet properties
data = notebook.get_data()
data['snippets'][0]['properties'].pop('functions')
data['snippets'][0]['properties'].pop('settings')
try:
with transaction.atomic():
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
generate_ddl_only = request.POST.get('is_embeddable', 'false') == 'true'
try:
design = SavedQuery.create_empty(app_name=source_type if source_type != 'hive' else 'beeswax', owner=request.user, data=hql_query('').dumps())
form_data = {
'path': load_form.cleaned_data['path'],
'overwrite': load_form.cleaned_data['overwrite'],
'partition_columns': [(column_name, load_form.cleaned_data[key]) for key, column_name in load_form.partition_columns.items()],
}
query_history = db.load_data(database, table.name, form_data, design, generate_ddl_only=generate_ddl_only)
if generate_ddl_only:
last_executed = json.loads(request.POST.get('start_time'), '-1')
job = make_notebook(
name=_('Load data in %s.%s') % (database, table.name),
editor_type=source_type,
statement=query_history.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
response = job.execute(request)
else:
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + on_success_url
response['status'] = 0
response['data'] = url
response['query_history_id'] = query_history.id
except QueryError as ex:
FROM `%(database)s`.`%(table_name)s`;''' % {
'database': database,
'final_table_name': final_table_name,
'table_name': table_name,
'extra_create_properties': extra_create_properties,
'columns_list': ', '.join(columns_list),
'comment': ' COMMENT "%s"' % comment if comment else ''
}
sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
'database': database,
'table_name': table_name
}
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + '?source_type=' + source_type
return make_notebook(
name=_('Creating table %(database)s.%(table)s') % {'database': database, 'table': final_table_name},
editor_type=editor_type,
statement=sql.strip(),
status='ready',
database=database,
on_success_url=on_success_url,
last_executed=start_time,
is_task=True
)