Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _pyspark_command(self, spark_context_variable_name, encode_result=True):
command = u'{}.toJSON()'.format(spark_context_variable_name)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_dataframe_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)
def _pyspark_command(self, sql_context_variable_name, encode_result=True):
command = u'{}.sql(u"""{} """).toJSON()'.format(sql_context_variable_name, self.query)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_dataframe_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)