How to use the sparkmagic.utils.configuration.pyspark_dataframe_encoding function in sparkmagic

To help you get started, we’ve selected a few sparkmagic examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / livyclientlib / sparkstorecommand.py View on Github external
def _pyspark_command(self, spark_context_variable_name, encode_result=True):
        command = u'{}.toJSON()'.format(spark_context_variable_name)
        if self.samplemethod == u'sample':
            command = u'{}.sample(False, {})'.format(command, self.samplefraction)
        if self.maxrows >= 0:
            command = u'{}.take({})'.format(command, self.maxrows)
        else:
            command = u'{}.collect()'.format(command)
        # Unicode support has improved in Python 3 so we don't need to encode.
        if encode_result:
            print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
                                                     conf.pyspark_dataframe_encoding())
        else:
            print_command = constants.LONG_RANDOM_VARIABLE_NAME
        command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
                                                    command,
                                                    print_command)
        return Command(command)
github jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / livyclientlib / sqlquery.py View on Github external
def _pyspark_command(self, sql_context_variable_name, encode_result=True):
        command = u'{}.sql(u"""{} """).toJSON()'.format(sql_context_variable_name, self.query)
        if self.samplemethod == u'sample':
            command = u'{}.sample(False, {})'.format(command, self.samplefraction)
        if self.maxrows >= 0:
            command = u'{}.take({})'.format(command, self.maxrows)
        else:
            command = u'{}.collect()'.format(command)
        # Unicode support has improved in Python 3 so we don't need to encode.
        if encode_result:
            print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
                                                     conf.pyspark_dataframe_encoding())
        else:
            print_command = constants.LONG_RANDOM_VARIABLE_NAME
        command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
                                                    command,
                                                    print_command)
        return Command(command)