Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _assert_input_is_pandas_dataframe(self, input_variable_name, input_variable_value):
if not isinstance(input_variable_value, pd.DataFrame):
wrong_type = input_variable_value.__class__.__name__
raise BadUserDataException(u'{} is not a Pandas DataFrame! Got {} instead.'.format(input_variable_name, wrong_type))
def to_command(self, kind, input_variable_name, input_variable_value, output_variable_name):
if kind == constants.SESSION_KIND_PYSPARK:
return self._pyspark_command(input_variable_name, input_variable_value, output_variable_name)
elif kind == constants.SESSION_KIND_SPARK:
return self._scala_command(input_variable_name, input_variable_value, output_variable_name)
elif kind == constants.SESSION_KIND_SPARKR:
return self._r_command(input_variable_name, input_variable_value, output_variable_name)
else:
raise BadUserDataException(u"Kind '{}' is not supported.".format(kind))
def parse_argstring_or_throw(magic_func, argstring, parse_argstring=parse_argstring):
"""An alternative to the parse_argstring method from IPython.core.magic_arguments.
Catches IPython.core.error.UsageError and propagates it as a
livyclientlib.exceptions.BadUserDataException."""
try:
return parse_argstring(magic_func, argstring)
except UsageError as e:
raise BadUserDataException(str(e))
def _assert_input_is_string_type(self, input_variable_name, input_variable_value):
if not isinstance(input_variable_value, str):
wrong_type = input_variable_value.__class__.__name__
raise BadUserDataException(u'{} is not a str or bytes! Got {} instead'.format(input_variable_name, wrong_type))
def to_command(self, kind, spark_context_variable_name):
if kind == constants.SESSION_KIND_PYSPARK:
return self._pyspark_command(spark_context_variable_name)
elif kind == constants.SESSION_KIND_PYSPARK3:
return self._pyspark_command(spark_context_variable_name, False)
elif kind == constants.SESSION_KIND_SPARK:
return self._scala_command(spark_context_variable_name)
elif kind == constants.SESSION_KIND_SPARKR:
return self._r_command(spark_context_variable_name)
else:
raise BadUserDataException(u"Kind '{}' is not supported.".format(kind))
def __init__(self, output_var, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None, coerce=None):
super(SparkStoreCommand, self).__init__("", spark_events)
if samplemethod is None:
samplemethod = conf.default_samplemethod()
if maxrows is None:
maxrows = conf.default_maxrows()
if samplefraction is None:
samplefraction = conf.default_samplefraction()
if samplemethod not in {u'take', u'sample'}:
raise BadUserDataException(u'samplemethod (-m) must be one of (take, sample)')
if not isinstance(maxrows, int):
raise BadUserDataException(u'maxrows (-n) must be an integer')
if not 0.0 <= samplefraction <= 1.0:
raise BadUserDataException(u'samplefraction (-r) must be a float between 0.0 and 1.0')
self.samplemethod = samplemethod
self.maxrows = maxrows
self.samplefraction = samplefraction
self.output_var = output_var
if spark_events is None:
spark_events = SparkEvents()
self._spark_events = spark_events
self._coerce = coerce