Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
spark_external_parameters = '''
{
"spark.driver.memory": "99G",
"spark.executor.instances": "99",
"spark.executor.memory": "90G"
}
'''
config = AppendLoadConfig(
self.local_run_dir,
self.env_setup,
["year", "month"],
["[0-9]{4}", "(?<=[0-9]{4})([0-9]{2})(?=[0-9]{2})", "(?<=[0-9]{6})([0-9]{2})"]
)
with pytest.raises(M3DIllegalArgumentException) as ex:
config.load_table(self.emr_cluster_id, spark_external_parameters)
assert str(ex.value).startswith("Lengths of target_partitions and regex_filename do not match")
def validate_parameters(self):
if "source_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Source table name is missing in the acon-file")
if "target_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Target table name is missing in the acon-file")
if "source_field" not in self._parameters:
raise M3DIllegalArgumentException(
"Source field name is missing in the acon-file")
if "substring_positions" not in self._parameters:
raise M3DIllegalArgumentException(
"Substring positions specification is missing in the acon-file"
)
if "select_conditions" in self._parameters and "target_partitions" not in self._parameters:
def validate_parameters(self):
if "source_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Source table name is missing in the acon-file")
if "target_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Target table name is missing in the acon-file")
if "source_field" not in self._parameters:
raise M3DIllegalArgumentException(
"Source field name is missing in the acon-file")
if "substring_positions" not in self._parameters:
raise M3DIllegalArgumentException(
"Substring positions specification is missing in the acon-file"
)
if "select_conditions" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_conditions for unpartitioned table")
if "select_rules" in self._parameters and "target_partitions" not in self._parameters:
if "substring_positions" not in self._parameters:
raise M3DIllegalArgumentException(
"Substring positions specification is missing in the acon-file"
)
if "select_conditions" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_conditions for unpartitioned table")
if "select_rules" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_rules for unpartitioned table")
if "select_conditions" in self._parameters and "select_rules" in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use both select_conditions and select_rules at the same time"
)
if "source_field" not in self._parameters:
raise M3DIllegalArgumentException(
"Source field name is missing in the acon-file")
if "substring_positions" not in self._parameters:
raise M3DIllegalArgumentException(
"Substring positions specification is missing in the acon-file"
)
if "select_conditions" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_conditions for unpartitioned table")
if "select_rules" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_rules for unpartitioned table")
if "select_conditions" in self._parameters and "select_rules" in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use both select_conditions and select_rules at the same time"
)
def validate_parameters(self):
if "source_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Source table name is missing in the acon-file")
if "target_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Target table name is missing in the acon-file")
if "source_field" not in self._parameters:
raise M3DIllegalArgumentException(
"Source field name is missing in the acon-file")
if "substring_positions" not in self._parameters:
raise M3DIllegalArgumentException(
"Substring positions specification is missing in the acon-file"
)
if "select_conditions" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_conditions for unpartitioned table")
if "select_rules" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_rules for unpartitioned table")
if "select_conditions" in self._parameters and "select_rules" in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use both select_conditions and select_rules at the same time"
)
def validate_parameters(self):
if "source_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Source table name is missing in the acon-file")
if "target_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Target table name is missing in the acon-file")
if "source_field" not in self._parameters:
raise M3DIllegalArgumentException(
"Source field name is missing in the acon-file")
if "substring_positions" not in self._parameters:
raise M3DIllegalArgumentException(
"Substring positions specification is missing in the acon-file"
)
if "select_conditions" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_conditions for unpartitioned table")
if "select_rules" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_rules for unpartitioned table")
if "select_conditions" in self._parameters and "select_rules" in self._parameters:
def validate_layers(data_layers):
if not data_layers:
raise M3DIllegalArgumentException("No data layer has been specified.")
valid_layers = DataLayers.return_all_layers()
for data_layer in data_layers:
if data_layer not in valid_layers:
raise M3DIllegalArgumentException("Not a valid data layer: {}".format(data_layer))
def validate_spark_params(spark_params):
for key in EMRSystem.SPARK_MANDATORY_KEYS:
if key not in spark_params:
raise M3DIllegalArgumentException("'{}' mandatory key of SparkParameters missing".format(key))
if not spark_params[key]:
raise M3DIllegalArgumentException("'{}' property of SparkParameters should have an assigned value".
format(key))