How to use the m3d.hadoop.algorithm.scala_classes.ScalaClasses function in m3d

To help you get started, we’ve selected a few m3d examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github adidas / m3d-api / test / integration / test_algorithm_gzip_decompressor_emr.py View on Github external
def run_command_in_cluster_patch(cmd, name):
            # Check command name
            assert "Running Spark Application" in str(name)
            print("Command is: {0}".format(cmd))
            command_components = cmd.split()

            # Check algorithm name from the spark command
            algorithm_class_name = command_components[-3]
            assert algorithm_class_name == ScalaClasses.GZIP_DECOMPRESSOR

            # Check configuration file content
            algorithm_config_file_name = command_components[-2]
            actual_config_file_content = self.get_object_content_from_s3(algorithm_config_file_name)
            print("Actual config content: {0}".format(actual_config_file_content))

            algorithm_config_file_dict = json.loads(actual_config_file_content)

            assert algorithm_config_file_dict == expected_param_dict
github adidas / m3d-api / m3d / hadoop / load / delta_load.py View on Github external
def get_scala_class(self):
        return ScalaClasses.DELTA_LOAD
github adidas / m3d-api / m3d / hadoop / algorithm / algorithm_partition_materialization.py View on Github external
def get_scala_class(self):
            return ScalaClasses.PARTITION_QUERY_MATERIALIZATION
github adidas / m3d-api / m3d / hadoop / algorithm / algorithm_algorithm_template.py View on Github external
def get_scala_class(self):
        return ScalaClasses.ALGORITHM_TEMPLATE
github adidas / m3d-api / m3d / hadoop / load / full_load.py View on Github external
def get_scala_class(self):
        return ScalaClasses.FULL_LOAD
github adidas / m3d-api / m3d / hadoop / load / append_load.py View on Github external
def get_scala_class(self):
        return ScalaClasses.APPEND_LOAD
github adidas / m3d-api / m3d / hadoop / algorithm / algorithm_partition_materialization.py View on Github external
def get_scala_class(self):
            return ScalaClasses.PARTITION_FULL_MATERIALIZATION
github adidas / m3d-api / m3d / hadoop / algorithm / algorithm_partition_materialization.py View on Github external
def get_scala_class(self):
            return ScalaClasses.PARTITION_RANGE_MATERIALIZATION
github adidas / m3d-api / m3d / hadoop / algorithm / algorithm_fixed_length_string_extractor.py View on Github external
def get_scala_class(self):
        return ScalaClasses.FIXED_LENGTH_STRING_EXTRACTOR