How to use the awscli.customizations.emr.constants function in awscli

To help you get started, we’ve selected a few awscli examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github aws / aws-cli / awscli / customizations / emr / sshutils.py View on Github external
def validate_and_find_master_dns(session, parsed_globals, cluster_id):
    """
    Utility method for ssh, socks, put and get command.
    Check if the cluster to be connected to is
     terminated or being terminated.
    Check if the cluster is running.
    Find master instance public dns of a given cluster.
    Return the latest created master instance public dns name.
    Throw MasterDNSNotAvailableError or ClusterTerminatedError.
    """
    cluster_state = emrutils.get_cluster_state(
        session, parsed_globals, cluster_id)

    if cluster_state in constants.TERMINATED_STATES:
        raise exceptions.ClusterTerminatedError

    emr = emrutils.get_client(session, parsed_globals)

    try:
        cluster_running_waiter = emr.get_waiter('cluster_running')
        if cluster_state in constants.STARTING_STATES:
            print("Waiting for the cluster to start.")
        cluster_running_waiter.wait(ClusterId=cluster_id)
    except WaiterError:
        raise exceptions.MasterDNSNotAvailableError

    return emrutils.find_master_public_dns(
        session=session, cluster_id=cluster_id,
        parsed_globals=parsed_globals)
github aws / aws-cli / awscli / customizations / emr / instancegroupsutils.py View on Github external
keys = instance_group.keys()
        if 'Name' in keys:
            ig_config['Name'] = instance_group['Name']
        else:
            ig_config['Name'] = instance_group['InstanceGroupType']
        ig_config['InstanceType'] = instance_group['InstanceType']
        ig_config['InstanceCount'] = instance_group['InstanceCount']
        ig_config['InstanceRole'] = instance_group['InstanceGroupType'].upper()

        if 'BidPrice' in keys:
            if instance_group['BidPrice'] != 'OnDemandPrice':
                ig_config['BidPrice'] = instance_group['BidPrice']
            ig_config['Market'] = constants.SPOT
        else:
            ig_config['Market'] = constants.ON_DEMAND
        if 'EbsConfiguration' in keys:
            ig_config['EbsConfiguration'] = instance_group['EbsConfiguration']

        if 'AutoScalingPolicy' in keys:
            ig_config['AutoScalingPolicy'] = instance_group['AutoScalingPolicy']

        instance_groups.append(ig_config)
    return instance_groups
github aws / aws-cli / awscli / customizations / emr / steputils.py View on Github external
def build_streaming_step(parsed_step):
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_STREAMING_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)

    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.STREAMING_STEP_CONFIG,
        name='Args',
        value=args)
    emrutils.check_empty_string_list(name='Args', value=args)
    return emrutils.build_step(
        jar=constants.STREAMING_JAR_PATH,
        args=args,
        name=name,
        action_on_failure=action_on_failure)
github aws / aws-cli / awscli / customizations / emr / hbase.py View on Github external
def _build_hbase_disable_backups_args(self, parsed_args):
        args = [constants.HBASE_MAIN, constants.HBASE_SCHEDULED_BACKUP,
                constants.FALSE]
        if parsed_args.full is False and parsed_args.incremental is False:
            error_message = 'Should specify at least one of --' +\
                            constants.FULL + ' and --' +\
                            constants.INCREMENTAL + '.'
            raise ValueError(error_message)
        if parsed_args.full is True:
            args.append(constants.HBASE_DISABLE_FULL_BACKUP)
        if parsed_args.incremental is True:
            args.append(constants.HBASE_DISABLE_INCREMENTAL_BACKUP)

        return args
github gkrizek / bash-lambda-layer / bin / awscli / customizations / emr / instancegroupsutils.py View on Github external
def _build_instance_group(
        instance_type, instance_count, instance_group_type):
    ig_config = {}
    ig_config['InstanceType'] = instance_type
    ig_config['InstanceCount'] = instance_count
    ig_config['InstanceRole'] = instance_group_type.upper()
    ig_config['Name'] = ig_config['InstanceRole']
    ig_config['Market'] = constants.ON_DEMAND
    return ig_config
github aws / aws-cli / awscli / customizations / emr / applicationutils.py View on Github external
def _build_impala_install_bootstrap_action(region, args=None):
    args_list = [
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(region=region),
        constants.IMPALA_VERSION,
        constants.LATEST]
    if args is not None:
        args_list.append(constants.IMPALA_CONF)
        args_list.append(','.join(args))
    return emrutils.build_bootstrap_action(
        name=constants.INSTALL_IMPALA_NAME,
        path=emrutils.build_s3_link(
            relative_path=constants.IMPALA_INSTALL_PATH,
            region=region),
        args=args_list)
github aws / aws-cli / awscli / customizations / emr / applicationutils.py View on Github external
def _build_install_hive_step(region,
                             action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
        constants.INSTALL_HIVE_ARG,
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.HIVE_BASE_PATH, region),
        constants.HIVE_VERSIONS,
        constants.LATEST]
    step = emrutils.build_step(
        name=constants.INSTALL_HIVE_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step
github aws / aws-cli / awscli / customizations / emr / hbase.py View on Github external
def _run_main_command(self, parsed_args, parsed_globals):
        steps = []
        args = hbaseutils.build_hbase_restore_from_backup_args(
            parsed_args.dir, parsed_args.backup_version)

        step_config = emrutils.build_step(
            jar=constants.HBASE_JAR_PATH,
            name=constants.HBASE_RESTORE_STEP_NAME,
            action_on_failure=constants.CANCEL_AND_WAIT,
            args=args)

        steps.append(step_config)
        parameters = {'JobFlowId': parsed_args.cluster_id,
                      'Steps': steps}
        emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                           parameters, parsed_globals)
        return 0
github aws / aws-cli / awscli / customizations / emr / createcluster.py View on Github external
def _build_enable_debugging(self, parsed_args, parsed_globals):
        if parsed_args.release_label:
            jar = constants.COMMAND_RUNNER
            args = [constants.DEBUGGING_COMMAND]
        else:
            jar = emrutils.get_script_runner(self.region)
            args = [emrutils.build_s3_link(
                relative_path=constants.DEBUGGING_PATH,
                region=self.region)]

        return emrutils.build_step(
            name=constants.DEBUGGING_NAME,
            action_on_failure=constants.TERMINATE_CLUSTER,
            jar=jar,
            args=args)
github aws / aws-cli / awscli / customizations / emr / steputils.py View on Github external
def build_step_config_list(parsed_step_list, region):
    step_config_list = []
    for step in parsed_step_list:
        step_type = step.get('Type')
        if step_type is None:
            step_type = constants.CUSTOM_JAR

        step_type = step_type.lower()
        step_config = {}
        if step_type == constants.CUSTOM_JAR:
            step_config = build_custom_jar_step(parsed_step=step)
        elif step_type == constants.STREAMING:
            step_config = build_streaming_step(parsed_step=step)
        elif step_type == constants.HIVE:
            step_config = build_hive_step(parsed_step=step, region=region)
        elif step_type == constants.PIG:
            step_config = build_pig_step(parsed_step=step, region=region)
        elif step_type == constants.IMPALA:
            step_config = build_impala_step(parsed_step=step, region=region)
        else:
            raise exceptions.UnknownStepTypeError(step_type=step_type)

        step_config_list.append(step_config)

    return step_config_list