How to use the awscli.customizations.emr.emrutils function in awscli

To help you get started, we’ve selected a few awscli examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github aws / aws-cli / awscli / customizations / emr / describecluster.py View on Github external
def _call(self, session, operation_name, parameters, parsed_globals):
        return emrutils.call(
            session, operation_name, parameters,
            region_name=self.region,
            endpoint_url=parsed_globals.endpoint_url,
            verify=parsed_globals.verify_ssl)
github aws / aws-cli / awscli / customizations / emr / applicationutils.py View on Github external
def _build_ganglia_install_bootstrap_action(region):
    return emrutils.build_bootstrap_action(
        name=constants.INSTALL_GANGLIA_NAME,
        path=emrutils.build_s3_link(
            relative_path=constants.GANGLIA_INSTALL_BA_PATH,
            region=region))
github aws / aws-cli / awscli / customizations / emr / applicationutils.py View on Github external
def _build_install_hive_step(region,
                             action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
        constants.INSTALL_HIVE_ARG,
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.HIVE_BASE_PATH, region),
        constants.HIVE_VERSIONS,
        constants.LATEST]
    step = emrutils.build_step(
        name=constants.INSTALL_HIVE_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step
github aws / aws-cli / awscli / customizations / emr / emrfsutils.py View on Github external
def _verify_child_args(actual_keys, child_keys, parent_object_name):
    if any(x in actual_keys for x in child_keys):
        invalid_keys = list(
            sorted(set(child_keys).intersection(set(actual_keys))))
        raise exceptions.InvalidEmrFsArgumentsError(
            invalid=emrutils.join(invalid_keys),
            parent_object_name=parent_object_name)
github aws / aws-cli / awscli / customizations / emr / steputils.py View on Github external
arg=parsed_step.get('Name'),
        value=constants.DEFAULT_IMPALA_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    args_list = [
        emrutils.build_s3_link(
            relative_path=constants.IMPALA_INSTALL_PATH, region=region),
        constants.RUN_IMPALA_SCRIPT]
    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.IMPALA_STEP_CONFIG, name='Args', value=args)
    args_list += args

    return emrutils.build_step(
        jar=emrutils.get_script_runner(region),
        args=args_list,
        name=name,
        action_on_failure=action_on_failure)
github aws / aws-cli / awscli / customizations / emr / ssh.py View on Github external
def _run_main_command(self, parsed_args, parsed_globals):
        master_dns = sshutils.validate_and_find_master_dns(
            session=self._session,
            parsed_globals=parsed_globals,
            cluster_id=parsed_args.cluster_id)

        key_file = parsed_args.key_pair_file
        sshutils.validate_ssh_with_key_file(key_file)
        f = tempfile.NamedTemporaryFile(delete=False)
        if (emrutils.which('ssh') or emrutils.which('ssh.exe')):
            command = ['ssh', '-o', 'StrictHostKeyChecking=no', '-o',
                       'ServerAliveInterval=10', '-i',
                       parsed_args.key_pair_file, constants.SSH_USER +
                       '@' + master_dns, '-t']
            if parsed_args.command:
                command.append(parsed_args.command)
        else:
            command = ['putty', '-ssh', '-i', parsed_args.key_pair_file,
                       constants.SSH_USER + '@' + master_dns, '-t']
            if parsed_args.command:
                f.write(parsed_args.command)
                f.write('\nread -n1 -r -p "Command completed. Press any key."')
                command.append('-m')
                command.append(f.name)

        f.close()
github aws / aws-cli / awscli / customizations / emr / hbase.py View on Github external
def _run_main_command(self, parsed_args, parsed_globals):
        steps = []

        args = self._build_hbase_disable_backups_args(parsed_args)

        step_config = emrutils.build_step(
            constants.HBASE_JAR_PATH,
            constants.HBASE_SCHEDULE_BACKUP_STEP_NAME,
            constants.CANCEL_AND_WAIT,
            args)

        steps.append(step_config)
        parameters = {'JobFlowId': parsed_args.cluster_id,
                      'Steps': steps}
        emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                           parameters, parsed_globals)
        return 0
github aws / aws-cli / awscli / customizations / emr / createcluster.py View on Github external
dest_params=instances['Placement'],
                dest_key='AvailabilityZones')

        emrutils.apply_params(
            src_params=parsed_attrs, src_key='InstanceProfile',
            dest_params=cluster, dest_key='JobFlowRole')

        emrutils.apply_params(
            src_params=parsed_attrs, src_key='EmrManagedMasterSecurityGroup',
            dest_params=instances, dest_key='EmrManagedMasterSecurityGroup')

        emrutils.apply_params(
            src_params=parsed_attrs, src_key='EmrManagedSlaveSecurityGroup',
            dest_params=instances, dest_key='EmrManagedSlaveSecurityGroup')

        emrutils.apply_params(
            src_params=parsed_attrs, src_key='ServiceAccessSecurityGroup',
            dest_params=instances, dest_key='ServiceAccessSecurityGroup')

        emrutils.apply_params(
            src_params=parsed_attrs, src_key='AdditionalMasterSecurityGroups',
            dest_params=instances, dest_key='AdditionalMasterSecurityGroups')

        emrutils.apply_params(
            src_params=parsed_attrs, src_key='AdditionalSlaveSecurityGroups',
            dest_params=instances, dest_key='AdditionalSlaveSecurityGroups')

        emrutils.apply(params=cluster, key='Instances', value=instances)

        return cluster
github aws / aws-cli / awscli / customizations / emr / createcluster.py View on Github external
if ('AvailabilityZone' in keys and 'AvailabilityZones' in keys):
            raise exceptions.MutualExclusiveOptionError(
                option1="AvailabilityZone",
                option2="AvailabilityZones")

        if ('SubnetId' in keys or 'SubnetIds' in keys) \
                and ('AvailabilityZone' in keys or 'AvailabilityZones' in keys):
            raise exceptions.SubnetAndAzValidationError

        emrutils.apply_params(
            src_params=parsed_attrs, src_key='KeyName',
            dest_params=instances, dest_key='Ec2KeyName')
        emrutils.apply_params(
            src_params=parsed_attrs, src_key='SubnetId',
            dest_params=instances, dest_key='Ec2SubnetId')
        emrutils.apply_params(
            src_params=parsed_attrs, src_key='SubnetIds',
            dest_params=instances, dest_key='Ec2SubnetIds')

        if 'AvailabilityZone' in keys:
            instances['Placement'] = dict()
            emrutils.apply_params(
                src_params=parsed_attrs, src_key='AvailabilityZone',
                dest_params=instances['Placement'],
                dest_key='AvailabilityZone')

        if 'AvailabilityZones' in keys:
            instances['Placement'] = dict()
            emrutils.apply_params(
                src_params=parsed_attrs, src_key='AvailabilityZones',
                dest_params=instances['Placement'],
                dest_key='AvailabilityZones')
github aws / aws-cli / awscli / customizations / emr / steputils.py View on Github external
def build_custom_jar_step(parsed_step):
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_CUSTOM_JAR_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    emrutils.check_required_field(
        structure=constants.CUSTOM_JAR_STEP_CONFIG,
        name='Jar',
        value=parsed_step.get('Jar'))
    return emrutils.build_step(
        jar=parsed_step.get('Jar'),
        args=parsed_step.get('Args'),
        name=name,
        action_on_failure=action_on_failure,
        main_class=parsed_step.get('MainClass'),
        properties=emrutils.parse_key_value_string(
            parsed_step.get('Properties')))