Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _call(self, session, operation_name, parameters, parsed_globals):
return emrutils.call(
session, operation_name, parameters,
region_name=self.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl)
def _build_ganglia_install_bootstrap_action(region):
return emrutils.build_bootstrap_action(
name=constants.INSTALL_GANGLIA_NAME,
path=emrutils.build_s3_link(
relative_path=constants.GANGLIA_INSTALL_BA_PATH,
region=region))
def _build_install_hive_step(region,
action_on_failure=constants.TERMINATE_CLUSTER):
step_args = [
emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
constants.INSTALL_HIVE_ARG,
constants.BASE_PATH_ARG,
emrutils.build_s3_link(constants.HIVE_BASE_PATH, region),
constants.HIVE_VERSIONS,
constants.LATEST]
step = emrutils.build_step(
name=constants.INSTALL_HIVE_NAME,
action_on_failure=action_on_failure,
jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
args=step_args)
return step
def _verify_child_args(actual_keys, child_keys, parent_object_name):
if any(x in actual_keys for x in child_keys):
invalid_keys = list(
sorted(set(child_keys).intersection(set(actual_keys))))
raise exceptions.InvalidEmrFsArgumentsError(
invalid=emrutils.join(invalid_keys),
parent_object_name=parent_object_name)
arg=parsed_step.get('Name'),
value=constants.DEFAULT_IMPALA_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
args_list = [
emrutils.build_s3_link(
relative_path=constants.IMPALA_INSTALL_PATH, region=region),
constants.RUN_IMPALA_SCRIPT]
args = parsed_step.get('Args')
emrutils.check_required_field(
structure=constants.IMPALA_STEP_CONFIG, name='Args', value=args)
args_list += args
return emrutils.build_step(
jar=emrutils.get_script_runner(region),
args=args_list,
name=name,
action_on_failure=action_on_failure)
def _run_main_command(self, parsed_args, parsed_globals):
master_dns = sshutils.validate_and_find_master_dns(
session=self._session,
parsed_globals=parsed_globals,
cluster_id=parsed_args.cluster_id)
key_file = parsed_args.key_pair_file
sshutils.validate_ssh_with_key_file(key_file)
f = tempfile.NamedTemporaryFile(delete=False)
if (emrutils.which('ssh') or emrutils.which('ssh.exe')):
command = ['ssh', '-o', 'StrictHostKeyChecking=no', '-o',
'ServerAliveInterval=10', '-i',
parsed_args.key_pair_file, constants.SSH_USER +
'@' + master_dns, '-t']
if parsed_args.command:
command.append(parsed_args.command)
else:
command = ['putty', '-ssh', '-i', parsed_args.key_pair_file,
constants.SSH_USER + '@' + master_dns, '-t']
if parsed_args.command:
f.write(parsed_args.command)
f.write('\nread -n1 -r -p "Command completed. Press any key."')
command.append('-m')
command.append(f.name)
f.close()
def _run_main_command(self, parsed_args, parsed_globals):
steps = []
args = self._build_hbase_disable_backups_args(parsed_args)
step_config = emrutils.build_step(
constants.HBASE_JAR_PATH,
constants.HBASE_SCHEDULE_BACKUP_STEP_NAME,
constants.CANCEL_AND_WAIT,
args)
steps.append(step_config)
parameters = {'JobFlowId': parsed_args.cluster_id,
'Steps': steps}
emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
parameters, parsed_globals)
return 0
dest_params=instances['Placement'],
dest_key='AvailabilityZones')
emrutils.apply_params(
src_params=parsed_attrs, src_key='InstanceProfile',
dest_params=cluster, dest_key='JobFlowRole')
emrutils.apply_params(
src_params=parsed_attrs, src_key='EmrManagedMasterSecurityGroup',
dest_params=instances, dest_key='EmrManagedMasterSecurityGroup')
emrutils.apply_params(
src_params=parsed_attrs, src_key='EmrManagedSlaveSecurityGroup',
dest_params=instances, dest_key='EmrManagedSlaveSecurityGroup')
emrutils.apply_params(
src_params=parsed_attrs, src_key='ServiceAccessSecurityGroup',
dest_params=instances, dest_key='ServiceAccessSecurityGroup')
emrutils.apply_params(
src_params=parsed_attrs, src_key='AdditionalMasterSecurityGroups',
dest_params=instances, dest_key='AdditionalMasterSecurityGroups')
emrutils.apply_params(
src_params=parsed_attrs, src_key='AdditionalSlaveSecurityGroups',
dest_params=instances, dest_key='AdditionalSlaveSecurityGroups')
emrutils.apply(params=cluster, key='Instances', value=instances)
return cluster
if ('AvailabilityZone' in keys and 'AvailabilityZones' in keys):
raise exceptions.MutualExclusiveOptionError(
option1="AvailabilityZone",
option2="AvailabilityZones")
if ('SubnetId' in keys or 'SubnetIds' in keys) \
and ('AvailabilityZone' in keys or 'AvailabilityZones' in keys):
raise exceptions.SubnetAndAzValidationError
emrutils.apply_params(
src_params=parsed_attrs, src_key='KeyName',
dest_params=instances, dest_key='Ec2KeyName')
emrutils.apply_params(
src_params=parsed_attrs, src_key='SubnetId',
dest_params=instances, dest_key='Ec2SubnetId')
emrutils.apply_params(
src_params=parsed_attrs, src_key='SubnetIds',
dest_params=instances, dest_key='Ec2SubnetIds')
if 'AvailabilityZone' in keys:
instances['Placement'] = dict()
emrutils.apply_params(
src_params=parsed_attrs, src_key='AvailabilityZone',
dest_params=instances['Placement'],
dest_key='AvailabilityZone')
if 'AvailabilityZones' in keys:
instances['Placement'] = dict()
emrutils.apply_params(
src_params=parsed_attrs, src_key='AvailabilityZones',
dest_params=instances['Placement'],
dest_key='AvailabilityZones')
def build_custom_jar_step(parsed_step):
name = _apply_default_value(
arg=parsed_step.get('Name'),
value=constants.DEFAULT_CUSTOM_JAR_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
emrutils.check_required_field(
structure=constants.CUSTOM_JAR_STEP_CONFIG,
name='Jar',
value=parsed_step.get('Jar'))
return emrutils.build_step(
jar=parsed_step.get('Jar'),
args=parsed_step.get('Args'),
name=name,
action_on_failure=action_on_failure,
main_class=parsed_step.get('MainClass'),
properties=emrutils.parse_key_value_string(
parsed_step.get('Properties')))