Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def validate_and_find_master_dns(session, parsed_globals, cluster_id):
"""
Utility method for ssh, socks, put and get command.
Check if the cluster to be connected to is
terminated or being terminated.
Check if the cluster is running.
Find master instance public dns of a given cluster.
Return the latest created master instance public dns name.
Throw MasterDNSNotAvailableError or ClusterTerminatedError.
"""
cluster_state = emrutils.get_cluster_state(
session, parsed_globals, cluster_id)
if cluster_state in constants.TERMINATED_STATES:
raise exceptions.ClusterTerminatedError
emr = emrutils.get_client(session, parsed_globals)
try:
cluster_running_waiter = emr.get_waiter('cluster_running')
if cluster_state in constants.STARTING_STATES:
print("Waiting for the cluster to start.")
cluster_running_waiter.wait(ClusterId=cluster_id)
except WaiterError:
raise exceptions.MasterDNSNotAvailableError
return emrutils.find_master_public_dns(
session=session, cluster_id=cluster_id,
parsed_globals=parsed_globals)
keys = instance_group.keys()
if 'Name' in keys:
ig_config['Name'] = instance_group['Name']
else:
ig_config['Name'] = instance_group['InstanceGroupType']
ig_config['InstanceType'] = instance_group['InstanceType']
ig_config['InstanceCount'] = instance_group['InstanceCount']
ig_config['InstanceRole'] = instance_group['InstanceGroupType'].upper()
if 'BidPrice' in keys:
if instance_group['BidPrice'] != 'OnDemandPrice':
ig_config['BidPrice'] = instance_group['BidPrice']
ig_config['Market'] = constants.SPOT
else:
ig_config['Market'] = constants.ON_DEMAND
if 'EbsConfiguration' in keys:
ig_config['EbsConfiguration'] = instance_group['EbsConfiguration']
if 'AutoScalingPolicy' in keys:
ig_config['AutoScalingPolicy'] = instance_group['AutoScalingPolicy']
instance_groups.append(ig_config)
return instance_groups
def build_streaming_step(parsed_step):
name = _apply_default_value(
arg=parsed_step.get('Name'),
value=constants.DEFAULT_STREAMING_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
args = parsed_step.get('Args')
emrutils.check_required_field(
structure=constants.STREAMING_STEP_CONFIG,
name='Args',
value=args)
emrutils.check_empty_string_list(name='Args', value=args)
return emrutils.build_step(
jar=constants.STREAMING_JAR_PATH,
args=args,
name=name,
action_on_failure=action_on_failure)
def _build_hbase_disable_backups_args(self, parsed_args):
args = [constants.HBASE_MAIN, constants.HBASE_SCHEDULED_BACKUP,
constants.FALSE]
if parsed_args.full is False and parsed_args.incremental is False:
error_message = 'Should specify at least one of --' +\
constants.FULL + ' and --' +\
constants.INCREMENTAL + '.'
raise ValueError(error_message)
if parsed_args.full is True:
args.append(constants.HBASE_DISABLE_FULL_BACKUP)
if parsed_args.incremental is True:
args.append(constants.HBASE_DISABLE_INCREMENTAL_BACKUP)
return args
def _build_instance_group(
instance_type, instance_count, instance_group_type):
ig_config = {}
ig_config['InstanceType'] = instance_type
ig_config['InstanceCount'] = instance_count
ig_config['InstanceRole'] = instance_group_type.upper()
ig_config['Name'] = ig_config['InstanceRole']
ig_config['Market'] = constants.ON_DEMAND
return ig_config
def _build_impala_install_bootstrap_action(region, args=None):
args_list = [
constants.BASE_PATH_ARG,
emrutils.build_s3_link(region=region),
constants.IMPALA_VERSION,
constants.LATEST]
if args is not None:
args_list.append(constants.IMPALA_CONF)
args_list.append(','.join(args))
return emrutils.build_bootstrap_action(
name=constants.INSTALL_IMPALA_NAME,
path=emrutils.build_s3_link(
relative_path=constants.IMPALA_INSTALL_PATH,
region=region),
args=args_list)
def _build_install_hive_step(region,
action_on_failure=constants.TERMINATE_CLUSTER):
step_args = [
emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
constants.INSTALL_HIVE_ARG,
constants.BASE_PATH_ARG,
emrutils.build_s3_link(constants.HIVE_BASE_PATH, region),
constants.HIVE_VERSIONS,
constants.LATEST]
step = emrutils.build_step(
name=constants.INSTALL_HIVE_NAME,
action_on_failure=action_on_failure,
jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
args=step_args)
return step
def _run_main_command(self, parsed_args, parsed_globals):
steps = []
args = hbaseutils.build_hbase_restore_from_backup_args(
parsed_args.dir, parsed_args.backup_version)
step_config = emrutils.build_step(
jar=constants.HBASE_JAR_PATH,
name=constants.HBASE_RESTORE_STEP_NAME,
action_on_failure=constants.CANCEL_AND_WAIT,
args=args)
steps.append(step_config)
parameters = {'JobFlowId': parsed_args.cluster_id,
'Steps': steps}
emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
parameters, parsed_globals)
return 0
def _build_enable_debugging(self, parsed_args, parsed_globals):
if parsed_args.release_label:
jar = constants.COMMAND_RUNNER
args = [constants.DEBUGGING_COMMAND]
else:
jar = emrutils.get_script_runner(self.region)
args = [emrutils.build_s3_link(
relative_path=constants.DEBUGGING_PATH,
region=self.region)]
return emrutils.build_step(
name=constants.DEBUGGING_NAME,
action_on_failure=constants.TERMINATE_CLUSTER,
jar=jar,
args=args)
def build_step_config_list(parsed_step_list, region):
step_config_list = []
for step in parsed_step_list:
step_type = step.get('Type')
if step_type is None:
step_type = constants.CUSTOM_JAR
step_type = step_type.lower()
step_config = {}
if step_type == constants.CUSTOM_JAR:
step_config = build_custom_jar_step(parsed_step=step)
elif step_type == constants.STREAMING:
step_config = build_streaming_step(parsed_step=step)
elif step_type == constants.HIVE:
step_config = build_hive_step(parsed_step=step, region=region)
elif step_type == constants.PIG:
step_config = build_pig_step(parsed_step=step, region=region)
elif step_type == constants.IMPALA:
step_config = build_impala_step(parsed_step=step, region=region)
else:
raise exceptions.UnknownStepTypeError(step_type=step_type)
step_config_list.append(step_config)
return step_config_list