Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_validate_invalid_download_source():
with pytest.raises(Error):
validate_download_source("https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-2.8.3/hadoop-2.8.3.tar.gz")
raises=Error,
)
def test_validate_valid_download_source():
validate_download_source("https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz")
validate_download_source("https://www.apache.org/dyn/closer.lua?action=download&filename=spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz")
"""
Get the block device map we should assign to instances launched from a given AMI.
This is how we configure storage on the instance.
"""
ec2 = boto3.resource(service_name='ec2', region_name=region)
block_device_mappings = []
try:
image = list(
ec2.images.filter(
Filters=[
{'Name': 'image-id', 'Values': [ami]}
]))[0]
except IndexError as e:
raise Error(
"Error: Could not find {ami} in region {region}.".format(
ami=ami,
region=region))
if image.root_device_type == 'ebs':
root_device = [
device for device in image.block_device_mappings
if device['DeviceName'] == image.root_device_name][0]
if root_device['Ebs']['VolumeSize'] < min_root_ebs_size_gb:
root_device['Ebs'].update({
# Max root volume size for instance store-backed AMIs is 10 GiB.
# See: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/add-instance-store-volumes.html
# Though, this code is probably incorrect for instance store-backed
# instances anyway, since boto3 doesn't seem to let you set the size
# of a root instance store volume.
'VolumeSize': min_root_ebs_size_gb,
class UnsupportedProviderError(UsageError):
def __init__(self, provider: str):
super().__init__(
"This provider is not supported: {p}".format(p=provider))
self.provider = provider
class Error(Exception):
pass
class ClusterNotFound(Error):
pass
class ClusterAlreadyExists(Error):
pass
class ClusterInvalidState(Error):
def __init__(self, *, attempted_command: str, state: str):
super().__init__(
"Cluster is in state '{s}'. Cannot execute {c}.".format(
c=attempted_command,
s=state))
self.attempted_command = attempted_command
self.state = state
class SSHError(Error):
def __init__(self, *, host: str, message: str):
super().__init__(
def check_external_dependency(executable_name: str):
if shutil.which(executable_name) is None:
raise Error(
"Error: Flintrock could not find '{executable}' on your PATH. "
"Flintrock needs this executable to carry out the operation you "
"requested. Please install it and try again."
.format(
executable=executable_name
)
cluster = ec2.get_cluster(
cluster_name=cluster_name,
region=ec2_region,
vpc_id=ec2_vpc_id)
user = ec2_user
identity_file = ec2_identity_file
provider_options = {
'min_root_ebs_size_gb': ec2_min_root_ebs_size_gb,
'spot_price': ec2_spot_price,
'tags': ec2_tags
}
else:
raise UnsupportedProviderError(provider)
if cluster.num_masters == 0:
raise Error(
"Cannot add slaves to cluster '{c}' since it does not "
"appear to have a master."
.format(
c=cluster_name))
cluster.load_manifest(
user=user,
identity_file=identity_file)
cluster.add_slaves_check()
if provider == 'ec2':
cluster.add_slaves(
user=user,
identity_file=identity_file,
num_slaves=num_slaves,
assume_yes=assume_yes,
class ClusterAlreadyExists(Error):
pass
class ClusterInvalidState(Error):
def __init__(self, *, attempted_command: str, state: str):
super().__init__(
"Cluster is in state '{s}'. Cannot execute {c}.".format(
c=attempted_command,
s=state))
self.attempted_command = attempted_command
self.state = state
class SSHError(Error):
def __init__(self, *, host: str, message: str):
super().__init__(
"[{h}] {m}".format(h=host, m=message))
self.host = host
self.message = message
class InterruptedEC2Operation(Error):
def __init__(self, *, instances: list):
super().__init__(
"Operation aborted."
)
self.instances = instances
class UsageError(Exception):
pass
class UnsupportedProviderError(UsageError):
def __init__(self, provider: str):
super().__init__(
"This provider is not supported: {p}".format(p=provider))
self.provider = provider
class Error(Exception):
pass
class ClusterNotFound(Error):
pass
class ClusterAlreadyExists(Error):
pass
class ClusterInvalidState(Error):
def __init__(self, *, attempted_command: str, state: str):
super().__init__(
"Cluster is in state '{s}'. Cannot execute {c}.".format(
c=attempted_command,
s=state))
self.attempted_command = attempted_command
self.state = state
if parsed_url.netloc == 'www.apache.org' and parsed_url.path == '/dyn/closer.lua':
logger.warning(
"Warning: "
"Downloading {software} from an Apache mirror. Apache mirrors are "
"often slow and unreliable, and typically only serve the most recent releases. "
"We strongly recommend you specify a custom download source. "
"For more background on this issue, please see: https://github.com/nchammas/flintrock/issues/238"
.format(
software=software,
)
)
try:
urllib.request.urlopen(url)
except urllib.error.HTTPError as e:
raise Error(
"Error: Could not access {software} download. Maybe try a more recent release?\n"
" - Automatically redirected to: {url}\n"
" - HTTP error: {code}"
.format(
software=software,
url=e.url,
code=e.code,
)