Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _config_with_environment_variables():
""" Reads configuration parameters from environmental variables
"""
config = SHConfig()
for param in config.get_params():
env_variable = param.upper()
if os.environ.get(env_variable):
setattr(config, param, os.environ.get(env_variable))
config.save()
return config
def _do_aws_request(request):
""" Executes download request from AWS service
:param request: A request
:type request: DownloadRequest
:return: Response of the request
:rtype: dict
:raises: AwsDownloadFailedException, ValueError
"""
if SHConfig().aws_access_key_id and SHConfig().aws_secret_access_key:
key_args = dict(aws_access_key_id=SHConfig().aws_access_key_id,
aws_secret_access_key=SHConfig().aws_secret_access_key)
else:
key_args = {}
aws_service, _, bucket_name, url_key = request.url.split('/', 3)
try:
s3_client = boto3.Session().client(aws_service.strip(':'), **key_args)
request.s3_client = s3_client # Storing the client prevents warning about unclosed socket
DownloadRequest.GLOBAL_AWS_CLIENT = s3_client
except KeyError: # Sometimes creation of client fails and we use the global client if it exists
if DownloadRequest.GLOBAL_AWS_CLIENT is None:
raise ValueError('Failed to create a client for download from AWS')
s3_client = DownloadRequest.GLOBAL_AWS_CLIENT
try:
return s3_client.get_object(Bucket=bucket_name, Key=url_key, RequestPayer='requester')
def _do_aws_request(request):
""" Executes download request from AWS service
:param request: A request
:type request: DownloadRequest
:return: Response of the request
:rtype: dict
:raises: AwsDownloadFailedException, ValueError
"""
if SHConfig().aws_access_key_id and SHConfig().aws_secret_access_key:
key_args = dict(aws_access_key_id=SHConfig().aws_access_key_id,
aws_secret_access_key=SHConfig().aws_secret_access_key)
else:
key_args = {}
aws_service, _, bucket_name, url_key = request.url.split('/', 3)
try:
s3_client = boto3.Session().client(aws_service.strip(':'), **key_args)
request.s3_client = s3_client # Storing the client prevents warning about unclosed socket
DownloadRequest.GLOBAL_AWS_CLIENT = s3_client
except KeyError: # Sometimes creation of client fails and we use the global client if it exists
if DownloadRequest.GLOBAL_AWS_CLIENT is None:
raise ValueError('Failed to create a client for download from AWS')
s3_client = DownloadRequest.GLOBAL_AWS_CLIENT
try:
return s3_client.get_object(Bucket=bucket_name, Key=url_key, RequestPayer='requester')
except NoCredentialsError:
def __init__(self, base_url=None, instance_id=None):
"""
:param base_url: base url of Sentinel Hub's OGC services. If `None`, the url specified in the configuration
file is taken.
:type base_url: str or None
:param instance_id: user's instance id granting access to Sentinel Hub's OGC services. If `None`, the instance
ID specified in the configuration file is taken.
:type instance_id: str or None
"""
self.base_url = SHConfig().ogc_base_url if not base_url else base_url
self.instance_id = SHConfig().instance_id if not instance_id else instance_id
if not self.instance_id:
raise ValueError('Instance ID is not set. '
'Set it either in request initialization or in configuration file. '
:type absolute_orbit: int
:return: An iterator returning dictionaries with info provided by Sentinel Hub OpenSearch REST service
:rtype: Iterator[dict]
"""
if bbox and bbox.crs is not CRS.WGS84:
bbox = bbox.transform(CRS.WGS84)
url_params = _prepare_url_params(tile_id, bbox, end_date, start_date, absolute_orbit)
url_params['maxRecords'] = SHConfig().max_opensearch_records_per_query
start_index = 1
while True:
url_params['index'] = start_index
url = '{}search.json?{}'.format(SHConfig().opensearch_url, urlencode(url_params))
LOGGER.debug("URL=%s", url)
response = get_json(url)
for tile_info in response["features"]:
yield tile_info
if len(response["features"]) < SHConfig().max_opensearch_records_per_query:
break
start_index += SHConfig().max_opensearch_records_per_query
def get_base_url(self, force_http=False):
""" Creates base URL path
:param force_http: `True` if HTTP base URL should be used and `False` otherwise
:type force_http: str
:return: base url string
:rtype: str
"""
base_url = SHConfig().aws_metadata_url.rstrip('/') if force_http else 's3:/'
aws_bucket = SHConfig().aws_s3_l1c_bucket if self.data_source is DataSource.SENTINEL2_L1C else \
SHConfig().aws_s3_l2a_bucket
return '{}/{}/'.format(base_url, aws_bucket)
def __init__(self, base_url=None, instance_id=None):
"""
:param base_url: base url of Sentinel Hub's OGC services. If `None`, the url specified in the configuration
file is taken.
:type base_url: str or None
:param instance_id: user's instance id granting access to Sentinel Hub's OGC services. If `None`, the instance
ID specified in the configuration file is taken.
:type instance_id: str or None
"""
self.base_url = SHConfig().ogc_base_url if not base_url else base_url
self.instance_id = SHConfig().instance_id if not instance_id else instance_id
if not self.instance_id:
raise ValueError('Instance ID is not set. '
'Set it either in request initialization or in configuration file. '
LOGGER.debug('Successful download from %s', request.url)
break
except requests.RequestException as exception:
try_num -= 1
if try_num > 0 and (_is_temporal_problem(exception) or
(isinstance(exception, requests.HTTPError) and
exception.response.status_code >= requests.status_codes.codes.INTERNAL_SERVER_ERROR) or
_request_limit_reached(exception)):
LOGGER.debug('Download attempt failed: %s\n%d attempts left, will retry in %ds', exception,
try_num, SHConfig().download_sleep_time)
sleep_time = SHConfig().download_sleep_time
if _request_limit_reached(exception):
sleep_time = max(sleep_time, 60)
time.sleep(sleep_time)
else:
if request.url.startswith(SHConfig().aws_metadata_url) and \
isinstance(exception, requests.HTTPError) and \
exception.response.status_code == requests.status_codes.codes.NOT_FOUND:
raise AwsDownloadFailedException('File in location %s is missing' % request.url)
raise DownloadFailedException(_create_download_failed_message(exception, request.url))
_save_if_needed(request, response_content)
if request.return_data:
return decode_data(response_content, request.data_type, entire_response=response)
return None
def get_base_url(self, force_http=False):
""" Creates base URL path
:param force_http: `True` if HTTP base URL should be used and `False` otherwise
:type force_http: str
:return: base url string
:rtype: str
"""
base_url = SHConfig().aws_metadata_url.rstrip('/') if force_http else 's3:/'
aws_bucket = SHConfig().aws_s3_l1c_bucket if self.data_source is DataSource.SENTINEL2_L1C else \
SHConfig().aws_s3_l2a_bucket
return '{}/{}/'.format(base_url, aws_bucket)
url_params = _prepare_url_params(tile_id, bbox, end_date, start_date, absolute_orbit)
url_params['maxRecords'] = SHConfig().max_opensearch_records_per_query
start_index = 1
while True:
url_params['index'] = start_index
url = '{}search.json?{}'.format(SHConfig().opensearch_url, urlencode(url_params))
LOGGER.debug("URL=%s", url)
response = get_json(url)
for tile_info in response["features"]:
yield tile_info
if len(response["features"]) < SHConfig().max_opensearch_records_per_query:
break
start_index += SHConfig().max_opensearch_records_per_query