Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
safe[main_folder][AwsConstants.AUX_DATA][self.get_aux_data_name()] = self.get_url(ecmwft_file)
# Old products also have DEM and MSI in aux folder
if self.is_early_compact_l2a():
safe[main_folder][AwsConstants.AUX_DATA][self.add_file_extension(AwsConstants.GIPP, remove_path=True)] =\
self.get_url(AwsConstants.GIPP)
safe[main_folder][AwsConstants.IMG_DATA] = {}
if self.data_source is DataSource.SENTINEL2_L1C:
for band in self.bands:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(band)] = self.get_url(band)
if self.safe_type == EsaSafeType.COMPACT_TYPE:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(AwsConstants.TCI)] =\
self.get_url(AwsConstants.TCI)
else:
for resolution in AwsConstants.RESOLUTIONS:
safe[main_folder][AwsConstants.IMG_DATA][resolution] = {}
for band_name in self.bands:
resolution, band = band_name.split('/')
if self._band_exists(band_name):
safe[main_folder][AwsConstants.IMG_DATA][resolution][self.get_img_name(band, resolution)] =\
self.get_url(band_name)
safe[main_folder][AwsConstants.QI_DATA] = {}
safe[main_folder][AwsConstants.QI_DATA][self.get_qi_name('CLOUDS')] = self.get_gml_url('CLOUDS')
for qi_type in AwsConstants.QI_LIST:
for band in AwsConstants.S2_L1C_BANDS:
safe[main_folder][AwsConstants.QI_DATA][self.get_qi_name(qi_type, band)] = self.get_gml_url(qi_type,
band)
if self.has_reports():
for metafile in AwsConstants.QUALITY_REPORTS:
def get_report_time(self):
""" Returns time when the L2A processing started and reports was created.
:return: String in a form YYYYMMDDTHHMMSS
:rtype: str
"""
tree = get_xml(self.get_url(AwsConstants.REPORT))
try:
timestamp = tree.find('check/inspection').attrib['execution']
return timestamp.split(',')[0].replace(' ', 'T').replace(':', '').replace('-', '')
except AttributeError:
warnings.warn('Could not obtain the L2A report creation time')
return 'unknown'
if self.data_source is not DataSource.SENTINEL2_L1C or self.baseline != '02.04':
ecmwft_file = AwsConstants.ECMWFT if self.data_source is DataSource.SENTINEL2_L1C or \
self.safe_type is EsaSafeType.OLD_TYPE else AwsConstants.AUX_ECMWFT
safe[main_folder][AwsConstants.AUX_DATA][self.get_aux_data_name()] = self.get_url(ecmwft_file)
# Old products also have DEM and MSI in aux folder
if self.is_early_compact_l2a():
safe[main_folder][AwsConstants.AUX_DATA][self.add_file_extension(AwsConstants.GIPP, remove_path=True)] =\
self.get_url(AwsConstants.GIPP)
safe[main_folder][AwsConstants.IMG_DATA] = {}
if self.data_source is DataSource.SENTINEL2_L1C:
for band in self.bands:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(band)] = self.get_url(band)
if self.safe_type == EsaSafeType.COMPACT_TYPE:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(AwsConstants.TCI)] =\
self.get_url(AwsConstants.TCI)
else:
for resolution in AwsConstants.RESOLUTIONS:
safe[main_folder][AwsConstants.IMG_DATA][resolution] = {}
for band_name in self.bands:
resolution, band = band_name.split('/')
if self._band_exists(band_name):
safe[main_folder][AwsConstants.IMG_DATA][resolution][self.get_img_name(band, resolution)] =\
self.get_url(band_name)
safe[main_folder][AwsConstants.QI_DATA] = {}
safe[main_folder][AwsConstants.QI_DATA][self.get_qi_name('CLOUDS')] = self.get_gml_url('CLOUDS')
for qi_type in AwsConstants.QI_LIST:
for band in AwsConstants.S2_L1C_BANDS:
safe[main_folder][AwsConstants.QI_DATA][self.get_qi_name(qi_type, band)] = self.get_gml_url(qi_type,
band)
def get_preview_name(self):
""" Returns .SAFE name of full resolution L1C preview
:return: name of preview file
:rtype: str
"""
if self.safe_type == EsaSafeType.OLD_TYPE:
name = _edit_name(self.tile_id, AwsConstants.PVI, delete_end=True)
else:
name = '_'.join([self.tile_id.split('_')[1], self.get_datatake_time(), AwsConstants.PVI])
return '{}.jp2'.format(name)
def get_url(self, filename, data_format=None):
""" Creates url of file location on AWS.
:param filename: name of file
:type filename: str
:param data_format: format of file, if `None` it will be set automatically
:type data_format: constants.MimeType or None
:return: url of file location
:rtype: str
"""
product_url = self.product_url
force_http = filename in [AwsConstants.PRODUCT_INFO, AwsConstants.METADATA]
if product_url is None or force_http:
product_url = self.get_product_url(force_http=force_http)
return '{}/{}'.format(product_url, self.add_file_extension(filename, data_format))
for mask in AwsConstants.CLASS_MASKS:
for resolution in [AwsConstants.R20m, AwsConstants.R60m]:
if self.baseline <= '02.06':
mask_name = self.get_img_name(mask, resolution)
else:
mask_name = self.get_qi_name('{}PRB'.format(mask), resolution.lstrip('R'), MimeType.JP2)
safe[main_folder][AwsConstants.QI_DATA][mask_name] =\
self.get_qi_url('{}_{}.jp2'.format(mask, resolution.lstrip('R')))
if self.is_early_compact_l2a():
safe[main_folder][AwsConstants.QI_DATA][self.get_img_name(AwsConstants.PVI)] = self.get_preview_url('L2A')
preview_type = 'L2A' if self.data_source is DataSource.SENTINEL2_L2A and self.baseline >= '02.07' else 'L1C'
safe[main_folder][AwsConstants.QI_DATA][self.get_preview_name()] = self.get_preview_url(preview_type)
safe[main_folder][self.get_tile_metadata_name()] = self.get_url(AwsConstants.METADATA)
return safe
def get_url(self, filename):
"""
Creates url of file location on AWS.
:param filename: name of file
:type filename: str
:return: url of file location
:rtype: str
"""
tile_url = self.tile_url
force_http = filename in [AwsConstants.TILE_INFO, AwsConstants.PRODUCT_INFO, AwsConstants.METADATA]
if tile_url is None or force_http:
tile_url = self.get_tile_url(force_http=force_http)
return '{}/{}'.format(tile_url, self.add_file_extension(filename))
def get_requests(self):
""" Creates product structure and returns list of files for download.
:return: List of download requests and list of empty folders that need to be created
:rtype: (list(download.DownloadRequest), list(str))
"""
self.download_list = [DownloadRequest(url=self.get_url(metafile), filename=self.get_filepath(metafile),
data_type=AwsConstants.AWS_FILES[metafile], data_name=metafile) for
metafile in self.metafiles if metafile in AwsConstants.PRODUCT_FILES]
tile_parent_folder = os.path.join(self.parent_folder, self.product_id)
for tile_info in self.product_info['tiles']:
tile_name, date, aws_index = self.url_to_tile(self.get_tile_url(tile_info))
if self.tile_list is None or AwsTile.parse_tile_name(tile_name) in self.tile_list:
tile_downloads, tile_folders = AwsTile(tile_name, date, aws_index, parent_folder=tile_parent_folder,
bands=self.bands, metafiles=self.metafiles,
data_source=self.data_source).get_requests()
self.download_list.extend(tile_downloads)
self.folder_list.extend(tile_folders)
self.sort_download_list()
return self.download_list, self.folder_list
def get_safe_struct(self):
""" Describes a structure inside tile folder of ESA product .SAFE structure.
:return: nested dictionaries representing .SAFE structure
:rtype: dict
"""
# pylint: disable=too-many-branches
safe = {}
main_folder = self.get_main_folder()
safe[main_folder] = {}
safe[main_folder][AwsConstants.AUX_DATA] = {}
# Not sure if 2nd condition of the following is correct:
if self.data_source is not DataSource.SENTINEL2_L1C or self.baseline != '02.04':
ecmwft_file = AwsConstants.ECMWFT if self.data_source is DataSource.SENTINEL2_L1C or \
self.safe_type is EsaSafeType.OLD_TYPE else AwsConstants.AUX_ECMWFT
safe[main_folder][AwsConstants.AUX_DATA][self.get_aux_data_name()] = self.get_url(ecmwft_file)
# Old products also have DEM and MSI in aux folder
if self.is_early_compact_l2a():
safe[main_folder][AwsConstants.AUX_DATA][self.add_file_extension(AwsConstants.GIPP, remove_path=True)] =\
self.get_url(AwsConstants.GIPP)
safe[main_folder][AwsConstants.IMG_DATA] = {}
if self.data_source is DataSource.SENTINEL2_L1C:
for band in self.bands:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(band)] = self.get_url(band)
if self.safe_type == EsaSafeType.COMPACT_TYPE:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(AwsConstants.TCI)] =\
self.get_url(AwsConstants.TCI)
else:
for resolution in AwsConstants.RESOLUTIONS:
def get_safe_struct(self):
""" Describes a structure inside tile folder of ESA product .SAFE structure
:return: nested dictionaries representing .SAFE structure
:rtype: dict
"""
safe = {}
main_folder = self.get_main_folder()
safe[main_folder] = {}
safe[main_folder][AwsConstants.AUX_DATA] = {}
safe[main_folder][AwsConstants.DATASTRIP] = {}
datastrip_list = self.get_datastrip_list()
for datastrip_folder, datastrip_url in datastrip_list:
safe[main_folder][AwsConstants.DATASTRIP][datastrip_folder] = {}
safe[main_folder][AwsConstants.DATASTRIP][datastrip_folder][AwsConstants.QI_DATA] = {}
# S-2 L1C reports are on AWS only stored with tiles and without RADIOMETRIC_QUALITY
if self.has_reports() and self.data_source is DataSource.SENTINEL2_L2A:
for metafile in AwsConstants.QUALITY_REPORTS:
metafile_name = self.add_file_extension(metafile)
safe[main_folder][AwsConstants.DATASTRIP][datastrip_folder][AwsConstants.QI_DATA][
metafile_name] = '{}/qi/{}'.format(datastrip_url, metafile_name)
safe[main_folder][AwsConstants.DATASTRIP][datastrip_folder][
self.get_datastrip_metadata_name(datastrip_folder)] = '{}/{}'.format(
datastrip_url, self.add_file_extension(AwsConstants.METADATA))