Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_adding_custom_datasource(self):
collectionid_datasourcename_wfsid = (
('0000d273-7e89-4f00-971e-9025f89a0000', 'BYOC_0000d273-7e89-4f00-971e-9025f89a0000',
'DSS10-0000d273-7e89-4f00-971e-9025f89a0000'),
)
for collection_id, data_source_name, wfs_id in collectionid_datasourcename_wfsid:
datasource = DataSource(collection_id)
wfsid_tested = DataSource.get_wfs_typename(datasource)
self.assertEqual(data_source_name, datasource.name, msg="Expected {}, got {}".
format(data_source_name, datasource.name))
self.assertEqual(wfs_id, wfsid_tested, msg="Expected {}, got {}".format(wfs_id, wfsid_tested))
self.assertTrue(datasource in DataSource.get_available_sources(), msg='Datasource should be in the list'
'of all datasources')
self.assertTrue(datasource in DataSource.get_custom_sources(), msg='Datasource should be in the list'
'of custom datasources')
def _band_exists(self, band_name):
if self.data_source is DataSource.SENTINEL2_L1C:
return True
resolution, band = band_name.split('/')
if self.safe_type is EsaSafeType.COMPACT_TYPE:
return not (self.baseline >= '02.07' and band.endswith(AwsConstants.VIS))
return band != AwsConstants.TCI and not (band == AwsConstants.SCL and resolution == AwsConstants.R60m)
safe[main_folder] = {}
safe[main_folder][AwsConstants.AUX_DATA] = {}
# Not sure if 2nd condition of the following is correct:
if self.data_source is not DataSource.SENTINEL2_L1C or self.baseline != '02.04':
ecmwft_file = AwsConstants.ECMWFT if self.data_source is DataSource.SENTINEL2_L1C or \
self.safe_type is EsaSafeType.OLD_TYPE else AwsConstants.AUX_ECMWFT
safe[main_folder][AwsConstants.AUX_DATA][self.get_aux_data_name()] = self.get_url(ecmwft_file)
# Old products also have DEM and MSI in aux folder
if self.is_early_compact_l2a():
safe[main_folder][AwsConstants.AUX_DATA][self.add_file_extension(AwsConstants.GIPP, remove_path=True)] =\
self.get_url(AwsConstants.GIPP)
safe[main_folder][AwsConstants.IMG_DATA] = {}
if self.data_source is DataSource.SENTINEL2_L1C:
for band in self.bands:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(band)] = self.get_url(band)
if self.safe_type == EsaSafeType.COMPACT_TYPE:
safe[main_folder][AwsConstants.IMG_DATA][self.get_img_name(AwsConstants.TCI)] =\
self.get_url(AwsConstants.TCI)
else:
for resolution in AwsConstants.RESOLUTIONS:
safe[main_folder][AwsConstants.IMG_DATA][resolution] = {}
for band_name in self.bands:
resolution, band = band_name.split('/')
if self._band_exists(band_name):
safe[main_folder][AwsConstants.IMG_DATA][resolution][self.get_img_name(band, resolution)] =\
self.get_url(band_name)
safe[main_folder][AwsConstants.QI_DATA] = {}
safe[main_folder][AwsConstants.QI_DATA][self.get_qi_name('CLOUDS')] = self.get_gml_url('CLOUDS')
def __init__(self, shape_list, crs, time_interval, tile_split_shape=1, data_source=DataSource.SENTINEL2_L1C,
instance_id=None, **kwargs):
super().__init__(shape_list, crs, **kwargs)
if data_source is DataSource.DEM:
raise ValueError('This splitter does not support splitting area by DEM tiles. Please specify some other '
'DataSource')
self.time_interval = time_interval
self.tile_split_shape = tile_split_shape
self.data_source = data_source
self.instance_id = instance_id
self.tile_dict = None
self._make_split()
\b
Examples with Sentinel-2 L1C data:
sentinelhub.aws --product S2A_MSIL1C_20170414T003551_N0204_R016_T54HVH_20170414T003551
sentinelhub.aws --product S2A_MSIL1C_20170414T003551_N0204_R016_T54HVH_20170414T003551 -i
sentinelhub.aws --product S2A_MSIL1C_20170414T003551_N0204_R016_T54HVH_20170414T003551 -f /home/ESA_Products
sentinelhub.aws --product S2A_MSIL1C_20170414T003551_N0204_R016_T54HVH_20170414T003551 --bands B08,B11
sentinelhub.aws --tile T54HVH 2017-04-14
sentinelhub.aws --tile T54HVH 2017-04-14 -e
\b
Examples with Sentinel-2 L2A data:
sentinelhub.aws --product S2A_MSIL2A_20180402T151801_N0207_R068_T33XWJ_20180402T202222
sentinelhub.aws --tile T33XWJ 2018-04-02 --l2a
"""
band_list = None if bands is None else bands.split(',')
data_source = DataSource.SENTINEL2_L2A if l2a else DataSource.SENTINEL2_L1C
if info:
if product is None:
click.echo(get_safe_format(tile=tile, entire_product=entire, data_source=data_source))
else:
click.echo(get_safe_format(product_id=product))
else:
if product is None:
download_safe_format(tile=tile, folder=folder, redownload=redownload, entire_product=entire,
bands=band_list, data_source=data_source)
else:
download_safe_format(product_id=product, folder=folder, redownload=redownload, bands=band_list)
def get_base_url(self, request):
""" Creates base url string.
:param request: OGC-type request with specified bounding box, cloud coverage for specific product.
:type request: OgcRequest or GeopediaRequest
:return: base string for url to Sentinel Hub's OGC service for this product.
:rtype: str
"""
url = self.base_url + request.service_type.value
# These 2 lines are temporal and will be removed after the use of uswest url wont be required anymore:
if hasattr(request, 'data_source') and request.data_source.is_uswest_source():
url = 'https://services-uswest2.sentinel-hub.com/ogc/{}'.format(request.service_type.value)
if hasattr(request, 'data_source') and request.data_source not in DataSource.get_available_sources():
raise ValueError("{} is not available for service at ogc_base_url={}".format(request.data_source,
SHConfig().ogc_base_url))
return url
safe[main_folder][AwsConstants.QI_DATA][self.get_qi_name('CLOUDS')] = self.get_gml_url('CLOUDS')
for qi_type in AwsConstants.QI_LIST:
for band in AwsConstants.S2_L1C_BANDS:
safe[main_folder][AwsConstants.QI_DATA][self.get_qi_name(qi_type, band)] = self.get_gml_url(qi_type,
band)
if self.has_reports():
for metafile in AwsConstants.QUALITY_REPORTS:
if metafile == AwsConstants.RADIOMETRIC_QUALITY and self.data_source is DataSource.SENTINEL2_L2A and \
self.baseline <= '02.07':
continue
metafile_name = self.add_file_extension(metafile, remove_path=True)
safe[main_folder][AwsConstants.QI_DATA][metafile_name] = self.get_qi_url(metafile_name)
if self.data_source is DataSource.SENTINEL2_L2A:
for mask in AwsConstants.CLASS_MASKS:
for resolution in [AwsConstants.R20m, AwsConstants.R60m]:
if self.baseline <= '02.06':
mask_name = self.get_img_name(mask, resolution)
else:
mask_name = self.get_qi_name('{}PRB'.format(mask), resolution.lstrip('R'), MimeType.JP2)
safe[main_folder][AwsConstants.QI_DATA][mask_name] =\
self.get_qi_url('{}_{}.jp2'.format(mask, resolution.lstrip('R')))
if self.is_early_compact_l2a():
safe[main_folder][AwsConstants.QI_DATA][self.get_img_name(AwsConstants.PVI)] = self.get_preview_url('L2A')
preview_type = 'L2A' if self.data_source is DataSource.SENTINEL2_L2A and self.baseline >= '02.07' else 'L1C'
safe[main_folder][AwsConstants.QI_DATA][self.get_preview_name()] = self.get_preview_url(preview_type)
safe[main_folder][self.get_tile_metadata_name()] = self.get_url(AwsConstants.METADATA)
import logging
import os
import warnings
from abc import ABC, abstractmethod
from .config import SHConfig
from .constants import AwsConstants, EsaSafeType, MimeType, DataSource
from .download import DownloadRequest, get_json, AwsDownloadFailedException
from .opensearch import get_tile_info, get_tile_info_id
from .time_utils import parse_time
LOGGER = logging.getLogger(__name__)
MAX_SUPPORTED_BASELINES = {
DataSource.SENTINEL2_L1C: '02.07',
DataSource.SENTINEL2_L2A: '02.11'
}
class AwsService(ABC):
""" Amazon Web Service (AWS) base class
"""
def __init__(self, parent_folder='', bands=None, metafiles=None):
"""
:param parent_folder: Folder where the fetched data will be saved.
:type parent_folder: str
:param bands: List of Sentinel-2 bands for request. If parameter is set to `None` all bands will be used.
:type bands: list(str) or None
:param metafiles: List of additional metafiles available on AWS
(e.g. ``['metadata', 'tileInfo', 'preview/B01', 'TCI']``).
If parameter is set to `None` the list will be set automatically.
def get_aws_index(self):
"""
Returns tile index on AWS. If `tile_index` was not set during class initialization it will be determined
according to existing tiles on AWS.
:return: Index of tile on AWS
:rtype: int
"""
if self.aws_index is not None:
return self.aws_index
tile_info_list = get_tile_info(self.tile_name, self.datetime, all_tiles=True)
if not tile_info_list:
raise ValueError('Cannot find aws_index for specified tile and time')
if self.data_source is DataSource.SENTINEL2_L2A:
for tile_info in sorted(tile_info_list, key=self._parse_aws_index):
try:
self.aws_index = self._parse_aws_index(tile_info)
self.get_tile_info()
return self.aws_index
except AwsDownloadFailedException:
pass
return self._parse_aws_index(tile_info_list[0])