Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def get_page(self, url):
# http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api
scheme, netloc, path, _, _, _ = compat.urlparse(url)
if scheme == 'file' and os.path.isdir(url2pathname(path)):
url = urljoin(ensure_slash(url), 'index.html')
# The `retrieve()` method follows any eventual redirects, so the
# initial url might be different from the final one
try:
response, final_url = self.opener.retrieve(url)
except urllib3.exceptions.MaxRetryError:
return
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)
def get_page(self, url):
# http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api
scheme, netloc, path, _, _, _ = compat.urlparse(url)
if scheme == 'file' and os.path.isdir(url2pathname(path)):
url = compat.urljoin(ensure_slash(url), 'index.html')
# The `retrieve()` method follows any eventual redirects, so the
# initial url might be different from the final one
try:
response, final_url = http_retrieve(self.opener, url)
except urllib3.exceptions.MaxRetryError:
return
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)