Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)