Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
raw_suffix_list_data = fetch_file(self.suffix_list_urls)
tlds = get_tlds_from_raw_suffix_list_data(raw_suffix_list_data)
if not tlds:
if self.fallback_to_snapshot:
with closing(pkg_resources.resource_stream(__name__, '.tld_set_snapshot')) as snapshot_file:
self._extractor = _PublicSuffixListTLDExtractor(pickle.load(snapshot_file))
return self._extractor
else:
raise Exception("tlds is empty, but fallback_to_snapshot is set"
" to false. Cannot proceed without tlds.")
LOG.info("computed TLDs: [%s, ...]", ', '.join(list(tlds)[:10]))
if LOG.isEnabledFor(logging.DEBUG):
import difflib
with closing(pkg_resources.resource_stream(__name__, '.tld_set_snapshot')) as snapshot_file:
snapshot = sorted(pickle.load(snapshot_file))
new = sorted(tlds)
for line in difflib.unified_diff(snapshot, new, fromfile=".tld_set_snapshot", tofile=self.cache_file):
if sys.version_info < (3,):
sys.stderr.write(line.encode('utf-8') + "\n")
else:
sys.stderr.write(line + "\n")
if self.cache_file:
try:
with open(self.cache_file, 'wb') as f:
pickle.dump(tlds, f)
except IOError as e:
LOG.warn("unable to cache TLDs in file %s: %s", self.cache_file, e)
self._extractor = _PublicSuffixListTLDExtractor(tlds)