Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def stream_from_pbox(itemname, filename):
# TODO: handle errors etc
archive_session = get_session(config_file=settings.IATOOL_CONFIG_PATH)
item = archive_session.get_item(itemname)
files = item.get_files(filename)
file = files.__next__()
return file.download(return_responses=True)
def start_ia_session(self):
""" starts an internet archive session """
config = dict(s3=dict(acccess=settings.INTERNET_ARCHIVE_ACCESS_KEY,
secret=settings.INTERNET_ARCHIVE_SECRET_KEY))
s = get_session(config=config,
debug=True)
s.access_key = settings.INTERNET_ARCHIVE_ACCESS_KEY
s.secret_key = settings.INTERNET_ARCHIVE_SECRET_KEY
return s
def start_ia_session(self):
""" starts an internet archive session """
config = dict(s3=dict(acccess=settings.INTERNET_ARCHIVE_ACCESS_KEY,
secret=settings.INTERNET_ARCHIVE_SECRET_KEY))
s = get_session(config=config,
debug=True)
s.access_key = settings.INTERNET_ARCHIVE_ACCESS_KEY
s.secret_key = settings.INTERNET_ARCHIVE_SECRET_KEY
return s
def __init__(self, top_dir, access_key, secret_key, loglevel, logfile):
self.top_dir = top_dir
self.access_key = access_key
self.secret_key = secret_key
self.headers = {'x-archive-keep-old-version': '0'}
session_data = {'access': access_key, 'secret': secret_key}
if logfile:
logconfig = {'logging': {'level': loglevel, 'file': logfile}}
else:
logconfig = {'logging': {'level': loglevel}}
self.session = get_session({'s3': session_data, 'logging': logconfig})
self.logger = logging.getLogger('gvision.ia')
from datetime import timedelta
from django.utils import timezone
import internetarchive
from perma.models import Link
from django.template.defaultfilters import truncatechars
start_date = timezone.now() - timedelta(days=3)
end_date = timezone.now() - timedelta(days=2)
links = Link.objects.filter(internet_archive_upload_status="completed", creation_timestamp__range=(start_date, end_date))
guid_results = dict()
all_results = dict()
c = {"s3":{"access":settings.INTERNET_ARCHIVE_ACCESS_KEY, "secret":settings.INTERNET_ARCHIVE_SECRET_KEY}}
internetarchive.get_session(config=c)
for link in links:
identifier = settings.INTERNET_ARCHIVE_IDENTIFIER_PREFIX + link.guid
item = internetarchive.get_item(identifier)
warc_name = "%s.warc.gz" % link.guid
try:
fnames = [f.name for f in internetarchive.get_files(identifier, glob_pattern="*gz")]
guid_results["uploaded_file"] = warc_name in fnames
if settings.INTERNET_ARCHIVE_COLLECTION == 'test_collection':
guid_results["collection"] = item.metadata["collection"] == settings.INTERNET_ARCHIVE_COLLECTION
else:
guid_results["collection"] = item.metadata["collection"][0] == settings.INTERNET_ARCHIVE_COLLECTION
guid_results["title"] = item.metadata["title"] == "%s: %s" % (link.guid, truncatechars(link.submitted_title, 50))
guid_results["mediatype"] = item.metadata["mediatype"]=="web"
guid_results["description"] = item.metadata["description"]=="Perma.cc archive of %s created on %s." % (link.submitted_url, link.creation_timestamp,)