Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def upload_single(self, name, f, ia_args):
with open(settings.keys, 'r') as keys:
access_key, secret_key = keys.read().strip().split(':')
try:
internetarchive.upload('archiveteam_newssites_{name}'.format(name=name),
os.path.join(settings.dir_ready, f),
metadata=ia_args,
access_key=access_key,
secret_key=secret_key,
queue_derive=True,
verify=True,
verbose=True,
delete=True,
retries=10,
retries_sleep=300)
except:
pass # see code below
self.concurrent_uploads -= 1
os.remove(os.path.join(settings.dir_ready, f+'.upload'))
if os.path.isfile(os.path.join(settings.dir_ready, f)):
settings.irc_bot.send('PRIVMSG', '{name} uploaded unsuccessful.'.format(
def upload_ia_item(self):
logger.debug("Uploading IA item for {}".format(self.ia_id))
if not self.has_image and not self.has_crop:
logger.debug("No images to upload")
return None
files = []
if self.has_image:
saved_image = self.save_image()
files.append(saved_image)
if self.has_crop:
saved_crop = self.save_crop()
files.append(saved_crop)
internetarchive.upload(
self.ia_id,
files,
metadata=self.ia_metadata,
access_key=settings.IA_ACCESS_KEY_ID,
secret_key=settings.IA_SECRET_ACCESS_KEY,
checksum=False,
verbose=True
)
if self.has_image:
os.remove(saved_image)
if self.has_crop:
os.remove(saved_crop)
return internetarchive.get_item(self.ia_id)
width,
1000
)
)
# Move the original file to Internet Archive namespace
shutil.copy(output_path, ssht.get_image_name())
# Save crop to Internet Archive namespace
crop.save(open(ssht.get_crop_name(), 'w'))
# Upload both images to Internet Archive
files = [ssht.get_image_name(), ssht.get_crop_name()]
try:
logger.debug("Uploading to internetarchive as {}".format(ssht.ia_id))
internetarchive.upload(
ssht.ia_id,
files,
metadata=ssht.ia_metadata,
access_key=settings.IA_ACCESS_KEY_ID,
secret_key=settings.IA_SECRET_ACCESS_KEY,
checksum=False,
verbose=True
)
except Exception, e:
logger.error("internetarchive error: %s" % e)
ScreenshotLog.objects.create(
update=update,
site=site,
message_type="error",
message="internetarchive error: %s" % e
)
# if item already exists (but has been removed),
# ia won't update its metadata in upload function
if item.exists and item.metadata['title'] == 'Removed':
item.modify_metadata(metadata,
access_key=settings.INTERNET_ARCHIVE_ACCESS_KEY,
secret_key=settings.INTERNET_ARCHIVE_SECRET_KEY,
)
warc_name = os.path.basename(link.warc_storage_file())
# copy warc to local disk storage for upload
temp_warc_file = tempfile.TemporaryFile()
copy_file_data(default_storage.open(link.warc_storage_file()), temp_warc_file)
temp_warc_file.seek(0)
success = internetarchive.upload(
identifier,
{warc_name: temp_warc_file},
metadata=metadata,
access_key=settings.INTERNET_ARCHIVE_ACCESS_KEY,
secret_key=settings.INTERNET_ARCHIVE_SECRET_KEY,
retries=10,
retries_sleep=60,
verbose=True,
)
if success:
link.internet_archive_upload_status = 'completed'
link.save()
else:
link.internet_archive_upload_status = 'failed'
def upload(ia_args, ia_files, ia_identifier):
ia_files_new = []
for filename in ia_files:
if not os.path.isfile(filename + '.upload'):
ia_files_new.append(filename)
ia_files = list(ia_files_new)
if len(ia_files) > 0:
for filename in ia_files:
with open(filename + '.upload', 'w') as file:
pass
upload_response = internetarchive.upload(ia_identifier, filename, metadata = ia_args, access_key = ia_access_key, secret_key = ia_secret_key, queue_derive = True, verify = True, verbose = True, delete = True, retries = 5, retries_sleep = 300)
os.remove(filename + '.upload')
metadata = {'ocr': 'google-cloud-vision IndianKanoon 1.0', \
'fts-ignore-ingestion-lang-filter': 'true'}
abby_files_gz = []
for abby_file in abby_filelist:
abby_file_gz, n = re.subn('xml$', 'gz', abby_file)
self.delete_imagepdf(ia_item, abby_file_gz)
compress_abbyy(abby_file, abby_file_gz)
abby_files_gz.append(abby_file_gz)
self.update_metadata(ia_item, metadata)
success = False
while not success:
try:
success = upload(ia_item, abby_files_gz, headers=self.headers,\
access_key = self.access_key, \
secret_key = self.secret_key, retries=100)
success = True
except Exception as e:
self.logger.warn('Error in upload for %s: %s', ia_item, e)
success = False
time.sleep(120)
return success