How to use the internetarchive.upload function in internetarchive

To help you get started, we’ve selected a few internetarchive examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github ArchiveTeam / NewsGrabber / server_main / upload.py View on Github external
def upload_single(self, name, f, ia_args):
        with open(settings.keys, 'r') as keys:
            access_key, secret_key = keys.read().strip().split(':')
        try:
            internetarchive.upload('archiveteam_newssites_{name}'.format(name=name),
                os.path.join(settings.dir_ready, f),
                metadata=ia_args,
                access_key=access_key,
                secret_key=secret_key,
                queue_derive=True,
                verify=True,
                verbose=True,
                delete=True,
                retries=10,
                retries_sleep=300)
        except:
            pass # see code below
        self.concurrent_uploads -= 1
        os.remove(os.path.join(settings.dir_ready, f+'.upload'))
        if os.path.isfile(os.path.join(settings.dir_ready, f)):
            settings.irc_bot.send('PRIVMSG', '{name} uploaded unsuccessful.'.format(
github pastpages / pastpages.org / archive / models.py View on Github external
def upload_ia_item(self):
        logger.debug("Uploading IA item for {}".format(self.ia_id))
        if not self.has_image and not self.has_crop:
            logger.debug("No images to upload")
            return None
        files = []
        if self.has_image:
            saved_image = self.save_image()
            files.append(saved_image)
        if self.has_crop:
            saved_crop = self.save_crop()
            files.append(saved_crop)
        internetarchive.upload(
            self.ia_id,
            files,
            metadata=self.ia_metadata,
            access_key=settings.IA_ACCESS_KEY_ID,
            secret_key=settings.IA_SECRET_ACCESS_KEY,
            checksum=False,
            verbose=True
        )
        if self.has_image:
            os.remove(saved_image)
        if self.has_crop:
            os.remove(saved_crop)
        return internetarchive.get_item(self.ia_id)
github pastpages / pastpages.org / archive / tasks / __init__.py View on Github external
width,
            1000
        )
    )

    # Move the original file to Internet Archive namespace
    shutil.copy(output_path, ssht.get_image_name())

    # Save crop to Internet Archive namespace
    crop.save(open(ssht.get_crop_name(), 'w'))

    # Upload both images to Internet Archive
    files = [ssht.get_image_name(), ssht.get_crop_name()]
    try:
        logger.debug("Uploading to internetarchive as {}".format(ssht.ia_id))
        internetarchive.upload(
            ssht.ia_id,
            files,
            metadata=ssht.ia_metadata,
            access_key=settings.IA_ACCESS_KEY_ID,
            secret_key=settings.IA_SECRET_ACCESS_KEY,
            checksum=False,
            verbose=True
        )
    except Exception, e:
        logger.error("internetarchive error: %s" % e)
        ScreenshotLog.objects.create(
            update=update,
            site=site,
            message_type="error",
            message="internetarchive error: %s" % e
        )
github harvard-lil / perma / perma_web / perma / tasks.py View on Github external
# if item already exists (but has been removed),
            # ia won't update its metadata in upload function
            if item.exists and item.metadata['title'] == 'Removed':
                item.modify_metadata(metadata,
                                     access_key=settings.INTERNET_ARCHIVE_ACCESS_KEY,
                                     secret_key=settings.INTERNET_ARCHIVE_SECRET_KEY,
                                     )

            warc_name = os.path.basename(link.warc_storage_file())

            # copy warc to local disk storage for upload
            temp_warc_file = tempfile.TemporaryFile()
            copy_file_data(default_storage.open(link.warc_storage_file()), temp_warc_file)
            temp_warc_file.seek(0)

            success = internetarchive.upload(
                identifier,
                {warc_name: temp_warc_file},
                metadata=metadata,
                access_key=settings.INTERNET_ARCHIVE_ACCESS_KEY,
                secret_key=settings.INTERNET_ARCHIVE_SECRET_KEY,
                retries=10,
                retries_sleep=60,
                verbose=True,
            )

            if success:
                link.internet_archive_upload_status = 'completed'
                link.save()

            else:
                link.internet_archive_upload_status = 'failed'
github ArchiveTeam / VideoBot / upload.py View on Github external
def upload(ia_args, ia_files, ia_identifier):
    ia_files_new = []
    for filename in ia_files:
        if not os.path.isfile(filename + '.upload'):
            ia_files_new.append(filename)
    ia_files = list(ia_files_new)
    if len(ia_files) > 0:
        for filename in ia_files:
            with open(filename + '.upload', 'w') as file:
                pass
            upload_response = internetarchive.upload(ia_identifier, filename, metadata = ia_args, access_key = ia_access_key, secret_key = ia_secret_key, queue_derive = True, verify = True, verbose = True, delete = True, retries = 5, retries_sleep = 300)
            os.remove(filename + '.upload')
github sushant354 / egazette / gvision.py View on Github external
metadata = {'ocr': 'google-cloud-vision IndianKanoon 1.0', \
                    'fts-ignore-ingestion-lang-filter': 'true'}

        abby_files_gz = []
        for abby_file in abby_filelist:
            abby_file_gz, n = re.subn('xml$', 'gz', abby_file)
            self.delete_imagepdf(ia_item, abby_file_gz)

            compress_abbyy(abby_file, abby_file_gz)
            abby_files_gz.append(abby_file_gz)

        self.update_metadata(ia_item, metadata)
        success = False 
        while not success:
            try:
                success = upload(ia_item, abby_files_gz, headers=self.headers,\
                                 access_key = self.access_key, \
                                 secret_key = self.secret_key, retries=100)
                success = True                 
            except Exception as e:
                self.logger.warn('Error in upload for %s: %s', ia_item, e)
                success = False
                time.sleep(120)
        return success