How to use the internetarchive.utils.get_md5 function in internetarchive

To help you get started, we’ve selected a few internetarchive examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github jhu-lcsr / costar_plan / costar_hyper / costar_block_stacking_internet_archive_upload.py View on Github external
# A successful upload should have status_code = 200
        if debug:
            pb.write('[DEBUG] item key = {}'.format(file_path))
            results_url.append(resp.url)
            results_path_url.append(resp.path_url)
            with open(os.path.join(path, file_path), 'rb') as f:
                md5_hash = internetarchive.utils.get_md5(f)
            success_count += 1
        elif resp.status_code is None:
            # NOTE: Response object is empty. This file is already on the server.
            # See definition of upload_file in internetarchive/item.py
            pb.write('{} is already on the server.'.format(
                file_path))
            # File already on server. Record the hash
            with open(os.path.join(path, file_path), 'rb') as f:
                md5_hash = internetarchive.utils.get_md5(f)
            skip_count += 1
        elif resp.status_code != 200:
            pb.write('Upload failed for {}, status code = {}'.format(
                file_path, resp.status_code))
            failed_count += 1
        else:
            results_url.append(resp.request.url)
            results_path_url.append(resp.request.path_url)
            # File successfully sent to server. Record the hash
            with open(os.path.join(path, file_path), 'rb') as f:
                md5_hash = internetarchive.utils.get_md5(f)
            success_count += 1
        file_hash_table[i] = np.array([file_path, md5_hash])

        if (success_count + 1) % 10 == 0:
            pb.write(timeStamped('[%d] Check point, saving csv' % i))
github jhu-lcsr / costar_plan / costar_hyper / costar_block_stacking_internet_archive_upload.py View on Github external
metadata=md,
            verify=True,
            checksum=True,
            retries=10,
            retries_sleep=30,
            queue_derive=False,
            debug=debug)

        # Check if the file is successfully uploaded
        # A successful upload should have status_code = 200
        if debug:
            pb.write('[DEBUG] item key = {}'.format(file_path))
            results_url.append(resp.url)
            results_path_url.append(resp.path_url)
            with open(os.path.join(path, file_path), 'rb') as f:
                md5_hash = internetarchive.utils.get_md5(f)
            success_count += 1
        elif resp.status_code is None:
            # NOTE: Response object is empty. This file is already on the server.
            # See definition of upload_file in internetarchive/item.py
            pb.write('{} is already on the server.'.format(
                file_path))
            # File already on server. Record the hash
            with open(os.path.join(path, file_path), 'rb') as f:
                md5_hash = internetarchive.utils.get_md5(f)
            skip_count += 1
        elif resp.status_code != 200:
            pb.write('Upload failed for {}, status code = {}'.format(
                file_path, resp.status_code))
            failed_count += 1
        else:
            results_url.append(resp.request.url)