How to use the blobfile.BlobFile function in blobfile

To help you get started, we’ve selected a few blobfile examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github openai / procgen / procgen-build / procgen_build / build_qt.py View on Github external
elif bf.exists(cache_path):
        print(f"downloading cache for {name}: {cache_path}")
        start = time.time()
        with bf.BlobFile(cache_path, "rb") as f:
            with tarfile.open(fileobj=f, mode="r") as tf:
                tf.extractall()
        print(f"download elapsed {time.time() - start}")
    else:
        print(f"building cache for {name}")
        start = time.time()
        build_fn()
        print(f"cache build elapsed {time.time() - start}")
        print(f"uploading cache for {name}")
        start = time.time()
        if not bf.exists(cache_path):
            with bf.BlobFile(cache_path, "wb") as f:
                with tarfile.open(fileobj=f, mode="w") as tf:
                    tf.add(dirpath)
        print(f"upload elapsed {time.time() - start}")
github openai / procgen / procgen-build / procgen_build / build_qt.py View on Github external
if os.path.exists(dirpath):
        print(f"cache for {name} found locally")
        return
    
    options_hash = hashlib.md5("|".join(options).encode("utf8")).hexdigest()
    cache_path = bf.join(f"gs://{GCS_BUCKET}", "cache", f"{name}-{options_hash}.tar")
    if "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ:
        # we don't have any credentials to do the caching, always build in this case
        print(f"building without cache for {name}")
        start = time.time()
        build_fn()
        print(f"build elapsed {time.time() - start}")
    elif bf.exists(cache_path):
        print(f"downloading cache for {name}: {cache_path}")
        start = time.time()
        with bf.BlobFile(cache_path, "rb") as f:
            with tarfile.open(fileobj=f, mode="r") as tf:
                tf.extractall()
        print(f"download elapsed {time.time() - start}")
    else:
        print(f"building cache for {name}")
        start = time.time()
        build_fn()
        print(f"cache build elapsed {time.time() - start}")
        print(f"uploading cache for {name}")
        start = time.time()
        if not bf.exists(cache_path):
            with bf.BlobFile(cache_path, "wb") as f:
                with tarfile.open(fileobj=f, mode="w") as tf:
                    tf.add(dirpath)
        print(f"upload elapsed {time.time() - start}")

blobfile

Read GCS, ABS and local paths with the same interface, clone of tensorflow.io.gfile

Unlicense
Latest version published 6 months ago

Package Health Score

70 / 100
Full package analysis