How to use the hub.fs function in hub

To help you get started, we’ve selected a few hub examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github snarkai / Hub / test / test_dataloader.py View on Github external
def test_pytorch():
    # Create arrays
    datahub = hub.fs("./data/cache").connect()
    images = datahub.array(
        name="test/dataloaders/images3",
        shape=(100, 100, 100),
        chunk=(1, 100, 100),
        dtype="uint8",
    )
    labels = datahub.array(
        name="test/dataloaders/labels3", shape=(100, 1), chunk=(100, 1), dtype="uint8"
    )
    # Create dataset
    ds = datahub.dataset(
        name="test/loaders/dataset2", components={"images": images, "labels": labels}
    )
    # Transform to Pytorch
    train_dataset = ds.to_pytorch()
    # Create data loader
github snarkai / Hub / test / test_basic.py View on Github external
def test_broadcasting():
    print("- Broadcasting")
    datahub = hub.fs("./data/cache").connect()
    shape = (100, 100, 100)
    chunk = (50, 50, 50)
    x = datahub.array(name="test/example:3", shape=shape, chunk=chunk, dtype="uint8")
    x[0, 0, 0] = 11
    assert x[0, 0, 0] == 11
    x[0] = 10
    assert x[0].mean() == 10
    x[1] = np.ones((100, 100), dtype="uint8")
    assert x[1].mean() == 1
    x[3, 90, :] = np.ones((1, 1, 100), dtype="uint8")
    assert x[3, 90].mean() == 1
    print("passed")
github snarkai / Hub / test / test_datasets.py View on Github external
def test_dataset():
    datahub = hub.fs("./data/cache").connect()
    x = datahub.array(name='test/example:input', shape = (100, 25, 25), chunk=(20, 5, 5), dtype='uint8')
    y = datahub.array(name='test/example:label', shape = (100, 4), chunk = (20, 2), dtype='uint8')

    ds = datahub.dataset(components={
        'input': x,
        'label': y
    }, name='test/dataset:train3')
    assert ds[0]['input'].shape == (25, 25)
    assert ds['input'].shape[0] == 100   # return single array
    assert ds['label', 0].mean() == 0  # equivalent ds['train'][0]
github snarkai / Hub / test / test_dataloader.py View on Github external
def test_to_tensorflow():
    print("testing Tensorflow")
    conn = hub.fs("./data/cache").connect()
    ds = conn.open("test/loaders/dataset2")

    # Transform to Tensorflow
    train_dataset = ds.to_tensorflow()
    batch = next(iter(train_dataset.batch(batch_size=16)))
    assert batch["images"].shape == (16, 100, 100)
    # TODO create dataloader 
github snarkai / Hub / test / test_dynamic_array.py View on Github external
def test_dynamic_array():
    conn = hub.fs("./data/cache").connect()
    arr = conn.array_create(
        "test/dynamic_array_3",
        shape=(10, 8, 4, 12),
        chunk=(5, 4, 2, 6),
        dtype="uint8",
        dsplit=2,
    )
    arr.darray[0:10, 0:5] = (2, 12)
    arr.darray[0:10, 5:8] = (6, 14)

    assert arr[5, 3, :, :].shape == (2, 12)
    assert arr[5, 6, :, :].shape == (6, 14)
    assert arr[5, 4:6, :, :].shape == (2, 6, 14)
github snarkai / Hub / test / test_basic.py View on Github external
def test_simple_upload_download():
    print("- Simple Chunk Upload and Download")
    datahub = hub.fs("./data/cache").connect()
    shape = (10, 10, 10, 10)
    chunk = (5, 5, 5, 5)
    datahub = hub.fs("./data/cache").connect()
    x = datahub.array(name="test/example:1", shape=shape, chunk=chunk, dtype="uint8")
    x[0] = np.ones((1, 10, 10, 10), dtype="uint8")
    assert x[0].mean() == 1
    print("passed")
github snarkai / Hub / test / test_basic.py View on Github external
def test_open_array():
    print("- Loading arrays")
    datahub = hub.fs("./data/cache").connect()
    x = datahub.open(name="test/example:4")
    print(x.shape)
    assert np.all(x.shape == np.array((10, 10, 10)))
    print("passed")
github snarkai / Hub / test / test_basic.py View on Github external
def test_chunk_shape():
    print("- Chunk shape")
    datahub = hub.fs("./data/cache").connect()
    shape = (10, 10, 10)
    chunk = (5, 5, 5)
    x = datahub.array(name="test/example:4", shape=shape, chunk=chunk, dtype="uint8")
    x[0:5, 0:5, 0:5] = 0
    print("passed")
github snarkai / Hub / test / test_basic.py View on Github external
def test_multiple_upload_download():
    datahub = hub.fs("./data/cache").connect()
    shape = (10, 10, 10, 10)
    chunk = (5, 5, 5, 5)
    x = datahub.array(name="test/example:1", shape=shape, chunk=chunk, dtype="uint8")
    x[0:3] = np.ones((3, 10, 10, 10), dtype="uint8")
    assert x[0:3].mean() == 1
    print("passed")
github snarkai / Hub / experiments / ex.py View on Github external
import math
import numpy as np
import itertools
import io

# tf.enable_eager_execution()

# from waymo_open_dataset.utils import range_image_utils
# from waymo_open_dataset.utils import transform_utils
# from waymo_open_dataset.utils import  frame_utils
# from waymo_open_dataset import dataset_pb2 as open_dataset
import hub
from PIL import Image

# client = hub.gs('snark_waymo_open_dataset', creds_path='.creds/gs.json').connect() 
client = hub.fs('/drive/upload').connect()
arr = client.array_open('validation/images')

for i in range(0, 5):
    img = arr[10, i]
    print(img.shape)
    Image.fromarray(img, 'RGB').save(f'output/image-{i}.jpg')