How to use h5pyd - 10 common examples

filename = self.getFileName("query_dset")
        f = h5py.File(filename, "w")

        count = 100
        dt = np.dtype([('a', np.int), ('b', np.int)])
        dset = f.create_dataset('dset', (count,), dtype=dt)

        elem = dset[0]
        for i in range(count):
            elem['a'] = i // 10
            elem['b'] = i % 10
            dset[i] = elem


        # select from dset1
        if h5py.__name__ == "h5pyd":
            count = 0
            for row in dset.read_where("b>4"):
                self.assertTrue(row[1] > 4)
                count += 1

            self.assertEqual(count, 50)

        f.close()

# scalar byte values
        g1.attrs['e1'] = "Hello"
        s = g1.attrs['e1']
        self.assertEqual(s, "Hello" )

        # scalar objref attribute
        g11 = g1.create_group('g1.1') # create subgroup g1/g1.1
        g11.attrs['name'] = 'g1.1'   # tag group with an attribute

        if is_hsds:
            # following is not working with h5serv
            g11_ref = g11.ref   # get ref to g1/g1.1
            self.assertTrue(isinstance(g11_ref, h5py.Reference))
            refdt = h5py.special_dtype(ref=h5py.Reference)  # create ref dtype
            g1.attrs.create('f1', g11_ref, dtype=refdt)     # create attribute with ref to g1.1
            ref = g1.attrs['f1'] # read back the attribute

            refobj = f[ref]  # get the ref'd object
            self.assertTrue('name' in refobj.attrs)  # should see the tag attribute
            self.assertEqual(refobj.attrs['name'], 'g1.1')  # check tag value


        # close file
        f.close()

# scalar byte values
        g1.attrs['e1'] = "Hello"
        s = g1.attrs['e1']
        self.assertEqual(s, "Hello" )

        # scalar objref attribute
        g11 = g1.create_group('g1.1') # create subgroup g1/g1.1
        g11.attrs['name'] = 'g1.1'   # tag group with an attribute

        if is_hsds:
            # following is not working with h5serv
            g11_ref = g11.ref   # get ref to g1/g1.1
            self.assertTrue(isinstance(g11_ref, h5py.Reference))
            refdt = h5py.special_dtype(ref=h5py.Reference)  # create ref dtype
            g1.attrs.create('f1', g11_ref, dtype=refdt)     # create attribute with ref to g1.1
            ref = g1.attrs['f1'] # read back the attribute

            refobj = f[ref]  # get the ref'd object
            self.assertTrue('name' in refobj.attrs)  # should see the tag attribute
            self.assertEqual(refobj.attrs['name'], 'g1.1')  # check tag value


        # close file
        f.close()

r = f['/']
        is_h5serv = False
        if isinstance(f.id.id, str) and not f.id.id.startswith("g-"):
            is_h5serv = True  # h5serv doesn't have support for objref datasets yet

        # create subgroup g1
        r.create_group('g1')
        self.assertTrue('g1' in r)
        g1 = r['g1']

        # create subgroup g1/g1.1
        g11 = g1.create_group('g1.1')

        # get ref to g1/g1.1
        g11_ref = g11.ref
        self.assertTrue(isinstance(g11_ref, h5py.Reference))
        self.assertEqual(g11.name, "/g1/g1.1")

        # create subgroup g2
        r.create_group('g2')
        self.assertEqual(len(r), 2)
        g2 = r['g2']

        # get ref to g1/g1.1 from g2
        g11ref = g2[g11_ref]

        # create subgroup /g1/g1.1/foo
        g11ref.create_group("foo")
        self.assertEqual(len(g11), 1)
        self.assertTrue("foo" in g11)

        # create datset /g2/d1

dtvlen = h5py.special_dtype(vlen=dtref)
        e0 = np.array((g1_1.ref,), dtype=dtref)
        e1 = np.array((g1_1.ref,g1_2.ref), dtype=dtref)
        e2 = np.array((g1_1.ref,g1_2.ref,g1_3.ref), dtype=dtref)
        data = [e0,e1,e2]

        g1.attrs.create("b1", data, shape=(3,),dtype=dtvlen)

        vlen_val = g1.attrs["b1"]  # read back attribute
        self.assertTrue(isinstance(vlen_val, np.ndarray))
        self.assertEqual(len(vlen_val), 3)
        for i in range(3):
            e = vlen_val[i]
            self.assertTrue(isinstance(e, np.ndarray))
            ref_type = h5py.check_dtype(ref=e.dtype)
            self.assertEqual(ref_type, h5py.Reference)
            # TBD - h5pyd is returning shape of () rather than (1,) for singletons
            if i>0:
                self.assertEqual(e.shape, ((i+1),))
                # first element is always a ref to g1
                refd_group = f[e[0]]
                self.assertEqual(refd_group.attrs['name'], 'g1_1')

        # create an attribute with compound type of vlen objref and int32
        dtcompound = np.dtype([('refs', dtvlen), ('number', 'int32')])
        # create np array with data for the attribute
        # note: two step process is needed, see: https://github.com/h5py/h5py/issues/573
        data = np.zeros((2,), dtype=dtcompound)
        data[0] = (e1, 1)
        data[1] = (e2, 2)

        g1.attrs.create("c1", data, shape=(2,), dtype=dtcompound)

ret_val = g1.attrs["a1"]
        self.assertTrue(isinstance(ret_val, np.ndarray))
        self.assertEqual(len(ret_val), 2)
        self.assertTrue(isinstance(ret_val[0], np.ndarray))
        # py36  attribute[a1]: [array([0, 1, 2], dtype=int32) array([0, 1, 2, 3], dtype=int32)]
        # py27  [(0, 1, 2) (0, 1, 2, 3)]
        self.assertEqual(list(ret_val[0]), [0,1,2])
        self.assertEqual(ret_val[0].dtype, np.dtype('int32'))
        self.assertTrue(isinstance(ret_val[1], np.ndarray))
        self.assertEqual(ret_val[1].dtype, np.dtype('int32'))

        self.assertEqual(list(ret_val[1]), [0,1,2,3])

        # create an attribute that is VLEN ObjRef
        dtref = h5py.special_dtype(ref=h5py.Reference)
        dtvlen = h5py.special_dtype(vlen=dtref)
        e0 = np.array((g1_1.ref,), dtype=dtref)
        e1 = np.array((g1_1.ref,g1_2.ref), dtype=dtref)
        e2 = np.array((g1_1.ref,g1_2.ref,g1_3.ref), dtype=dtref)
        data = [e0,e1,e2]

        g1.attrs.create("b1", data, shape=(3,),dtype=dtvlen)

        vlen_val = g1.attrs["b1"]  # read back attribute
        self.assertTrue(isinstance(vlen_val, np.ndarray))
        self.assertEqual(len(vlen_val), 3)
        for i in range(3):
            e = vlen_val[i]
            self.assertTrue(isinstance(e, np.ndarray))
            ref_type = h5py.check_dtype(ref=e.dtype)
            self.assertEqual(ref_type, h5py.Reference)

# get ref to d1
        d1_ref = d1.ref
        dt = h5py.special_dtype(ref=h5py.Reference)
        self.assertTrue(dt.metadata['ref'] is h5py.Reference)
        ref = h5py.check_dtype(ref=dt)
        self.assertEqual(ref, h5py.Reference)


        if is_h5serv:
            return  # ref types not supported in h5serv

        # create dataset of ref types
        dset = g1.create_dataset('myrefs', (10,), dtype=dt)
        ref = h5py.check_dtype(ref=dset.dtype)
        self.assertEqual(ref, h5py.Reference)

        dset[0] = g11_ref
        dset[1] = d1_ref

        a_ref = dset[0]
        obj = f[a_ref]
        if not config.get("use_h5py"):
            self.assertEqual(obj.id.id, g11.id.id)  # ref to g1.1
        self.assertEqual(obj.name, "/g1/g1.1")

        b_ref = dset[1]
        obj = f[b_ref]
        if not config.get("use_h5py"):
            self.assertEqual(obj.id.id, d1.id.id)  # ref to d1
        self.assertEqual(obj.name, "/g2/d1")

def test_create(self):
        filename = self.getFileName("new_file")
        print("filename:", filename)
        now = time.time()
        f = h5py.File(filename, 'w')
        self.assertEqual(f.filename, filename)
        self.assertEqual(f.name, "/")
        self.assertTrue(f.id.id is not None)
        self.assertEqual(len(f.keys()), 0)
        self.assertEqual(f.mode, 'r+')
        is_hsds = False
        if isinstance(f.id.id, str) and f.id.id.startswith("g-"):
            is_hsds = True  # HSDS has different permission defaults
        if h5py.__name__ == "h5pyd":
            self.assertTrue(f.id.http_conn.endpoint.startswith("http"))
        self.assertTrue(f.id.id is not None)
        self.assertTrue('/' in f)
        # Check domain's timestamps
        if h5py.__name__ == "h5pyd" and is_hsds:
            # TBD: remove is_hsds when h5serv timestamp changed to float
            #print("modified:", datetime.fromtimestamp(f.modified), f.modified)
            #print("created: ", datetime.fromtimestamp(f.created), f.created)
            #print("now:     ", datetime.fromtimestamp(now), now)
            # verify the timestamps make sense
            # we add a 30-sec margin to account for possible time skew
            # between client and server
            self.assertTrue(f.created - 30.0 < now)
            self.assertTrue(f.created + 30.0 > now)
            self.assertTrue(f.modified - 30.0 < now)
            self.assertTrue(f.modified + 30.0 > now)

self.assertTrue(isinstance(val, np.int64))

        # try will ellipsis
        val = dset[...]
        self.assertEqual(val, 24)
        self.assertTrue(isinstance(val, np.ndarray))

        # try setting value using tuple
        dset[()] = 99
        val = dset[()]
        self.assertEqual(val, 99)

        self.assertEqual(dset.file.filename, filename)

        # Check dataset's last modified time
        if h5py.__name__ == "h5pyd":
            self.assertTrue(isinstance(dset.modified, datetime))

        f.close()

# create group using nested path
        g2 = r['g2']
        r['g1/g1.3'] = g2

        self.assertEqual(len(r), 4)

        # try creating a link with a space in the name
        r["a space"] = g2
        self.assertEqual(len(r), 5)

        # re-create softlink
        r['mysoftlink'] = h5py.SoftLink('/g1/g1.1')

        # Check group's last modified time
        if h5py.__name__ == "h5pyd":
            self.assertTrue(isinstance(g1.modified, datetime))
            #self.assertEqual(g1.modified.tzname(), six.u('UTC'))

        f.close()

        # re-open file in read-only mode
        f = h5py.File(filename, 'r')
        self.assertEqual(len(f), 6)
        for name in ("g1", "g2", "g4", "g1.1", "a space", "mysoftlink"):
            self.assertTrue(name in f)
        self.assertTrue("/g1/g1.1" in f)
        g1_1 = f["/g1/g1.1"]
        linkee_class = r.get('mysoftlink', getclass=True)
        self.assertEqual(linkee_class, h5py.Group)
        link_class = r.get('mysoftlink', getclass=True, getlink=True)
        self.assertEqual(link_class, h5py.SoftLink)