How to use the disdat.hyperframe_pb2 function in disdat

To help you get started, we’ve selected a few disdat examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github kyocum / disdat / disdat / hyperframe.py View on Github external
def get_proto_byteorder(numpy_byteorder):
        numpy_endianness = {
            '<': hyperframe_pb2.LITTLE,
            '>': hyperframe_pb2.BIG,
            '|': hyperframe_pb2.NA,
        }

        system_endianness = {
            'little': hyperframe_pb2.LITTLE,
            'big': hyperframe_pb2.BIG,
        }

        if numpy_byteorder is '=':
            return system_endianness[sys.byteorder]

        return numpy_endianness[numpy_byteorder]
github kyocum / disdat / disdat / hyperframe.py View on Github external
def __init__(self, owner=None, human_name=None, processing_name=None, uuid=None,
                 frames=None, lin_obj=None, tags=None, presentation=hyperframe_pb2.DEFAULT):
        """
        Create a HyperFrame

        Note: human_name used to be "bundle_name" -- a special tag.

        Note: Unlike old-style bundles, we no longer have a url / size.  Those are inside the frames.

        Note: This object has a "frame cache."  The PB only refers to frames by UUID.  As the user works with this
        object, they may ask for the referred to frames.  We fill in the cache as they request them.

        Note: This is denormalized.  Lineage, Tags, Frames all use this HyperFrame's UUID as their index.

        Args:
            owner (str): user or group that made the object -- special tag.
            human_name (str): the simple or given name of the bundle, e.g., dsdt add "STR.Results" -- special tag.
            processing_name (str): machine generated (pipe.unique_id()) name -- special tag.
github kyocum / disdat / disdat / hyperframe.py View on Github external
def _pb_type():
        """
        Return the hyperframe_pb2.()
        :return:
        """
        return hyperframe_pb2.Lineage()
github kyocum / disdat / disdat / hyperframe.py View on Github external
def make_numpy_array(self):
        """
        Create a np ndarray from native bytes in frame

        Returns:
            (`numpy.ndarray`)
        """

        assert (self.pb.type != hyperframe_pb2.LINK)
        assert (self.pb.type != hyperframe_pb2.HFRAME)
        assert (self.pb.type != hyperframe_pb2.STRING)

        dtype = np.dtype(FrameRecord.get_numpy_type(self.pb.type))
        dtype = dtype.newbyteorder(FrameRecord.get_numpy_byteorder(self.pb.byteorder))

        nda = np.frombuffer(self.pb.data, dtype=dtype)
        nda = nda.reshape(self.pb.shape)

        return nda
github kyocum / disdat / disdat / hyperframe.py View on Github external
def _pb_type():
        """
        Return the hyperframe_pb2.()
        :return:
        """
        return hyperframe_pb2.LinkAuth()
github kyocum / disdat / disdat / hyperframe.py View on Github external
def get_numpy_type(proto_type):
        numpy_types = {
            hyperframe_pb2.BOOL:    np.bool_,
            hyperframe_pb2.INT8:    np.int8,
            hyperframe_pb2.INT16:   np.int16,
            hyperframe_pb2.INT32:   np.int32,
            hyperframe_pb2.INT64:   np.int64,
            hyperframe_pb2.UINT8:   np.uint8,
            hyperframe_pb2.UINT16:  np.uint16,
            hyperframe_pb2.UINT32:  np.uint32,
            hyperframe_pb2.UINT64:  np.uint64,
            hyperframe_pb2.FLOAT16: np.float16,
            hyperframe_pb2.FLOAT32: np.float32,
            hyperframe_pb2.FLOAT64: np.float64,
            hyperframe_pb2.OBJECT:  np.object_
            # Special Case -- manual conversion on string type -- hyperframe_pb2.STRING:  np.string_
        }

        if proto_type in numpy_types:
            return numpy_types[proto_type]
        raise KeyError('Could not find a message array type for {}'.format(proto_type))
github kyocum / disdat / disdat / pipe_base.py View on Github external
np.uint32,
            np.uint64,
            np.float16,
            np.float32,
            np.float64,
            six.binary_type,
            six.text_type,
            np.unicode_,
            np.string_
        )

        frames = []

        if val is None:
            """ None's stored as json.dumps([None]) or '[null]' """
            presentation = hyperframe_pb2.JSON
            frames.append(data_context.convert_scalar2frame(hfid, common.DEFAULT_FRAME_NAME + ':0', val))

        elif isinstance(val, HyperFrameRecord):
            presentation = hyperframe_pb2.HF
            frames.append(FrameRecord.make_hframe_frame(hfid, pipe.human_id(), [val]))

        elif isinstance(val, np.ndarray) or isinstance(val, list):
            presentation = hyperframe_pb2.TENSOR
            if isinstance(val, list):
                val = np.array(val)
            frames.append(data_context.convert_serieslike2frame(hfid, common.DEFAULT_FRAME_NAME + ':0', val))

        elif isinstance(val, tuple):
            presentation = hyperframe_pb2.ROW
            val = np.array(val)
            frames.append(data_context.convert_serieslike2frame(hfid, common.DEFAULT_FRAME_NAME + ':0', val))
github kyocum / disdat / disdat / hyperframe.py View on Github external
def get_numpy_type(proto_type):
        numpy_types = {
            hyperframe_pb2.BOOL:    np.bool_,
            hyperframe_pb2.INT8:    np.int8,
            hyperframe_pb2.INT16:   np.int16,
            hyperframe_pb2.INT32:   np.int32,
            hyperframe_pb2.INT64:   np.int64,
            hyperframe_pb2.UINT8:   np.uint8,
            hyperframe_pb2.UINT16:  np.uint16,
            hyperframe_pb2.UINT32:  np.uint32,
            hyperframe_pb2.UINT64:  np.uint64,
            hyperframe_pb2.FLOAT16: np.float16,
            hyperframe_pb2.FLOAT32: np.float32,
            hyperframe_pb2.FLOAT64: np.float64,
            hyperframe_pb2.OBJECT:  np.object_
            # Special Case -- manual conversion on string type -- hyperframe_pb2.STRING:  np.string_
        }

        if proto_type in numpy_types:
            return numpy_types[proto_type]
        raise KeyError('Could not find a message array type for {}'.format(proto_type))
github kyocum / disdat / disdat / hyperframe.py View on Github external
def _pb_type():
        """

        Returns:
            the hyperframe_pb2.()

        """

        return hyperframe_pb2.Frame()
github kyocum / disdat / disdat / hyperframe.py View on Github external
def get_numpy_type(proto_type):
        numpy_types = {
            hyperframe_pb2.BOOL:    np.bool_,
            hyperframe_pb2.INT8:    np.int8,
            hyperframe_pb2.INT16:   np.int16,
            hyperframe_pb2.INT32:   np.int32,
            hyperframe_pb2.INT64:   np.int64,
            hyperframe_pb2.UINT8:   np.uint8,
            hyperframe_pb2.UINT16:  np.uint16,
            hyperframe_pb2.UINT32:  np.uint32,
            hyperframe_pb2.UINT64:  np.uint64,
            hyperframe_pb2.FLOAT16: np.float16,
            hyperframe_pb2.FLOAT32: np.float32,
            hyperframe_pb2.FLOAT64: np.float64,
            hyperframe_pb2.OBJECT:  np.object_
            # Special Case -- manual conversion on string type -- hyperframe_pb2.STRING:  np.string_
        }

        if proto_type in numpy_types:
            return numpy_types[proto_type]
        raise KeyError('Could not find a message array type for {}'.format(proto_type))