How to use the synapse.lib.msgpack.en function in synapse

To help you get started, we’ve selected a few synapse examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github vertexproject / synapse / synapse / cores / lmdb.py View on Github external
def _encValKey(v):
    '''
    Encode a value as used in a key.

    Non-negative numbers are msgpack encoded.  Negative numbers are encoded as a marker, then the
    encoded negative of that value, so that the ordering of the encodings is easily mapped to the
    ordering of the negative numbers.  Strings too long are hashed.  Note that this scheme prevents
    interleaving of value types: all string encodings compare larger than all negative number
    encodings compare larger than all nonnegative encodings.
    '''
    if isinstance(v, int):
        if v >= 0:
            return s_msgpack.en(v)
        else:
            return NEGATIVE_VAL_MARKER_ENC + s_msgpack.en(-v)
    else:
        if len(v) >= LARGE_STRING_SIZE:
            return (HASH_VAL_MARKER_ENC + s_msgpack.en(xxhash.xxh64(v).intdigest()))
        else:
            return STRING_VAL_MARKER_ENC + s_msgpack.en(v)
github vertexproject / synapse / synapse / lib / migrate.py View on Github external
for layr in self.layers:

            async with self.getTempSlab() as slab:

                i = 0
                async for buid, valu in layr.iterPropRows(prop.form.name, prop.name):
                    i = i + 1
                    if i % modulus == 0:
                        logger.debug(f'Consumed {prop.form.name} count: {i}')

                    norm, info = prop.type.norm(valu)
                    if norm == valu:
                        continue

                    slab.put(buid, s_msgpack.en(norm))

                if i:
                    logger.debug(f'Total {prop.form.name} count: {i}')

                i = 0
                for buid, byts in slab.scanByFull():
                    norm = s_msgpack.un(byts)
                    await layr.storPropSet(buid, prop, norm)
                    i = i + 1
                    if i % modulus == 0:
                        logger.debug(f'Set prop count: {i}')
                if i:
                    logger.debug(f'Total propset count: {i}')

        logger.debug(f'Done norming: {prop.form.name}')
github vertexproject / synapse / synapse / lib / lmdblayer.py View on Github external
def _storPropSetCommon(self, buid, penc, bpkey, pvpref, univ, valu, indx):

        bpval = s_msgpack.en((valu, indx))
        pvvalu = s_msgpack.en((buid,))

        byts = self.layrslab.replace(bpkey, bpval, db=self.bybuid)
        if byts is not None:

            oldv, oldi = s_msgpack.un(byts)
            if oldi is not None:

                if isinstance(oldi, bytes):

                    self.layrslab.delete(pvpref + oldi, pvvalu, db=self.byprop)
                    if univ:
                        self.layrslab.delete(penc + oldi, pvvalu, db=self.byuniv)

                else:
                    for oldibyts in oldi:
                        self.layrslab.delete(pvpref + oldibyts, pvvalu, db=self.byprop)
github vertexproject / synapse / synapse / cmds / cortex.py View on Github external
def encodeMsg(self, mesg):
        '''Get byts for a message'''

        fmt = self.locs.get('log:fmt')
        if fmt == 'jsonl':
            s = json.dumps(mesg, sort_keys=True) + '\n'
            buf = s.encode()
            return buf

        elif fmt == 'mpk':
            buf = s_msgpack.en(mesg)
            return buf

        mesg = f'Unknown encoding format: {fmt}'
        raise s_exc.SynErr(mesg=mesg)
github vertexproject / synapse / synapse / lib / kv.py View on Github external
def set(self, prop, valu):
        '''
        Set a property in the KvLook.

        Args:
            prop (str): The property name to set.
            valu (obj): A msgpack compatible value.

        Returns:
            None
        '''
        lkey = self.iden + prop.encode('utf8')
        self.stor.setKvProp(lkey, s_msgpack.en(valu))
github vertexproject / synapse / synapse / cores / storage.py View on Github external
dependent. In purely memory backed cortexes, this KV store may not be
        persistent, even if the tufo-layer is persistent, through something
        such as the savefile mechanism.

        Notes:
            Data which is stored in the KV store is msgpacked, so caveats with
            that apply.

        Args:
            key (str): Name of the value to store.
            valu: Value to store in the KV store.

        Returns:
            The input value, unchanged.
        '''
        buf = s_msgpack.en(valu)
        self._setBlobValu(key, buf)
        self.savebus.fire('syn:core:blob:set', key=key, valu=buf)
        return valu
github vertexproject / synapse / synapse / tools / migrate_010.py View on Github external
if not curs.set_key(formname.encode('utf8')):
                    continue
                logger.info('Stage 2a: (%2d/%2d) processing nodes from %s', i + 1, len(self.first_forms), formname)
                for enc_iden in curs.iternext_dup():
                    props = None
                    try:
                        with txn.cursor(self.iden_tbl) as curs:
                            if not curs.set_key(enc_iden):
                                raise ConsistencyError('missing iden', hexlify(s_msgpack.un(enc_iden)))
                            props = self._get_props_from_cursor(txn, curs)
                        node = self.convert_props(props)
                        if node is None:
                            logger.debug('cannot convert %s', props)
                            continue
                        if formname == 'file:bytes':
                            if not txn.put(props['node:ndef'].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl):
                                raise ConsistencyError('put failure')
                            if not txn.put(props[formname].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl):
                                raise ConsistencyError('put failure')
                        else:
                            txn.put(props[formname].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl)
                        self.write_node_to_file(node)
                    except Exception:
                        logger.debug('Failed on processing node of form %s', formname, exc_info=True)
                        if props is not None and self.rejects_fh is not None:
                            self.rejects_fh.write(s_msgpack.en(props))

        comp_node_time = time.time()
        logger.debug('Stage 2a complete in %.1fs', comp_node_time - start_time)
github vertexproject / synapse / synapse / cores / lmdb.py View on Github external
def _calcFirstLastKeys(prop, valu, mintime, maxtime):
    '''
    Returns the encoded bytes for the start and end keys to the pt or pvt
    index.  Helper function for _{get,del}RowsByProp
    '''
    p_enc = _encProp(prop)
    v_key_enc = b'' if valu is None else _encValKey(valu)
    v_is_hashed = valu is not None and (v_key_enc[0] == HASH_VAL_MARKER_ENC)
    if mintime is None and maxtime is None:
        return (p_enc + v_key_enc, None, v_is_hashed, True)
    mintime_enc = b'' if mintime is None else s_msgpack.en(mintime)
    maxtime_enc = MAX_TIME_ENC if maxtime is None else s_msgpack.en(maxtime)

    first_key = p_enc + v_key_enc + mintime_enc
    last_key = p_enc + v_key_enc + maxtime_enc
    return (first_key, last_key, v_is_hashed, False)