How to use the ijson.backends.yajl2_cffi function in ijson

To help you get started, we’ve selected a few ijson examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github data61 / anonlink-entity-service / backend / entityservice.py View on Github external
def counting_generator():
        try:
            for clk in ijson.items(raw_stream, 'clks.item'):
                # Often the clients upload base64 strings with newlines
                # We remove those here
                raw = ''.join(clk.split('\n')).encode() + b'\n'
                store['count'] += 1
                store['totalbytes'] += len(raw)
                yield raw
        except ijson.common.IncompleteJSONError as e:
            store['count'] = 0
            app.logger.warning("Stopping as we have received incomplete json")
            return
github zentralwerkstatt / locbook / locbook.py View on Github external
def import_google(filename):
    logging.info('Importing from ' + filename)
    # Needs to be rb!
    with open(filename, 'rb') as f:
        data = ijson.items(f, 'locations.item')
        c=0
        for o in data:
            c+=1
            p = (round(o['longitudeE7']/10000000, precision), round(o['latitudeE7']/10000000, precision))
            d, t = tst_to_dt(int(o['timestampMs'][:-3]))
            make_history(p, d, t, False)
    f.close()
    logging.info(str(c) + ' items imported from ' + filename)
    logging.info('History size: ' + str(len(history)) + ' points')
    pickle.dump(history, open('history.pickle', 'wb'))
    write_js()
github johncsnyder / SwiftKitten / SwiftKitten.py View on Github external
def _get_structure_info(self, view):
        """
        """
         #get structure info command
        text = view.substr(Region(0, view.size()))
        cmd = self.get_structure_info_cmd(view, text)
        timeout = self.get_settings(view, "sourcekitten_timeout", 1.0)

        # run structure info command
        p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
        structure_info = list(ijson.items(p.stdout,''))[0]

        return structure_info
github vidjil / vidjil / server / web2py / applications / vidjil / controllers / sample_set.py View on Github external
def getResultsStats(file_name, dest):
    import ijson.backends.yajl2_cffi as ijson
    log.debug("getResultsStats()")
    file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
    distributions = []
    with open(file_path, 'rb') as results:
        i = "1"
        while True:
            results.seek(0, 0)
            tmp =  [d for d in ijson.items(results, "reads-distribution-%s.item" % i)]
            if len(tmp) == 0:
                break
            else:
                distributions.append((i, tmp[0]))
                i += "0"
    dest['distribution'] = distributions
    return dest
github kbogas / medknow / data_loader.py View on Github external
def get_collection_count(source, type):
    """
    Helper function to get total collection length.
    Input:
        - source: str, value denoting where we will read from (e.g 'mongo')
        - type: str, value denoting what we will read (e.g. text, edges)
    Output:
        - N_collection: int,
        number of items in the collection
    """
    if source == 'file':
        inp_path = settings['load']['path']['file_path'] 
        # Document iterator field in the collection
        infield = settings['load'][type]['itemfield']
        with open(inp_path, 'r') as f:
            docs = ijson2.items(f, '%s.item' % infield)
            N_collection = 0
            for item in docs:
                N_collection += 1
    elif source == 'mongo':
        # input mongo variables from settings.yaml
        uri = settings['load']['mongo']['uri']
        db_name = settings['load']['mongo']['db']
        collection_name = settings['load']['mongo']['collection']
        client = pymongo.MongoClient(uri)
        db = client[db_name]
        collection = db[collection_name]
        N_collection = collection.count()
    else:
        time_log("Can't calculate total collection count for source type %s" % settings['in']['source'])
        raise NotImplementedError
    return N_collection
github slub / efre-lod-elasticsearch-tools / luigi / update_gnd.py View on Github external
def yield_obj(path, basepath):
    with gzip.open(path, "r") as fin:
        builder = ijson.common.ObjectBuilder()
        for prefix, event, val in ijson.parse(fin):
            try:
                builder.event(event, val)
            except:
                if hasattr(builder, "value"):
                    print(builder.value)
            if prefix == basepath and event == "end_map":
                if hasattr(builder, "value"):
                    yield builder.value
                builder = ijson.common.ObjectBuilder()
github vidjil / vidjil / tools / vidjilparser.py View on Github external
def validate(self, filepath):
        with open(filepath, 'rb') as vfile:
            parser = ijson.parse(vfile)
            model = list(self._model_prefixes)
            for prefix, event, value in parser:
                pair = (prefix, event)
                if pair in model:
                    model.remove(pair)
            return len(model) == 0