How to use the keyvi.JumpConsistentHashString function in keyvi

To help you get started, we’ve selected a few keyvi examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github KeyviDev / keyvi / python / examples / lookup / compile_json.py View on Github external
input_fd = gzip.open(input_file)
        else:
            input_fd = open(input_file)

        for line in input_fd:
            try:
                parts = line.split("\t")
                key = parts[0]

                if key != remove_control_chars(key):
                    print "skip key: " + ":".join("{:02x}".format(ord(c)) for c in key) + " due to containing control characters"
                    skipped_keys +=1

                value = parts[1]

                shard = keyvi.JumpConsistentHashString(key, shards)
                compilers[shard].Add(key, value)
            except:
                print "failed to add: " + line
        print "Skipped keys " + str(skipped_keys)

    for i in range(jobs):
         t = threading.Thread(target=compile_worker)
         t.daemon = True
         t.start()

    if shards == 1:
        compile_queue.put((compilers[i], output))
    else:
        for i in range (0, shards):
            compile_queue.put((compilers[i], output + "-" + str(i)))
github KeyviDev / keyvi / python / scripts / compile_json.py View on Github external
input_fd = gzip.open(input_file)
        else:
            input_fd = open(input_file)

        for line in input_fd:
            try:
                parts = line.split("\t")
                key = parts[0]

                if key != remove_control_chars(key):
                    print "skip key: " + ":".join("{:02x}".format(ord(c)) for c in key) + " due to containing control characters"
                    skipped_keys +=1

                value = parts[1]

                shard = keyvi.JumpConsistentHashString(key, shards)
                compilers[shard].Add(key, value)
            except:
                print "failed to add: " + line
        print "Skipped keys " + str(skipped_keys)

    for i in range(jobs):
         t = threading.Thread(target=compile_worker)
         t.daemon = True
         t.start()

    for i in range (0, shards):
        compile_queue.put((compilers[i], output + "-" + str(i)))

    compile_queue.join()