Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
input_fd = gzip.open(input_file)
else:
input_fd = open(input_file)
for line in input_fd:
try:
parts = line.split("\t")
key = parts[0]
if key != remove_control_chars(key):
print "skip key: " + ":".join("{:02x}".format(ord(c)) for c in key) + " due to containing control characters"
skipped_keys +=1
value = parts[1]
shard = keyvi.JumpConsistentHashString(key, shards)
compilers[shard].Add(key, value)
except:
print "failed to add: " + line
print "Skipped keys " + str(skipped_keys)
for i in range(jobs):
t = threading.Thread(target=compile_worker)
t.daemon = True
t.start()
if shards == 1:
compile_queue.put((compilers[i], output))
else:
for i in range (0, shards):
compile_queue.put((compilers[i], output + "-" + str(i)))
input_fd = gzip.open(input_file)
else:
input_fd = open(input_file)
for line in input_fd:
try:
parts = line.split("\t")
key = parts[0]
if key != remove_control_chars(key):
print "skip key: " + ":".join("{:02x}".format(ord(c)) for c in key) + " due to containing control characters"
skipped_keys +=1
value = parts[1]
shard = keyvi.JumpConsistentHashString(key, shards)
compilers[shard].Add(key, value)
except:
print "failed to add: " + line
print "Skipped keys " + str(skipped_keys)
for i in range(jobs):
t = threading.Thread(target=compile_worker)
t.daemon = True
t.start()
for i in range (0, shards):
compile_queue.put((compilers[i], output + "-" + str(i)))
compile_queue.join()