Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
for num, line in enumerate(jsonl_input, 1): # records start from 1
if num < start_record:
continue
if max_records is not None and num >= start_record + max_records:
break
yield num, line
cmd = _worker_command_line(thing, arguments)
processes = int(arguments['--processes'])
if hasattr(ckan, 'parallel_limit'):
# add your sites to ckanapi.remoteckan.MY_SITES instead of removing
processes = min(processes, ckan.parallel_limit)
stats = completion_stats(processes)
pool = worker_pool(cmd, processes, line_reader())
with quiet_int_pipe() as errors:
for job_ids, finished, result in pool:
if not result:
# child exited with traceback
return 1
timestamp, action, error, response = json.loads(
result.decode('utf-8'))
if not arguments['--quiet']:
stderr.write(('%s %s %s %s %s %s\n' % (
finished,
job_ids,
next(stats),
action,
error,
compact_json(response).decode('utf-8') if response else ''
)).encode('utf-8'))
break
yield num, compact_json(name)
cmd = _worker_command_line(thing, arguments)
processes = int(arguments['--processes'])
if hasattr(ckan, 'parallel_limit'):
# add your sites to ckanapi.remoteckan.MY_SITES instead of removing
processes = min(processes, ckan.parallel_limit)
stats = completion_stats(processes)
if not arguments['ID_OR_NAME']:
pool = worker_pool(cmd, processes, name_reader())
else:
pool = worker_pool(cmd, processes, enumerate(
(compact_json(n) + b'\n' for n in arguments['ID_OR_NAME']), 1))
with quiet_int_pipe() as errors:
for job_ids, finished, result in pool:
if not result:
# child exited with traceback
return 1
timestamp, error, response = json.loads(
result.decode('utf-8'))
if not arguments['--quiet']:
stderr.write(('%s %s %s %s %s\n' % (
finished,
job_ids,
next(stats),
error,
compact_json(response).decode('utf-8') if response else ''
)).encode('utf-8'))
names = [rec.get('name',rec.get('id')) for rec in names]
if arguments['--datapackages']:
arguments['--datastore-fields'] = True
cmd = _worker_command_line(thing, arguments)
processes = int(arguments['--processes'])
if hasattr(ckan, 'parallel_limit'):
# add your sites to ckanapi.remoteckan.MY_SITES instead of removing
processes = min(processes, ckan.parallel_limit)
stats = completion_stats(processes)
pool = worker_pool(cmd, processes,
enumerate(compact_json(n) + b'\n' for n in names))
results = {}
expecting_number = 0
with quiet_int_pipe() as errors:
for job_ids, finished, result in pool:
if not result:
# child exited with traceback
return 1
timestamp, error, record = json.loads(result.decode('utf-8'))
results[finished] = record
if not arguments['--quiet']:
stderr.write('{0} {1} {2} {3} {4}\n'.format(
finished,
job_ids,
next(stats),
error,
record.get('name', '') if record else '',
).encode('utf-8'))