Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async def actor_execute(
self, comm=None, actor=None, function=None, args=(), kwargs={}
):
separate_thread = kwargs.pop("separate_thread", True)
key = actor
actor = self.actors[key]
func = getattr(actor, function)
name = key_split(key) + "." + function
if iscoroutinefunction(func):
result = await func(*args, **kwargs)
elif separate_thread:
result = await self.executor_submit(
name,
apply_function_actor,
args=(
func,
args,
kwargs,
self.execution_state,
name,
self.active_threads,
self.active_threads_lock,
),
def color_of_message(msg):
if msg["status"] == "OK":
split = key_split(msg["key"])
return color_of(split)
else:
return "black"
def get(self):
resp = {addr: [ensure_string(key_split(t)) for t in tasks]
for addr, tasks in self.server.processing.items()}
self.write(resp)
def task_stream_append(lists, msg, workers, palette=task_stream_palette):
start, stop = msg['compute_start'], msg['compute_stop']
lists['start'].append((start + stop) / 2 * 1000)
lists['duration'].append(1000 * (stop - start))
key = msg['key']
name = key_split(key)
if msg['status'] == 'OK':
color = palette[incrementing_index(name) % len(palette)]
else:
color = 'black'
lists['key'].append(key)
lists['name'].append(name)
lists['color'].append(color)
lists['alpha'].append(1)
lists['worker'].append(msg['worker'])
worker_thread = '%s-%d' % (msg['worker'], msg['thread'])
lists['worker_thread'].append(worker_thread)
if worker_thread not in workers:
workers[worker_thread] = len(workers)
lists['y'].append(workers[worker_thread])
def get_task_duration(self, key, default=0.5):
"""
Get the estimated computation cost of the given key
(not including any communication cost).
"""
ks = key_split(key)
try:
return self.task_duration[ks]
except KeyError:
self.unknown_durations[ks].add(key)
return default
def process_msg(self, msg):
try:
def func(k):
return msg["keys"].get(k, 0)
status_key = max(msg["keys"], key=func)
typ = self.worker.types.get(status_key, object).__name__
keyname = key_split(status_key)
d = {
"nbytes": msg["total"],
"duration": msg["duration"],
"bandwidth": msg["bandwidth"],
"count": len(msg["keys"]),
"type": typ,
"type-color": color_of(typ),
"key": keyname,
"key-color": color_of(keyname),
"start": msg["start"],
"stop": msg["stop"],
}
return d
except Exception as e:
logger.exception(e)
raise
def func(scheduler):
""" Get tasks running or waiting on each worker """
workers = [k for k, v in sorted(scheduler.ncores.items(),
key=lambda x: x[0], reverse=True)]
processing = [list(map(key_split, scheduler.processing[w]))
for w in workers]
nprocessing = list(map(len, processing))
nstacks = [len(scheduler.stacks[w]) for w in workers]
return {'host': [w.split(':')[0] for w in workers],
'processing': processing,
'nprocessing': nprocessing,
'waiting': nstacks}
We stop the task from being stolen in the future, and change task
duration accounting as if the task has stopped.
"""
if 'stealing' in self.extensions:
self.extensions['stealing'].remove_key_from_stealable(key)
try:
actual_worker = self.rprocessing[key]
except KeyError:
logger.debug("Received long-running signal from duplicate task. "
"Ignoring.")
return
if compute_duration:
ks = key_split(key)
old_duration = self.task_duration.get(ks, 0)
new_duration = compute_duration
if not old_duration:
avg_duration = new_duration
else:
avg_duration = (0.5 * old_duration
+ 0.5 * new_duration)
self.task_duration[ks] = avg_duration
worker = self.rprocessing[key]
self.occupancy[actual_worker] -= self.processing[actual_worker][key]
self.processing[actual_worker][key] = 0
finish: string
Final state of the transition.
*args, **kwargs: More options passed when transitioning
This may include worker ID, compute time, etc.
"""
if key not in self.scheduler.tasks:
return
kwargs["key"] = key
startstops = kwargs.get("startstops", [])
for startstop in startstops:
color = colors[startstop["action"]]
if type(color) is not str:
color = color(kwargs)
data = {
"key": key,
"name": key_split(key),
"color": color,
**kwargs,
**startstop,
}
self.socket.send("transition", data)
'cpu': 0.0,
'last-seen': 0.003068,
'latency': 0.01584628690034151,
'addresses': ['tcp://127.0.0.1:54871', 'tcp://127.0.0.1:50943'],
'processing': {'inc': 2, 'add': 1},
'disk-read': 1234,
'disk-write': 1234,
'network-send': 1234,
'network-recv': 1234,
'memory': 16701911040,
'memory_percent': 85}}
"""
hosts = {host: d['addresses']
for host, d in s.host_info.items()}
processing = {host: countby(key_split, concat(s.processing[w] for w in addrs))
for host, addrs in hosts.items()}
now = time()
result = {}
for host, info in s.host_info.items():
info = info.copy()
# info = dissoc(info, 'heartbeat', 'heartbeat-port')
info['processing'] = processing[host]
result[host] = info
info['addresses'] = sorted(info['addresses'])
if 'last-seen' in info:
info['last-seen'] = (now - info['last-seen'])
return result