Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def assign(mean_list, expression=expression):
logger.debug("assigning %r to %s", mean_list, expression)
mean_map[expression] = mean_list
subspace.mean().then(assign, on_error).end()
with dialogs.ProgressExecution(self, "Calculating means", executor=executor):
executor.execute()
var_map = {}
for expression in expressions:
subspace = self.dataset(expression, executor=executor, delay=True)
def assign(mean_list, expression=expression):
logger.debug("assigning %r to %s", mean_list, expression)
var_map[expression] = mean_list[0].tolist()
subspace.var(means=mean_map[expression]).then(assign, on_error).end()
with dialogs.ProgressExecution(self, "Calculating variances", executor=executor):
executor.execute()
means = [mean_map[expressions[0]] for expressions in pairs]
variances = [var_map[expressions[0]] for expressions in pairs]
correlation_map = {}
for pair in pairs:
means = [mean_map[expression] for expression in pair]
vars = [var_map[expression] for expression in pair]
subspace = self.dataset(*pair, executor=executor, delay=True)
def assign(correlation, pair=pair):
logger.debug("assigning %r to %s", correlation, pair)
correlation_map[pair] = correlation
subspace.correlation(means, vars).then(assign, on_error).end()
if index is None:
return
endian_option = ["=", "<", ">"][index]
logger.debug("export endian: %r", endian_option)
if type == "hdf5":
filename = dialogs.get_path_save(self, "Save to HDF5", name, "HDF5 *.hdf5")
else:
filename = dialogs.get_path_save(self, "Save to col-fits", name, "FITS (*.fits)")
logger.debug("export to file: %r", filename)
# print args
filename = str(filename)
if not filename.endswith("." + type):
filename += "." + type
if filename:
with dialogs.ProgressExecution(self, "Copying data...", "Abort export") as progress_dialog:
if type == "hdf5":
vaex.export.export_hdf5(dataset, filename, column_names=selected_column_names, shuffle=shuffle, selection=export_selection, byteorder=endian_option, progress=progress_dialog.progress)
if type == "fits":
vaex.export.export_fits(dataset, filename, column_names=selected_column_names, shuffle=shuffle, selection=export_selection, progress=progress_dialog.progress)
logger.debug("export done")
# print means_promise, type(means_promise), subs
with dialogs.ProgressExecution(self, "Calculating means", executor=executor) as progress:
progress.add_task(means_promise)
progress.execute()
means = means_promise.get()
variances_promise = subspaces.var(means=means)
with dialogs.ProgressExecution(self, "Calculating variances", executor=executor) as progress:
progress.add_task(variances_promise)
progress.execute()
vars = variances_promise.get()
# means = subspaces._unpack(means_packed)
# vars = subspaces._unpack(vars_packed)
tasks = []
with dialogs.ProgressExecution(self, "Calculating correlation", executor=executor) as progress:
for subspace, mean, var in zip(subspaces.subspaces, means, vars):
task = subspace.correlation(means=mean, vars=var)
progress.add_task(task).end()
tasks.append(task)
progress.execute()
correlations = [task.get() for task in tasks]
correlation_map = dict(zip(pairs, correlations))
table.set_correlations(correlation_map)
return
# mean_map = dict(zip(expressions, means))
# var_map = dict(zip(expressions, variances))
else:
mean_map = {}
def on_error(exc):
if layer.dataset.is_local():
executor = vaex.execution.Executor()
else:
executor = vaex.remote.ServerExecutor()
subspace = layer.dataset.subspace(*layer.state.expressions, executor=executor, delay=True)
means = subspace.mean()
with dialogs.ProgressExecution(plot_window, "Calculating mean", executor=executor) as progress:
progress.add_task(means)
progress.execute()
logger.debug("get means")
means = means.get()
logger.debug("got means")
vars = subspace.var(means=means)
with dialogs.ProgressExecution(plot_window, "Calculating variance", executor=executor) as progress:
progress.add_task(vars)
progress.execute()
#limits = limits.get()
vars = vars.get()
stds = vars**0.5
sigmas = 3
limits = list(zip(means-sigmas*stds, means+sigmas*stds))
#plot_window.ranges_show = limits
plot_window.set_ranges(range(len(limits)), limits, add_to_history=True, reason="3 sigma region")
#plot_window.update_all_layers()
#for layer in plot_window.layers:
# layer.flag_needs_update()
logger.debug("means=%r", means)
logger.debug("vars=%r", vars)
logger.debug("limits=%r", limits)
plot_window.queue_history_change("task: 3 sigma region")
def onCalculateMinMax(self):
pairs = self.table1d.getSelected()
logger.debug("estimate min/max for %r" % pairs)
if self.dataset.is_local():
executor = vaex.execution.Executor(buffer_size=buffer_size)
else:
executor = vaex.remote.ServerExecutor()
expressions = [pair[0] for pair in pairs]
assert len(pairs[0]) == 1
self.range_map = {}
try:
with dialogs.ProgressExecution(self, "Calculating min/max", executor=executor) as progress:
subspace = self.dataset.subspace(*expressions, executor=executor, delay=True)
minmax = subspace.minmax()
progress.add_task(minmax).end()
progress.execute()
ranges = minmax.get()
self.table1d.setRanges(pairs, ranges)
self.fill_range_map()
except:
logger.exception("Error in min/max or cancelled")
# dialog.hide()
def subtract_mean(plot_window):
if plot_window.layers:
layer = plot_window.layers[0]
executor = vaex.execution.Executor()
subspace = layer.dataset.subspace(*layer.state.expressions, executor=executor, delay=True)
means = subspace.mean()
with dialogs.ProgressExecution(plot_window, "Calculating mean", executor=executor):
executor.execute()
means = means.get()
new_expressions = ["(%s) - %s" % (expression, mean) for expression, mean in zip(layer.state.expressions, means)]
for i in range(len(new_expressions)):
vmin, vmax = layer.plot_window.state.ranges_viewport[i]
vmin -= means[i]
vmax -= means[i]
layer.plot_window.set_range(vmin, vmax, i)
for i in range(len(new_expressions)):
layer.set_expression(new_expressions[i], i)
plot_window.update_all_layers()
plot_window.queue_history_change("task: remove mean")
def onCalculateMinMax3Sigma(self):
pairs = self.table1d.getSelected()
expressions = [pair[0] for pair in pairs]
if self.dataset.is_local():
executor = vaex.execution.Executor(buffer_size=buffer_size)
else:
executor = vaex.remote.ServerExecutor()
if self.dataset.is_local():
executor = vaex.execution.Executor()
else:
executor = vaex.remote.ServerExecutor()
subspace = self.dataset.subspace(*expressions, executor=executor, delay=True)
means = subspace.mean()
with dialogs.ProgressExecution(self, "Calculating mean", executor=executor) as progress:
progress.add_task(means).end()
progress.execute()
logger.debug("get means")
means = means.get()
logger.debug("got means")
vars = subspace.var(means=means)
with dialogs.ProgressExecution(self, "Calculating variance", executor=executor) as progress:
progress.add_task(vars).end()
progress.execute()
# limits = limits.get()
vars = vars.get()
stds = vars**0.5
sigmas = 3
ranges = list(zip(means - sigmas * stds, means + sigmas * stds))
self.table1d.setRanges(pairs, ranges)
def sigma3(plot_window):
if plot_window.layers:
layer = plot_window.layers[0]
if layer.dataset.is_local():
executor = vaex.execution.Executor()
else:
executor = vaex.remote.ServerExecutor()
subspace = layer.dataset.subspace(*layer.state.expressions, executor=executor, delay=True)
means = subspace.mean()
with dialogs.ProgressExecution(plot_window, "Calculating mean", executor=executor) as progress:
progress.add_task(means)
progress.execute()
logger.debug("get means")
means = means.get()
logger.debug("got means")
vars = subspace.var(means=means)
with dialogs.ProgressExecution(plot_window, "Calculating variance", executor=executor) as progress:
progress.add_task(vars)
progress.execute()
#limits = limits.get()
vars = vars.get()
stds = vars**0.5
sigmas = 3
limits = list(zip(means-sigmas*stds, means+sigmas*stds))
#plot_window.ranges_show = limits