Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _check_filename(self, file_name):
if not os.path.exists(file_name):
if self.is_master:
print_and_log(["The file %s can not be found!" %file_name], 'error', logger)
sys.exit(1)
else:
all_times[elec, min_times[midx]:max_times[midx]] = True
comm.Barrier()
sys.stderr.flush()
print_and_log(['Node %d has collected %d spikes and rejected %d spikes' % (comm.rank, elt_count, rejected)], 'debug', logger)
gdata = all_gather_array(numpy.array([elt_count], dtype=numpy.float32), comm, 0)
gdata2 = gather_array(numpy.array([rejected], dtype=numpy.float32), comm, 0)
nb_elements = numpy.int64(numpy.sum(gdata))
nb_rejected = numpy.int64(numpy.sum(gdata2))
nb_total = numpy.int64(nb_elts*comm.size)
if ((smart_search and (gpass == 0)) or (not smart_search and (gpass == 1))) and nb_elements == 0:
if comm.rank == 0:
print_and_log(['No waveforms found! Are the data properly loaded??'], 'error', logger)
sys.exit(0)
if nb_elements == 0:
gpass = nb_repeats
if comm.rank == 0:
if gpass != 1:
print_and_log(["Found %d spikes over %d requested" %(nb_elements, nb_total)], 'default', logger)
if nb_elements == 0:
print_and_log(["No more spikes in the recording, stop searching"], 'info', logger)
else:
if isolation:
print_and_log(["Found %d isolated spikes over %d requested (%d rejected)" %(nb_elements, nb_total, nb_rejected)], 'default', logger)
else:
print_and_log(["Found %d spikes over %d requested (%d rejected)" %(nb_elements, nb_total, nb_rejected)], 'default', logger)
if nb_elements < 0.2*nb_total:
trig_in_ms = params.getboolean('triggers', 'trig_in_ms')
artefacts = numpy.loadtxt(params.get('triggers', 'trig_file'), comments =['#','//'])
windows = numpy.loadtxt(params.get('triggers', 'trig_windows'), comments =['#','//'])
make_plots = params.get('triggers', 'make_plots')
plot_path = os.path.join(params.get('data', 'file_out_suff'), 'plots')
if len(windows.shape) == 1:
windows = windows.reshape(1, 2)
if len(artefacts.shape) == 1:
artefacts = artefacts.reshape(1, 2)
if trig_in_ms:
if comm.rank == 0:
print_and_log(['Artefact times are read in ms'], 'debug', logger)
artefacts[:, 1] *= numpy.int64(data_file.sampling_rate*1e-3)
windows[:, 1] *= numpy.int64(data_file.sampling_rate*1e-3)
else:
if comm.rank == 0:
print_and_log(['Artefact times are read in timesteps'], 'debug', logger)
artefacts = artefacts.astype(numpy.int64)
windows = windows.astype(numpy.int64)
nb_stimuli = len(numpy.unique(artefacts[:, 0]))
mytest = numpy.all(numpy.in1d(numpy.unique(artefacts[:, 0]), numpy.unique(windows[:, 0])))
if not mytest:
if comm.rank == 0:
print_and_log(['Error in the trigger file: not all artefacts are defined'], 'error', logger)
sys.exit(0)
local_mergings = 0
cluster_results = {}
for p in search_peaks:
cluster_results[p] = {}
if gpass > 1:
for ielec in xrange(N_e):
for p in search_peaks:
result['tmp_%s_' %p + str(ielec)] = gather_array(result['tmp_%s_' %p + str(ielec)], comm, numpy.mod(ielec, comm.size), 1, compress=blosc_compress)
elif gpass == 1:
for ielec in xrange(comm.rank, N_e, comm.size):
result['times_' + str(ielec)] = numpy.copy(result['loc_times_' + str(ielec)])
if comm.rank == 0:
if gpass == 0:
print_and_log(["Estimating amplitudes distributions..."], 'default', logger)
elif gpass == 1:
print_and_log(["Computing density estimations..."], 'default', logger)
else:
print_and_log(["Refining density estimations..."], 'default', logger)
if not os.path.exists(plot_path):
os.makedirs(plot_path)
if gpass == 1:
dist_file = tempfile.NamedTemporaryFile()
tmp_file = os.path.join(tmp_path_loc, os.path.basename(dist_file.name)) + '.hdf5'
dist_file.close()
result['dist_file'] = tmp_file
tmp_h5py = h5py.File(result['dist_file'], 'w', libver='earliest')
print_and_log(["Node %d will use temp file %s" %(comm.rank, tmp_file)], 'debug', logger)
elif gpass > 1:
tmp_h5py = h5py.File(result['dist_file'], 'r', libver='earliest')
# Retrieve the key parameters.
data_file = params.data_file
N_e = params.getint('data', 'N_e')
N_t = params.getint('detection', 'N_t')
file_out_suff = params.get('data', 'file_out_suff')
max_chunk = params.getfloat('fitting', 'max_chunk')
hdf5_compress = params.getboolean('data', 'hdf5_compress')
data_length = data_stats(params, show=False)
duration = data_length
templates = load_data(params, 'norm-templates')
refractory = params.getint('fitting', 'refractory')
N_tm = len(templates)
collect_all = params.getboolean('fitting', 'collect_all')
debug = params.getboolean('fitting', 'debug')
print_and_log(["Gathering spikes from %d nodes..." %nb_threads], 'default', logger)
# Initialize data collection.
result = {'spiketimes' : {}, 'amplitudes' : {}, 'info' : {'duration' : numpy.array([duration], dtype=numpy.uint64)}}
if with_real_amps:
result['real_amps'] = {}
if with_voltages:
result['voltages'] = {}
if collect_all:
result['gspikes'] = {}
result['gtemps'] = {}
for i in xrange(N_tm//2):
result['spiketimes']['temp_' + str(i)] = numpy.empty(shape=0, dtype=numpy.uint32)
result['amplitudes']['temp_' + str(i)] = numpy.empty(shape=(0, 2), dtype=numpy.float32)
if with_real_amps:
result['real_amps']['temp_' + str(i)] = numpy.empty(shape=0, dtype=numpy.float32)
my_file = h5py.File(self.file_name, mode='r')
all_matches = [re.findall('\d+', u) for u in my_file.keys()]
all_streams = []
for m in all_matches:
if len(m) > 0:
all_streams += [int(m[0])]
idx = numpy.argsort(all_streams)
for i in xrange(len(all_streams)):
params['h5_key'] = my_file.keys()[idx[i]]
new_data = type(self)(self.file_name, params)
sources += [new_data]
to_write += ['We found the datafile %s with t_start %d and duration %d' %(new_data.file_name, new_data.t_start, new_data.duration)]
print_and_log(to_write, 'debug', logger)
return sources
elif stream_mode == 'multi-files':
return H5File.set_streams(stream_mode)
for item in result['spiketimes'].keys():
count += len(result['spiketimes'][item])
if collect_all:
gcount = 0
for item in result['gspikes'].keys():
gcount += len(result['gspikes'][item])
# Print log message.
if benchmark:
to_print = "injected"
else:
to_print = "fitted"
to_write = ["Number of spikes %s : %d" % (to_print, count)]
if collect_all:
to_write += ["Number of spikes not fitted (roughly): %d [%g percent]" % (gcount, 100 * gcount / float(count))]
print_and_log(to_write, 'info', logger)
if erase:
purge(file_out_suff, '.data')
if comm.rank == 0:
print_and_log(["Merging similar templates..."], 'default', logger)
merged1 = algo.merging_cc(params, parallel_hdf5)
comm.Barrier()
if remove_mixture:
if comm.rank == 0:
print_and_log(["Removing mixtures..."], 'default', logger)
merged2 = algo.delete_mixtures(params, parallel_hdf5)
else:
merged2 = [0, 0]
if comm.rank == 0:
print_and_log(["Number of global merges : %d" %merged1[1],
"Number of mixtures removed : %d" %merged2[1]], 'info', logger)
comm.Barrier()
io.get_overlaps(params, erase=True, parallel_hdf5=parallel_hdf5)
data_file.close()
## We will perform several passes to enhance the quality of the clustering
while gpass < (nb_repeats + 1):
comm.Barrier()
if gpass == 1:
sdata = all_gather_array(smart_searches[search_peaks[0]][comm.rank::comm.size], comm, 0)
if comm.rank == 0:
if gpass == 0:
print_and_log(["Searching random spikes to sample amplitudes..."], 'default', logger)
elif gpass == 1:
if not numpy.all(sdata > 0):
lines = ["Smart Search disabled on %d electrodes" %(numpy.sum(sdata == 0))]
print_and_log(lines, 'debug', logger)
if numpy.any(sdata > 0):
if isolation:
print_and_log(["Smart Search of good isolated spikes for the clustering (%d/%d)..." %(gpass, nb_repeats)], 'default', logger)
else:
print_and_log(["Smart Search of good spikes for the clustering (%d/%d)..." %(gpass, nb_repeats)], 'default', logger)
else:
print_and_log(["Searching random spikes for the clustering (%d/%d) (no smart search)" %(gpass, nb_repeats)], 'default', logger)
else:
print_and_log(["Searching random spikes to refine the clustering (%d/%d)..." %(gpass, nb_repeats)], 'default', logger)
for i in xrange(N_e):
if gpass == 0:
for p in search_peaks:
result['tmp_%s_' %p + str(i)] = numpy.zeros(0, dtype=numpy.float32)
result['nb_chunks_%s_' %p + str(i)] = 0
all_files.sort(key=natural_keys)
sources = []
to_write = []
global_time = 0
params = self.get_description()
for fname in all_files:
params['ncs_pattern'] = self.params['ncs_pattern']
new_data = type(self)(os.path.join(os.path.abspath(dirname), fname), params)
new_data._t_start = global_time
global_time += new_data.duration
sources += [new_data]
to_write += ['We found the datafile %s with t_start %s and duration %s' %(new_data.file_name, new_data.t_start, new_data.duration)]
print_and_log(to_write, 'debug', logger)
return sources