Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def load_data(path):
try:
data_filename = get_filename(path, 'trackedobjects.pickle')
pd = pandas.read_pickle(data_filename)
except:
data_filename = get_filename(path, 'trackedobjects.hdf5')
data_filename_pickled = data_filename.split('.')[0] + '.pickle'
try:
pd = pandas.read_pickle(data_filename_pickled)
except:
pd = mta.read_hdf5_file_to_pandas.load_data_as_pandas_dataframe_from_hdf5_file(data_filename)
pandas.to_pickle(pd, data_filename_pickled)
pd = read_hdf5_file_to_pandas.remove_rows_with_above_speed_threshold(pd, speed_threshold=2)
return pd
print( '!!! MRS-Reaction: {}'.format(mrs_reaction_path))
mrs_reaction = mg.load_dataframe(mrs_reaction_path)
c2r = None
else:
print("!!! loading compound-reactions table")
# Loading c2r
c2r = pd.read_pickle(my_settings.c2r)
reference_compounds = None
mrs_reaction = None
# load the MST chemical network
with open(my_settings.mst_path, 'r') as f:
chemical_network = nx.read_graphml(f, node_type=int)
#chemnet files
print( '!!! loading chemnet files')
cpd_group_lookup = pd.read_pickle(my_settings.chemnet_pickle)
return c2r, mrs_reaction, reference_compounds, cpd_group_lookup, chemical_network
def loadData(fileName):
dataFile = os.path.join(dataPath, fileName)
pickleDump = '{}.pickle'.format(dataFile)
if os.path.exists(pickleDump):
df = pd.read_pickle(pickleDump)
else:
df = pd.read_csv(dataFile)
df = df.dropna()
df = shuffle(df)
df.to_pickle(pickleDump)
return df
def time_packers_read_pickle(self):
pd.read_pickle(self.f)
if not os.path.exists( trainset_path ) or not os.path.exists( testset_path ):
imagenet_images = []
for dir, _, _, in os.walk(dataset_path):
imagenet_images.extend( glob( os.path.join(dir, '*.jpg')))
imagenet_images = np.hstack(imagenet_images)
trainset = pd.DataFrame({'image_path':imagenet_images[:int(len(imagenet_images)*0.9)]})
testset = pd.DataFrame({'image_path':imagenet_images[int(len(imagenet_images)*0.9):]})
trainset.to_pickle( trainset_path )
testset.to_pickle( testset_path )
else:
trainset = pd.read_pickle( trainset_path )
testset = pd.read_pickle( testset_path )
testset.index = range(len(testset))
testset = testset.ix[np.random.permutation(len(testset))]
is_train = tf.placeholder( tf.bool )
learning_rate = tf.placeholder( tf.float32, [])
images_tf = tf.placeholder( tf.float32, [batch_size, 128, 128, 3], name="images")
labels_D = tf.concat( 0, [tf.ones([batch_size]), tf.zeros([batch_size])] )
labels_G = tf.ones([batch_size])
images_hiding = tf.placeholder( tf.float32, [batch_size, hiding_size, hiding_size, 3], name='images_hiding')
model = Model()
bn1, bn2, bn3, bn4, bn5, bn6, debn4, debn3, debn2, debn1, reconstruction_ori, reconstruction = model.build_reconstruction(images_tf, is_train)
adversarial_pos = model.build_adversarial(images_hiding, is_train)
def load(self, filename, filetype=None):
"""Load file, if no filetype given assume it's msgpack format"""
if filetype == '.pickle':
self.df = pd.read_pickle(filename)
else:
self.df = pd.read_msgpack(filename)
#print (len(self.df))
return
print("> Reading dataset")
# If file already exists on disk check when was previously downloaded
if os.path.isfile(self.dir_path + "/data/" + self.city + "/" + self.city + ".pkl"):
mtime = os.path.getmtime(self.dir_path + "/data/" + self.city + "/" + self.city + ".pkl")
last_modified_date = datetime.fromtimestamp(mtime)
timeDiff = datetime.now() - last_modified_date
if timeDiff.days < self.dbDaysQueryThresshold:
print("> Dataset was downloaded " + str(timeDiff.days) + " days ago.")
dataset = pd.read_pickle(self.dir_path + "/data/" + self.city + "/" + self.city + ".pkl")
self.timer.stop("Dataset was downloaded " + str(timeDiff.days) + " days ago.")
# If the data os old enough query the server
else:
# Query to make to the db
query_all = 'select * from bikes'
dataset = pd.DataFrame(self.client.query(query_all, chunked=True).get_points())
#dataset.drop(dataset.columns[[0]], axis = 1, inplace = True)
dataset["value"] = pd.to_numeric(dataset["value"])
if no_date_split == False:
mpl.rcParams['font.size'] = 13
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import ticker
from cogspaces.datasets.utils import get_output_dir
from exps.grids.gather_quantitative import get_full_subjects
output_dir = get_output_dir()
baselines = [
pd.read_pickle(join(output_dir, 'logistic_tc', 'accuracies_mean.pkl')),
pd.read_pickle(join(output_dir, 'logistic_tc_2', 'accuracies_mean.pkl'))]
baseline = pd.concat(baselines, axis=0)
factored = [
pd.read_pickle(join(output_dir, 'training_curves', 'accuracies_mean.pkl')),
pd.read_pickle(
join(output_dir, 'training_curves_2', 'accuracies_mean.pkl'))]
factored = pd.concat(factored, axis=0)
data = pd.concat((baseline, factored), keys=['baseline', 'factored'],
names=['method'])
data = data.drop('henson2010faces', axis=0, level='study')
subjects = get_full_subjects()
print(subjects)
import sys
import processing_utils as utils
import argparse
PATH = '../data/'
MIN_LENGTH_PAST = 10
MIN_LENGTH_FUTURE= 15
VELOCITY_FRAMES = 5
parser = argparse.ArgumentParser()
parser.add_argument('--detector', '-d', help="Use detections from 'yolo' or 'faster-rcnn'", type= str, default='yolo')
args = parser.parse_args()
if args.detector == 'yolo':
features = pd.read_pickle(PATH + 'bdd_10k_location_features_yolo.pkl')
else:
features = pd.read_pickle(PATH + 'bdd_10k_location_features_faster-rcnn.pkl')
features = features[features['Labeled']==1]
features['Final_x'] = features['Future_x'].apply(lambda x: x[-1])
features['Final_y'] = features['Future_y'].apply(lambda y: y[-1])
##################### Compute velocity and make predictions #####################
print('Computing velocity...')
features['Velocity_x'] = features['Past_x'].apply(lambda x: utils.mean_velocity(x,VELOCITY_FRAMES))
features['Velocity_y'] = features['Past_y'].apply(lambda y: utils.mean_velocity(y,VELOCITY_FRAMES))
if __name__ == "__main__":
data_dir = Path(".").resolve() / "old_kw94_one_pickles"
dataset_names = [
"data_5000_5000_emax_draws",
"data_5000",
"data_2000_5000_emax_draws",
"data_2000",
"data_1000_5000_emax_draws",
"data_1000",
]
dataset_dict = {}
for dname in dataset_names:
path = (data_dir / dname).with_suffix(".pickle")
df = pd.read_pickle(path)
dataset_dict[dname] = df
start_params = pd.read_pickle("old_kw94_one_pickles/params.pickle")
_, base_options = rp.get_example_model("kw_94_one", with_data=False)
# calculate many scenarios with small dataset and not too many draws
sequences = ["random", "sobol"]
draws = [(3000, 2000), (500, 200), (1000, 400), (1500, 600), (2000, 800)]
taus = [10, 20, 50, 100, 200, 500]
# sequences = ["random"]
# draws = [(200, 100)]
# taus = [500]
size = 51