How to use the pandas.read_pickle function in pandas

To help you get started, we’ve selected a few pandas examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github florisvb / multi_tracker / multi_tracker_analysis / stitch_trajectories_gui.py View on Github external
def load_data(path):
    try:
        data_filename = get_filename(path, 'trackedobjects.pickle')
        pd = pandas.read_pickle(data_filename)
    except:
        data_filename = get_filename(path, 'trackedobjects.hdf5')
        data_filename_pickled = data_filename.split('.')[0] + '.pickle'
        try:
            pd = pandas.read_pickle(data_filename_pickled)
        except:
            pd = mta.read_hdf5_file_to_pandas.load_data_as_pandas_dataframe_from_hdf5_file(data_filename)
            pandas.to_pickle(pd, data_filename_pickled)
            
    pd = read_hdf5_file_to_pandas.remove_rows_with_above_speed_threshold(pd, speed_threshold=2)
            
    return pd
github biorack / magi / workflow / magi_workflow_compound_to_reaction.py View on Github external
print( '!!! MRS-Reaction: {}'.format(mrs_reaction_path))
        mrs_reaction = mg.load_dataframe(mrs_reaction_path)
        c2r = None
    else:
        print("!!! loading compound-reactions table")
        # Loading c2r
        c2r = pd.read_pickle(my_settings.c2r)
        reference_compounds = None
        mrs_reaction = None

    # load the MST chemical network
    with open(my_settings.mst_path, 'r') as f:
        chemical_network = nx.read_graphml(f, node_type=int)
    #chemnet files
    print( '!!! loading chemnet files')
    cpd_group_lookup = pd.read_pickle(my_settings.chemnet_pickle)

    return c2r, mrs_reaction, reference_compounds, cpd_group_lookup, chemical_network
github rambasnet / DeepLearning-IDS / fastai-expriments.py View on Github external
def loadData(fileName):
    dataFile = os.path.join(dataPath, fileName)
    pickleDump = '{}.pickle'.format(dataFile)
    if os.path.exists(pickleDump):
        df = pd.read_pickle(pickleDump)
    else:
        df = pd.read_csv(dataFile)
        df = df.dropna()
        df = shuffle(df)
        df.to_pickle(pickleDump)
    return df
github pandas-dev / pandas / asv_bench / benchmarks / packers.py View on Github external
def time_packers_read_pickle(self):
        pd.read_pickle(self.f)
github jazzsaxmafia / Inpainting / src / train.imagenet.py View on Github external
if not os.path.exists( trainset_path ) or not os.path.exists( testset_path ):
    imagenet_images = []
    for dir, _, _, in os.walk(dataset_path):
        imagenet_images.extend( glob( os.path.join(dir, '*.jpg')))

    imagenet_images = np.hstack(imagenet_images)

    trainset = pd.DataFrame({'image_path':imagenet_images[:int(len(imagenet_images)*0.9)]})
    testset = pd.DataFrame({'image_path':imagenet_images[int(len(imagenet_images)*0.9):]})

    trainset.to_pickle( trainset_path )
    testset.to_pickle( testset_path )
else:
    trainset = pd.read_pickle( trainset_path )
    testset = pd.read_pickle( testset_path )

testset.index = range(len(testset))
testset = testset.ix[np.random.permutation(len(testset))]
is_train = tf.placeholder( tf.bool )

learning_rate = tf.placeholder( tf.float32, [])
images_tf = tf.placeholder( tf.float32, [batch_size, 128, 128, 3], name="images")

labels_D = tf.concat( 0, [tf.ones([batch_size]), tf.zeros([batch_size])] )
labels_G = tf.ones([batch_size])
images_hiding = tf.placeholder( tf.float32, [batch_size, hiding_size, hiding_size, 3], name='images_hiding')

model = Model()

bn1, bn2, bn3, bn4, bn5, bn6, debn4, debn3, debn2, debn1, reconstruction_ori, reconstruction = model.build_reconstruction(images_tf, is_train)
adversarial_pos = model.build_adversarial(images_hiding, is_train)
github dmnfarrell / pandastable / pandastable / data.py View on Github external
def load(self, filename, filetype=None):
        """Load file, if no filetype given assume it's msgpack format"""

        if filetype == '.pickle':
            self.df = pd.read_pickle(filename)
        else:
            self.df = pd.read_msgpack(filename)
            #print (len(self.df))
        return
github javierdemartin / neural-bikes / Data_mgmt.py View on Github external
print("> Reading dataset")
		
		# If file already exists on disk check when was previously downloaded
		if os.path.isfile(self.dir_path + "/data/" + self.city + "/"  + self.city + ".pkl"):
	
			mtime = os.path.getmtime(self.dir_path + "/data/" + self.city + "/"  + self.city + ".pkl")
	
			last_modified_date = datetime.fromtimestamp(mtime)
	
			timeDiff = datetime.now() - last_modified_date
	
			if timeDiff.days < self.dbDaysQueryThresshold:
	
				print("> Dataset was downloaded " + str(timeDiff.days) + " days ago.")
				dataset = pd.read_pickle(self.dir_path + "/data/" + self.city + "/"  + self.city + ".pkl")

				self.timer.stop("Dataset was downloaded " + str(timeDiff.days) + " days ago.")
					 
			# If the data os old enough query the server
			else: 
			
				# Query to make to the db       
				query_all = 'select * from bikes'

				dataset = pd.DataFrame(self.client.query(query_all, chunked=True).get_points())
				
				#dataset.drop(dataset.columns[[0]], axis = 1, inplace = True) 

				dataset["value"] = pd.to_numeric(dataset["value"])

				if no_date_split == False:
github arthurmensch / cogspaces / exps / analyse / plot_training_curves.py View on Github external
mpl.rcParams['font.size'] = 13

import matplotlib.pyplot as plt

import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import ticker

from cogspaces.datasets.utils import get_output_dir
from exps.grids.gather_quantitative import get_full_subjects

output_dir = get_output_dir()

baselines = [
    pd.read_pickle(join(output_dir, 'logistic_tc', 'accuracies_mean.pkl')),
    pd.read_pickle(join(output_dir, 'logistic_tc_2', 'accuracies_mean.pkl'))]
baseline = pd.concat(baselines, axis=0)
factored = [
    pd.read_pickle(join(output_dir, 'training_curves', 'accuracies_mean.pkl')),
    pd.read_pickle(
        join(output_dir, 'training_curves_2', 'accuracies_mean.pkl'))]
factored = pd.concat(factored, axis=0)

data = pd.concat((baseline, factored), keys=['baseline', 'factored'],
                 names=['method'])

data = data.drop('henson2010faces', axis=0, level='study')

subjects = get_full_subjects()
print(subjects)
github olly-styles / Dynamic-Trajectory-Predictor / preprocessing / compute_cv_correction_bdd.py View on Github external
import sys
import processing_utils as utils
import argparse

PATH = '../data/'
MIN_LENGTH_PAST = 10
MIN_LENGTH_FUTURE= 15
VELOCITY_FRAMES = 5

parser = argparse.ArgumentParser()
parser.add_argument('--detector', '-d', help="Use detections from 'yolo' or 'faster-rcnn'", type= str, default='yolo')
args = parser.parse_args()


if args.detector == 'yolo':
    features = pd.read_pickle(PATH + 'bdd_10k_location_features_yolo.pkl')
else:
    features = pd.read_pickle(PATH + 'bdd_10k_location_features_faster-rcnn.pkl')



features = features[features['Labeled']==1]

features['Final_x'] = features['Future_x'].apply(lambda x: x[-1])
features['Final_y'] = features['Future_y'].apply(lambda y: y[-1])

##################### Compute velocity and make predictions #####################
print('Computing velocity...')

features['Velocity_x'] = features['Past_x'].apply(lambda x: utils.mean_velocity(x,VELOCITY_FRAMES))
features['Velocity_y'] = features['Past_y'].apply(lambda y: utils.mean_velocity(y,VELOCITY_FRAMES))
github OpenSourceEconomics / respy / run.py View on Github external
if __name__ == "__main__":

    data_dir = Path(".").resolve() / "old_kw94_one_pickles"
    dataset_names = [
        "data_5000_5000_emax_draws",
        "data_5000",
        "data_2000_5000_emax_draws",
        "data_2000",
        "data_1000_5000_emax_draws",
        "data_1000",
    ]

    dataset_dict = {}
    for dname in dataset_names:
        path = (data_dir / dname).with_suffix(".pickle")
        df = pd.read_pickle(path)
        dataset_dict[dname] = df

    start_params = pd.read_pickle("old_kw94_one_pickles/params.pickle")
    _, base_options = rp.get_example_model("kw_94_one", with_data=False)

    # calculate many scenarios with small dataset and not too many draws

    sequences = ["random", "sobol"]
    draws = [(3000, 2000), (500, 200), (1000, 400), (1500, 600), (2000, 800)]
    taus = [10, 20, 50, 100, 200, 500]
    # sequences = ["random"]
    # draws = [(200, 100)]
    # taus = [500]

    size = 51