How to use deeplabcut - 10 common examples

For details on TensorPack check out:

A Neural Net Training Interface on TensorFlow, with focus on speed + flexibility
https://github.com/tensorpack/tensorpack
"""


import os
from pathlib import Path
os.environ['DLClight']='True'

import deeplabcut
# Loading example data set
path_config_file = os.path.join(os.getcwd(),'openfield-Pranav-2018-10-30/config.yaml')
cfg=deeplabcut.auxiliaryfunctions.read_config(path_config_file)




deeplabcut.load_demo_data(path_config_file)

##create one split and make Shuffle 2 and 3 have the same split.
'''
trainIndexes, testIndexes=deeplabcut.mergeandsplit(path_config_file,trainindex=0,uniform=True)
deeplabcut.create_training_dataset(path_config_file,Shuffles=[2],trainIndexes=trainIndexes,testIndexes=testIndexes)
deeplabcut.create_training_dataset(path_config_file,Shuffles=[3],trainIndexes=trainIndexes,testIndexes=testIndexes)
for shuffle in [2,3]:
	if shuffle==3:
		posefile=os.path.join(cfg['project_path'],'dlc-models/iteration-'+str(cfg['iteration'])+'/'+ cfg['Task'] + cfg['date'] + '-trainset' + str(int(cfg['TrainingFraction'][0] * 100)) + 'shuffle' + str(shuffle),'train/pose_cfg.yaml')

		DLC_config=deeplabcut.auxiliaryfunctions.read_plainconfig(posefile)

newvideo=deeplabcut.ShortenVideo(video[0],start='00:00:00',stop='00:00:00.4',outsuffix='short',outpath=os.path.join(cfg['project_path'],'videos'))
    vname=Path(newvideo).stem
except: # if ffmpeg is broken
    vname='brief'
    newvideo=os.path.join(cfg['project_path'],'videos',vname+'.mp4')
    from moviepy.editor import VideoFileClip,VideoClip
    clip = VideoFileClip(video[0])
    clip.reader.initialize()
    def make_frame(t):
        return clip.get_frame(1)

    newclip = VideoClip(make_frame, duration=1)
    newclip.write_videofile(newvideo,fps=30)


deeplabcut.analyze_videos(path_config_file,[newvideo],save_as_csv=True, destfolder=dfolder, dynamic=(True,.1,5))

print("analyze again...")
deeplabcut.analyze_videos(path_config_file,[newvideo],save_as_csv=True, destfolder=dfolder)

print("CREATE VIDEO")
deeplabcut.create_labeled_video(path_config_file,[newvideo], destfolder=dfolder,save_frames=True)

print("Making plots")
deeplabcut.plot_trajectories(path_config_file,[newvideo], destfolder=dfolder)

print("EXTRACT OUTLIERS")
deeplabcut.extract_outlier_frames(path_config_file,[newvideo],outlieralgorithm='jump',epsilon=0,automatic=True, destfolder=dfolder)
file=os.path.join(cfg['project_path'],'labeled-data',vname,"machinelabels-iter"+ str(cfg['iteration']) + '.h5')

print("RELABELING")
DF=pd.read_hdf(file,'df_with_missing')

vname=Path(newvideo2).stem
except: # if ffmpeg is broken
    vname='brief'
    newvideo2=os.path.join(cfg['project_path'],'videos',vname+'.mp4')
    from moviepy.editor import VideoFileClip,VideoClip
    clip = VideoFileClip(video[0])
    clip.reader.initialize()
    def make_frame(t):
        return clip.get_frame(1)

    newclip = VideoClip(make_frame, duration=1)
    newclip.write_videofile(newvideo2,fps=30)


print("Inference with direct cropping")
deeplabcut.analyze_videos(path_config_file,[newvideo2],destfolder=dfolder,cropping=[0,50,0,50],save_as_csv=True)

print("Extracting skeleton distances, filter and plot filtered output")
deeplabcut.analyzeskeleton(path_config_file, [newvideo], save_as_csv=True, destfolder=dfolder)
deeplabcut.filterpredictions(path_config_file,[newvideo])

#deeplabcut.create_labeled_video(path_config_file,[newvideo], destfolder=dfolder,filtered=True)
deeplabcut.create_labeled_video(path_config_file,[newvideo2], destfolder=dfolder,displaycropped=True,filtered=True)
deeplabcut.plot_trajectories(path_config_file,[newvideo2], destfolder=dfolder,filtered=True)

print("CREATING TRAININGSET for shuffle 2")
print("will be used for 3D testscript...")
deeplabcut.create_training_dataset(path_config_file,Shuffles=[2],net_type=net_type,augmenter_type=augmenter_type3)

posefile=os.path.join(cfg['project_path'],'dlc-models/iteration-'+str(cfg['iteration'])+'/'+ cfg['Task'] + cfg['date'] + '-trainset' + str(int(cfg['TrainingFraction'][0] * 100)) + 'shuffle' + str(2),'train/pose_cfg.yaml')

DLC_config=deeplabcut.auxiliaryfunctions.read_plainconfig(posefile)

###Note that the new function in DLC 2.1 does that much easier...
deeplabcut.create_training_model_comparison(path_config_file,num_shuffles=1,net_types=['resnet_50'],augmenter_types=['imgaug','default','tensorpack'])

for shuffle in [2,3]:
	print("TRAIN NETWORK", shuffle)
	deeplabcut.train_network(path_config_file, shuffle=shuffle,saveiters=10000,displayiters=200,maxiters=5,max_snapshots_to_keep=11)

	print("EVALUATE")
	deeplabcut.evaluate_network(path_config_file, Shuffles=[shuffle],plotting=True)

	print("Analyze Video")

	videofile_path = os.path.join(os.getcwd(),'openfield-Pranav-2018-10-30','videos','m3v1mp4.mp4')

	deeplabcut.analyze_videos(path_config_file,[videofile_path], shuffle=shuffle)

	print("Create Labeled Video and plot")
	deeplabcut.create_labeled_video(path_config_file,[videofile_path], shuffle=shuffle)
	deeplabcut.plot_trajectories(path_config_file,[videofile_path], shuffle=shuffle)

print("TRAIN")
deeplabcut.train_network(path_config_file)

#this is much easier now: deeplabcut.train_network(path_config_file,gputouse=0,max_snapshots_to_keep=None,saveiters=1)

print("EVALUATE")
deeplabcut.evaluate_network(path_config_file,plotting=True)

print("CUT SHORT VIDEO AND ANALYZE")

# Make super short video (so the analysis is quick!)
vname='brief'
newvideo=os.path.join(cfg['project_path'],'videos',vname+'.avi')

subprocess.call(['ffmpeg','-i',video[0],'-ss','00:00:00','-to','00:00:00.4','-c','copy',newvideo])
deeplabcut.analyze_videos(path_config_file,[newvideo])

print("CREATE VIDEO")
deeplabcut.create_labeled_video(path_config_file,[newvideo])


print("EXTRACT OUTLIERS")
deeplabcut.extract_outlier_frames(path_config_file,[newvideo],outlieralgorithm='jump',epsilon=0,automatic=True)

file=os.path.join(cfg['project_path'],'labeled-data',vname,"machinelabels-iter"+ str(cfg['iteration']) + '.h5')

print("RELABELING")
DF=pd.read_hdf(file,'df_with_missing')
DLCscorer=np.unique(DF.columns.get_level_values(0))[0]
DF.columns.set_levels([scorer.replace(DLCscorer,scorer)],level=0,inplace=True)
DF =DF.drop('likelihood',axis=1,level=2)
DF.to_csv(os.path.join(cfg['project_path'],'labeled-data',vname,"CollectedData_" + scorer + ".csv"))

deeplabcut.extract_outlier_frames(path_config_file,[newvideo],outlieralgorithm='jump',epsilon=0,automatic=True, destfolder=dfolder)
file=os.path.join(cfg['project_path'],'labeled-data',vname,"machinelabels-iter"+ str(cfg['iteration']) + '.h5')

print("RELABELING")
DF=pd.read_hdf(file,'df_with_missing')
DLCscorer=np.unique(DF.columns.get_level_values(0))[0]
DF.columns.set_levels([scorer.replace(DLCscorer,scorer)],level=0,inplace=True)
DF =DF.drop('likelihood',axis=1,level=2)
DF.to_csv(os.path.join(cfg['project_path'],'labeled-data',vname,"CollectedData_" + scorer + ".csv"))
DF.to_hdf(os.path.join(cfg['project_path'],'labeled-data',vname,"CollectedData_" + scorer + '.h5'),'df_with_missing',format='table', mode='w')

print("MERGING")
deeplabcut.merge_datasets(path_config_file)

print("CREATING TRAININGSET")
deeplabcut.create_training_dataset(path_config_file,net_type=net_type,augmenter_type=augmenter_type2)

cfg=deeplabcut.auxiliaryfunctions.read_config(path_config_file)
posefile=os.path.join(cfg['project_path'],'dlc-models/iteration-'+str(cfg['iteration'])+'/'+ cfg['Task'] + cfg['date'] + '-trainset' + str(int(cfg['TrainingFraction'][0] * 100)) + 'shuffle' + str(1),'train/pose_cfg.yaml')
DLC_config=deeplabcut.auxiliaryfunctions.read_plainconfig(posefile)
DLC_config['save_iters']=numiter
DLC_config['display_iters']=1
DLC_config['multi_step']=[[0.001,numiter]]

print("CHANGING training parameters to end quickly!")
deeplabcut.auxiliaryfunctions.write_config(posefile,DLC_config)

print("TRAIN")
deeplabcut.train_network(path_config_file)

try: #you need ffmpeg command line interface
    #subprocess.call(['ffmpeg','-i',video[0],'-ss','00:00:00','-to','00:00:00.4','-c','copy',newvideo])

columnindex = pd.MultiIndex.from_product([[scorer], [bodypart], ['x', 'y']],names=['scorer', 'bodyparts', 'coords'])
        frame = pd.DataFrame(100+np.ones((len(frames),2))*50*index, columns = columnindex, index = [os.path.join('labeled-data',videoname,fn) for fn in frames])
        if index==0:
            dataFrame=frame
        else:
            dataFrame = pd.concat([dataFrame, frame],axis=1)

dataFrame.to_csv(os.path.join(cfg['project_path'],'labeled-data',videoname,"CollectedData_" + scorer + ".csv"))
dataFrame.to_hdf(os.path.join(cfg['project_path'],'labeled-data',videoname,"CollectedData_" + scorer + '.h5'),'df_with_missing',format='table', mode='w')

print("Plot labels...")

deeplabcut.check_labels(path_config_file)

print("CREATING TRAININGSET")
deeplabcut.create_training_dataset(path_config_file)

posefile=os.path.join(cfg['project_path'],'dlc-models/iteration-'+str(cfg['iteration'])+'/'+ cfg['Task'] + cfg['date'] + '-trainset' + str(int(cfg['TrainingFraction'][0] * 100)) + 'shuffle' + str(1),'train/pose_cfg.yaml')

DLC_config=deeplabcut.auxiliaryfunctions.read_plainconfig(posefile)
DLC_config['save_iters']=10
DLC_config['display_iters']=1
DLC_config['multi_step']=[[0.001,10]]

print("CHANGING training parameters to end quickly!")
deeplabcut.auxiliaryfunctions.write_plainconfig(posefile,DLC_config)

print("TRAIN")
deeplabcut.train_network(path_config_file)

print("TRAIN again... different loss?")
deeplabcut.train_network(path_config_file)

file=os.path.join(cfg['project_path'],'labeled-data',vname,"machinelabels-iter"+ str(cfg['iteration']) + '.h5')

print("RELABELING")
DF=pd.read_hdf(file,'df_with_missing')
DLCscorer=np.unique(DF.columns.get_level_values(0))[0]
DF.columns.set_levels([scorer.replace(DLCscorer,scorer)],level=0,inplace=True)
DF =DF.drop('likelihood',axis=1,level=2)
DF.to_csv(os.path.join(cfg['project_path'],'labeled-data',vname,"CollectedData_" + scorer + ".csv"))
DF.to_hdf(os.path.join(cfg['project_path'],'labeled-data',vname,"CollectedData_" + scorer + '.h5'),'df_with_missing',format='table', mode='w')

print("MERGING")
deeplabcut.merge_datasets(path_config_file)

print("CREATING TRAININGSET")
deeplabcut.create_training_dataset(path_config_file)

cfg=read_config(path_config_file)
posefile=os.path.join(cfg['project_path'],'dlc-models/iteration-'+str(cfg['iteration'])+'/'+ cfg['Task'] + cfg['date'] + '-trainset' + str(int(cfg['TrainingFraction'][0] * 100)) + 'shuffle' + str(1),'train/pose_cfg.yaml')
DLC_config=read_config(posefile)
DLC_config['save_iters']=5
DLC_config['display_iters']=1
DLC_config['multi_step']=[[0.05,5]]

print("CHANGING training parameters to end quickly!")
write_config(posefile,DLC_config)

print("TRAIN")
deeplabcut.train_network(path_config_file)


print("ALL DONE!!! - default cases are functional.")

print("Inference with direct cropping")
deeplabcut.analyze_videos(path_config_file,[newvideo2],destfolder=dfolder,cropping=[0,50,0,50],save_as_csv=True)

print("Extracting skeleton distances, filter and plot filtered output")
deeplabcut.analyzeskeleton(path_config_file, [newvideo], save_as_csv=True, destfolder=dfolder)
deeplabcut.filterpredictions(path_config_file,[newvideo])

#deeplabcut.create_labeled_video(path_config_file,[newvideo], destfolder=dfolder,filtered=True)
deeplabcut.create_labeled_video(path_config_file,[newvideo2], destfolder=dfolder,displaycropped=True,filtered=True)
deeplabcut.plot_trajectories(path_config_file,[newvideo2], destfolder=dfolder,filtered=True)

print("CREATING TRAININGSET for shuffle 2")
print("will be used for 3D testscript...")
deeplabcut.create_training_dataset(path_config_file,Shuffles=[2],net_type=net_type,augmenter_type=augmenter_type3)

posefile=os.path.join(cfg['project_path'],'dlc-models/iteration-'+str(cfg['iteration'])+'/'+ cfg['Task'] + cfg['date'] + '-trainset' + str(int(cfg['TrainingFraction'][0] * 100)) + 'shuffle' + str(2),'train/pose_cfg.yaml')

DLC_config=deeplabcut.auxiliaryfunctions.read_plainconfig(posefile)
DLC_config['save_iters']=10
DLC_config['display_iters']=2
DLC_config['multi_step']=[[0.001,10]]

print("CHANGING training parameters to end quickly!")
deeplabcut.auxiliaryfunctions.write_plainconfig(posefile,DLC_config)

print("TRAINING shuffle 2, with smaller allocated memory")
deeplabcut.train_network(path_config_file,shuffle=2,allow_growth=True)

print("ANALYZING some individual frames")
deeplabcut.analyze_time_lapse_frames(path_config_file,os.path.join(cfg['project_path'],'labeled-data/reachingvideo1/'))

The analysis of the video takes 41 seconds (batch size 32) and creating the frames 8 seconds (+ a few seconds for ffmpeg) to create the video.
"""

# Importing the toolbox (takes several seconds)
import deeplabcut
import os
from pathlib import Path

# Loading example data set
path_config_file = os.path.join(os.getcwd(),'openfield-Pranav-2018-10-30/config.yaml')
#deeplabcut.load_demo_data(path_config_file)

#shuffle=11 #>> imageio functions!
shuffle=12

deeplabcut.create_training_dataset(path_config_file,Shuffles=[shuffle])
cfg=deeplabcut.auxiliaryfunctions.read_config(path_config_file)
posefile=os.path.join(cfg['project_path'],'dlc-models/iteration-'+str(cfg['iteration'])+'/'+ cfg['Task'] + cfg['date'] + '-trainset' + str(int(cfg['TrainingFraction'][0] * 100)) + 'shuffle' + str(shuffle),'train/pose_cfg.yaml')

DLC_config=deeplabcut.auxiliaryfunctions.read_plainconfig(posefile)
DLC_config['save_iters']=10
DLC_config['display_iters']=2
DLC_config['multi_step']=[[0.005,15001]]
deeplabcut.auxiliaryfunctions.write_plainconfig(posefile,DLC_config)


print("TRAIN NETWORK")
deeplabcut.train_network(path_config_file, shuffle=shuffle,saveiters=15000,displayiters=1000,max_snapshots_to_keep=15)

print("EVALUATE")
deeplabcut.evaluate_network(path_config_file, Shuffles=[shuffle],plotting=True)