How to use joblib - 10 common examples

def load_rec():
    return load(join(get_output_dir(), 'benchmark', 'rec.pkl'))


# compute_rec()

exp_dirs = join(get_output_dir(), 'single_exp', '17')
models = []
rec = load_rec()
mask_img = fetch_mask()
masker = MultiNiftiMasker(mask_img=mask_img).fit()

for exp_dir in [exp_dirs]:
    estimator = load(join(exp_dirs, 'estimator.pkl'))
    transformer = load(join(exp_dirs, 'transformer.pkl'))
    for coef, (dataset, sc), (_, lbin) in zip(estimator.coef_, transformer.scs_.items(),
                                      transformer.lbins_.items()):
        print(dataset)
        classes = lbin.classes_
        print(classes)
        coef /= sc.scale_
        coef_rec = coef.dot(rec)
        # coef_rec -= np.mean(coef_rec, axis=0)
        print(join(exp_dirs, 'maps_%s.nii.gz' % dataset))
        imgs = masker.inverse_transform(coef_rec)
        imgs.to_filename(join(exp_dirs, 'maps_%s.nii.gz' % dataset))
        fig, axes = plt.subplots(len(classes) // 4, 4, figsize=(24, len(classes) // 4 * 3))
        axes = axes.ravel()
        for ax, img, this_class in zip(axes, iter_img(imgs), classes):
            this_class = this_class.replace('_', ' ')
            this_class = this_class.replace('&', ' ')

def run(n_jobs=1, n_epochs=10):
    # Exp def
    redundancies = [25]
    global_exp = dict(n_components=100, alpha=1,
                      l1_ratio=0,
                      pen_l1_ratio=.9,
                      learning_rate=0.9,
                      Dx_agg='average',
                      G_agg='average',
                      AB_agg='full')
    ref_batch_size = 200
    exps = [dict(batch_size=int(ref_batch_size),
                 reduction=reduction)
            for reduction in np.linspace(5, 5, 1)]

    mem = Memory(cachedir=expanduser('~/cache'))
    face = misc.face(gray=True)

    # Convert from uint8 representation with values between 0 and 255 to
    # a floating point representation with values between 0 and 1.
    face = face / 255

    height, width = face.shape

    # Distort the right half of the image
    print('Distorting image...')
    distorted = face.copy()
    # distorted[:, width // 2:] += 0.075 * np.random.randn(height, width // 2)

    # Extract all reference patches from the left half of the image
    print('Extracting reference patches...')
    t0 = time()

----------
        X : np.array
            List of datapoints

        Returns
        -------
        n_jobs : int
            Number of jobs
        list of int
            List of number of datapoints per job
        list of int
            List of start values for every job list

        """
        n_datapoints = len(X)
        n_jobs = min(effective_n_jobs(self.n_jobs), n_datapoints)

        n_datapoints_per_job = np.full(
            n_jobs, n_datapoints // n_jobs, dtype=np.int)

        n_datapoints_per_job[:n_datapoints % n_jobs] += 1
        starts = np.cumsum(n_datapoints_per_job)

        return n_jobs, n_datapoints_per_job.tolist(), [0] + starts.tolist()

for i in range (count):
                            result += [ (idx, 'GPU', '%s #%d' % (dev_name,i) , dev_vram) ]

                return result

            if cpu_only:
                if manual:
                    return [ (0, 'CPU', 'CPU', 0 ) ]
                else:
                    return [ (i, 'CPU', 'CPU%d' % (i), 0 ) for i in range( min(8, multiprocessing.cpu_count() // 2) ) ]

        elif type == 'final':
            return [ (i, 'CPU', 'CPU%d' % (i), 0 ) for i in (range(min(8, multiprocessing.cpu_count())) if not DEBUG else [0]) ]

class DeletedFilesSearcherSubprocessor(Subprocessor):
    class Cli(Subprocessor.Cli):
        #override
        def on_initialize(self, client_dict):
            self.debug_paths_stems = client_dict['debug_paths_stems']
            return None

        #override
        def process_data(self, data):
            input_path_stem = Path(data[0]).stem
            return any ( [ input_path_stem == d_stem for d_stem in self.debug_paths_stems] )

        #override
        def get_data_name (self, data):
            #return string identificator of your data
            return data[0]

    #override

def calculate_potential_3D_parallel(true_csd, ele_xx, ele_yy, ele_zz, 
                                    csd_x, csd_y, csd_z):
    """
    For Mihav's implementation to compute the LFP generated
    """

    xlin = csd_x[:,0,0]
    ylin = csd_y[0,:,0]
    zlin = csd_z[0,0,:]
    xlims = [xlin[0], xlin[-1]]
    ylims = [ylin[0], ylin[-1]]
    zlims = [zlin[0], zlin[-1]]
    sigma = 1.0
    #tic = time.time()
    pots = Parallel(n_jobs=num_cores)(delayed(integrate_3D)(ele_xx[ii],ele_yy[ii],ele_zz[ii],
                                                            xlims, ylims, zlims, true_csd,
                                                            xlin, ylin, zlin,
                                                            csd_x, csd_y, csd_z) for ii in range(len(ele_xx)))
    pots = np.array(pots)
    pots /= 4*np.pi*sigma
    #toc = time.time() - tic
    #print toc, 'Total time taken - parallel, sims '
    return pots
1

pipeline_2 = ResumablePipeline([], cache_folder=tmpdir)
    pipeline_2.name = 'pipeline2'
    pipeline_2.sub_steps_savers = [
        (SOME_STEP_2, []),
        (CHECKPOINT, []),
        (SOME_STEP_3, []),
    ]
    dump(pipeline_2, create_pipeline2_path(tmpdir, True))

    given_saved_some_step(multiply_by=2, name=SOME_STEP_1, path=create_some_step1_path(tmpdir, True))
    given_saved_some_step(multiply_by=4, name=SOME_STEP_2, path=create_some_step2_path(tmpdir, True))
    given_saved_some_step(multiply_by=6, name=SOME_STEP_3, path=create_some_step3_path(tmpdir, True))

    checkpoint = DefaultCheckpoint()
    checkpoint.name = CHECKPOINT
    dump(checkpoint, create_some_checkpoint_path(tmpdir, True))

    p = ResumablePipeline([
        (SOME_STEP_1, MultiplyByN(multiply_by=1)),
        (PIPELINE_2, ResumablePipeline([
            (SOME_STEP_2, MultiplyByN(multiply_by=1)),
            (CHECKPOINT, DefaultCheckpoint()),
            (SOME_STEP_3, MultiplyByN(multiply_by=1))
        ]))
    ], cache_folder=tmpdir)
    p.name = ROOT

    return p

internal_dim=2)
    
    test_policy = EpsilonGreedyPolicy(learning_algo, env.nActions(), rng, 1.)

    # --- Instantiate agent ---
    agent = NeuralAgent(
        env,
        learning_algo,
        parameters.replay_memory_size,
        max(env.inputDimensions()[i][0] for i in range(len(env.inputDimensions()))),
        parameters.batch_size,
        rng,
        test_policy=test_policy)

    # --- Create unique filename for FindBestController ---
    h = hash(vars(parameters), hash_name="sha1")
    fname = "test_" + h
    print("The parameters hash is: {}".format(h))
    print("The parameters are: {}".format(parameters))

    # As for the discount factor and the learning rate, one can update periodically the parameter of the epsilon-greedy
    # policy implemented by the agent. This controllers has a bit more capabilities, as it allows one to choose more
    # precisely when to update epsilon: after every X action, episode or epoch. This parameter can also be reset every
    # episode or epoch (or never, hence the resetEvery='none').
    agent.attach(bc.EpsilonController(
        initial_e=parameters.epsilon_start,
        e_decays=parameters.epsilon_decay,
        e_min=parameters.epsilon_min,
        evaluate_on='action',
        periodicity=1,
        reset_every='none'))

for line in in_map:
            line = line.rstrip()
            line_split = line.split()
            func_map[line_split[0]] += line_split[1].split(",")

    # Get set of all unique functions.
    functions = list(set(in_df['function']))

    chunk_size = int(len(functions) / proc) + 1

    function_chunks = [functions[i:i + chunk_size]
                       for i in range(0, len(functions), chunk_size)]

    # Loop over all functions in parallel and return pandas dataframe for each
    # function with regrouped ids (which will are combined together afterwards).
    raw_new_ids_dfs = Parallel(n_jobs=proc)(delayed(
                                    convert_func_ids)(func_subset,
                                                      in_df,
                                                      func_map)
                                    for func_subset in function_chunks)

    # Combine all returned DFs into a single DF.
    raw_new_ids_combined = pd.concat(raw_new_ids_dfs, sort=False)

    if in_format == "contrib":
        regrouped_table = pd.pivot_table(raw_new_ids_combined,
                                         index=['sample', 'function', 'taxon',
                                                'taxon_abun',
                                                'taxon_rel_abun'],
                                         aggfunc=np.sum)

    elif in_format == "strat":

else:
            raise "Unknown format"
    else:
        format = imgIn.dtype.char
        if format == 'f2':
            args.format = 'f16'
        else:
            format = 'f4'
            args.format = 'f32'

    if imgIn.dtype.char != format:
        print "Converting to ", args.format, "(",format,")"
    imgIn = imgIn.astype(format)
            
    print (args.output_file.rsplit('.', 1) if args.output_file else args.input_file.rsplit('.',1))
    result = Parallel(n_jobs=8)(delayed(convertFace)(args,imgIn,i) for i in xrange(6))
    
    if args.flatten_cubemap:
        components = args.output_file.rsplit('.', 1) if args.output_file else args.input_file.rsplit('.',1)
        filename = components[0] + "_" + str(imgIn.shape[0]/2) + "_full-cubemap_" +args.format+".raw"
        result_arrays = result.sort(key=lambda x: FACE_REMAP_ORDER[x[0]])
        numpy.concatenate(tuple([x[1] for x in result])).tofile(open(filename,"wb"))
    
    print "Time elapsed: ",(time.time() - start)
    print "Saved processed image as: " + filename
    print '== PVRTexTool Wrap Raw Data settings =='
    print '= Width x Height : %d x %d' % (imgIn.shape[0]/2 , imgIn.shape[0]/2)
    print '= Variable type  : "Signed Floating Point"'
    print '= Colour space   : Linear RGB'
    print '= Faces          : 6'
    print '= MIP-Map levels : 1'
    print '= Array Members  : 1'

clf = svm.SVC(
            verbose=True,
            probability=True,
            C=0.0001,
            kernel="rbf",
            gamma=0.001,
            class_weight="balanced",
        )
        clf.fit(train_features, train_labels)

        # print("Best estimator found by grid search:")
        # print(clf.best_estimator_)

        # joblib.dump(clf, saved_classifier_filename)
    else:
        clf = joblib.load(saved_classifier_filename)

    # (test_features, test_labels) = get_test_features_and_labels(load_test_features)

    test_labels_bin = label_binarize(test_labels, classes=[-1, 1])

    pred_labels = clf.predict(test_features)

    pred_confidences = clf.predict_proba(test_features)

    plot_roc_curve(test_labels_bin, pred_confidences)

    from sklearn.metrics import (
        roc_curve,
        accuracy_score,
        confusion_matrix,
        roc_auc_score,