How to use the sacred.observers.FileStorageObserver function in sacred

def _experiment(inner_cv, outer_cv):
    e = experiment(_dataset, _estimator)
    e.observers.append(FileStorageObserver('.results'))
    e.run(config_updates={'dataset': {'inner_cv': inner_cv,
                                      'outer_cv': outer_cv}})

from sacred.observers import FileStorageObserver
from sklearn.externals.joblib import load, dump
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelBinarizer, StandardScaler
from sklearn.utils import gen_batches, check_random_state

from cogspaces.model import make_model, init_tensorflow, make_adversaries
from cogspaces.model_selection import StratifiedGroupShuffleSplit
from cogspaces.utils import get_output_dir

idx = pd.IndexSlice

predict_contrast_exp = Experiment('predict_contrast')

base_artifact_dir = join(get_output_dir(), 'predict')
observer = FileStorageObserver.create(basedir=base_artifact_dir)

predict_contrast_exp.observers.append(observer)


def scale(X, train, per_dataset_std):
    X_train = X.iloc[train]
    if per_dataset_std:
        standard_scaler = {}
        corr = np.sum(np.sqrt(
            X_train[0].groupby(level='dataset').aggregate('count').values))
        for dataset, this_X_train in X_train.groupby(level='dataset'):
            this_standard_scaler = StandardScaler()
            this_standard_scaler.fit(this_X_train)
            this_standard_scaler.scale_ /= sqrt(this_X_train.shape[0]) / corr
            standard_scaler[dataset] = this_standard_scaler
        new_X = []

from sacred.observers import FileStorageObserver
from sklearn.externals.joblib import load, dump
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelBinarizer, StandardScaler
from sklearn.utils import gen_batches, check_random_state

from cogspaces.utils import get_output_dir
from cogspaces.model import make_model, init_tensorflow, make_adversaries
from cogspaces.model_selection import StratifiedGroupShuffleSplit

idx = pd.IndexSlice

predict_contrast_exp = Experiment('predict_contrast')

base_artifact_dir = join(get_output_dir(), 'predict')
observer = FileStorageObserver.create(basedir=base_artifact_dir)

predict_contrast_exp.observers.append(observer)


def scale(X, train, per_dataset_std):
    X_train = X.iloc[train]
    if per_dataset_std:
        standard_scaler = {}
        corr = np.sum(np.sqrt(
            X_train[0].groupby(level='dataset').aggregate('count').values))
        for dataset, this_X_train in X_train.groupby(level='dataset'):
            this_standard_scaler = StandardScaler()
            this_standard_scaler.fit(this_X_train)
            this_standard_scaler.scale_ /= sqrt(this_X_train.shape[0]) / corr
            standard_scaler[dataset] = this_standard_scaler
        new_X = []

from joblib import dump
from sacred import Experiment
from sacred.observers import FileStorageObserver
from sklearn.linear_model import LogisticRegressionCV, LogisticRegression
from sklearn.model_selection import GridSearchCV, StratifiedShuffleSplit

from cogspaces.model.non_convex_pytorch import TransferEstimator
from cogspaces.models.trace import TransferTraceNormEstimator
from cogspaces.pipeline import get_output_dir, make_data_frame, split_folds, \
    MultiDatasetTransformer

idx = pd.IndexSlice

exp = Experiment('single_exp')
basedir = join(get_output_dir(), 'single_exp')
exp.observers.append(FileStorageObserver.create(basedir=basedir))


@exp.config
def config():
    datasets = ['brainomics', 'hcp']
    reduced_dir = join(get_output_dir(), 'reduced')
    unmask_dir = join(get_output_dir(), 'unmasked')
    # source = 'mix'
    source = 'hcp_new'
    test_size = {'hcp': .1, 'archi': .5, 'brainomics': .5, 'camcan': .5,
                 'la5c': .5, 'full': .5}
    train_size = dict(hcp=None, archi=None, la5c=None, brainomics=None,
                      camcan=None,
                      human_voice=None)
    dataset_weights = {'brainomics': 1, 'archi': 1, 'hcp': 1}
    model = 'trace'

def run_single_experiment(dataset: str,
                          savedir: str,
                          named_configs: List,
                          config_updates: Dict[str, Any]):
    from tape.__main__ import proteins

    config_updates.update({
        'training': {'learning_rate': 1e-4, 'use_memory_saving_gradients': True},
        'num_epochs': 1000,
        'steps_per_epoch': 200,
        'tasks': dataset})

    if not os.path.exists(savedir):
        os.mkdir(savedir)
    shutil.rmtree(proteins.observers[0].basedir)
    proteins.observers[0] = FileStorageObserver.create(
        os.path.join(savedir, dataset))

    proteins.run(
        named_configs=named_configs,
        config_updates=config_updates)

def single_run(config_updates, rundir, _id):
    run = single_exp._create_run(config_updates=config_updates)
    observer = FileStorageObserver.create(basedir=rundir)
    run._id = _id
    run.observers = [observer]
    try:
        run()
    except:
        pass

from os.path import join

import pandas as pd
from sacred import Experiment
from sacred.observers import FileStorageObserver

from cogspaces.trace_norm import TraceNormEstimator
from cogspaces.utils import get_output_dir, make_data_frame, split_folds, \
    MultiDatasetTransformer

idx = pd.IndexSlice

exp = Experiment('Clean')
basedir = join(get_output_dir(), 'clean')
exp.observers.append(FileStorageObserver.create(basedir=basedir))


@exp.config
def config():
    datasets = ['brainomics']
    reduced_dir = join(get_output_dir(), 'reduced')
    unmask_dir = join(get_output_dir(), 'unmasked')
    source = 'hcp_rs'
    n_subjects = None
    test_size = {'hcp': .1, 'archi': .5, 'brainomics': .5, 'camcan': .5,
                 'la5c': .5}
    train_size = {'hcp': .9, 'archi': .5, 'brainomics': .5, 'camcan': .5,
                  'la5c': .5}
    alpha = 0
    beta = 0
    n_iter = 3000

def single_run(config_updates, rundir, _id):
    for i in range(3):
        try:
            run = single_exp._create_run(config_updates=config_updates)
            observer = FileStorageObserver.create(basedir=rundir)
            run._id = _id
            run.observers = [observer]
            run()
            break
        except TypeError:
            if i < 2:
                print("Run %i failed at start, retrying..." % _id)
            else:
                print("Giving up %i" % _id)
            continue

def main():
    observer = FileStorageObserver(osp.join("data", "sacred", "train"))
    train_ex.observers.append(observer)
    train_ex.run_commandline()

matrices (using the largest logits), and re-optimize using L-BFGS to find
    the nearest local optima.
    """
    # Hack: create new instance without call __init__, since trainable.__init__
    # creates result_dir and log_dir in the wrong place (~/ray_results)
    trainable_cls = TrainableBP
    trainable = trainable_cls.__new__(trainable_cls)
    trainable._setup(trial.config)
    trainable.restore(str(Path(trial.logdir) / trial._checkpoint.value))
    loss = trainable.polish(N_LBFGS_STEPS, save_to_self_model=True)
    torch.save(trainable.model.state_dict(), str((Path(trial.logdir) / trial._checkpoint.value).parent / 'polished_model.pth'))
    return loss


ex = Experiment('Transform_factorization')
ex.observers.append(FileStorageObserver.create('logs_new'))
slack_config_path = Path('config/slack.json')  # Add webhook_url there for Slack notification
if slack_config_path.exists():
    ex.observers.append(SlackObserver.from_config(str(slack_config_path)))


@ex.config
def default_config():
    model = 'BP'
    target = 'dft'  # The target matrix to factor ('dft', 'idft', 'dct', 'hadamard')
    size = 8  # Size of matrix to factor, must be power of 2
    complex = True  # Whether to use complex factorization or real factorization
    fixed_order = True  # Whether the order of the factors are fixed
    ntrials = 20  # Number of trials for hyperparameter tuning
    nsteps = 400  # Number of steps per epoch
    nepochsvalid = 5  # Frequency of validation (polishing), in terms of epochs
    nmaxepochs = 200  # Maximum number of epochs