How to use dnn - 10 common examples

"""
        import time, copy
        if x_dev == None or y_dev == None:
            from sklearn.cross_validation import train_test_split
            x_train, x_dev, y_train, y_dev = train_test_split(x_train, y_train,
                    test_size=split_ratio, random_state=42)
        if method == 'sgd':
            train_fn = self.get_SGD_trainer()
        elif method == 'adagrad':
            train_fn = self.get_adagrad_trainer()
        elif method == 'adadelta':
            train_fn = self.get_adadelta_trainer()
        elif method == 'rmsprop':
            train_fn = self.get_rmsprop_trainer(with_step_adapt=True,
                    nesterov=False)
        train_set_iterator = DatasetMiniBatchIterator(x_train, y_train)
        dev_set_iterator = DatasetMiniBatchIterator(x_dev, y_dev)
        train_scoref = self.score_classif(train_set_iterator)
        dev_scoref = self.score_classif(dev_set_iterator)
        best_dev_loss = numpy.inf
        epoch = 0
        # TODO early stopping (not just cross val, also stop training)
        if plot:
            verbose = True
            self._costs = []
            self._train_errors = []
            self._dev_errors = []
            self._updates = []

        init_lr = INIT_LR
        if method == 'rmsprop':
            init_lr = 1.E-6  # TODO REMOVE HACK

import time, copy
        if x_dev == None or y_dev == None:
            from sklearn.cross_validation import train_test_split
            x_train, x_dev, y_train, y_dev = train_test_split(x_train, y_train,
                    test_size=split_ratio, random_state=42)
        if method == 'sgd':
            train_fn = self.get_SGD_trainer()
        elif method == 'adagrad':
            train_fn = self.get_adagrad_trainer()
        elif method == 'adadelta':
            train_fn = self.get_adadelta_trainer()
        elif method == 'rmsprop':
            train_fn = self.get_rmsprop_trainer(with_step_adapt=True,
                    nesterov=False)
        train_set_iterator = DatasetMiniBatchIterator(x_train, y_train)
        dev_set_iterator = DatasetMiniBatchIterator(x_dev, y_dev)
        train_scoref = self.score_classif(train_set_iterator)
        dev_scoref = self.score_classif(dev_set_iterator)
        best_dev_loss = numpy.inf
        epoch = 0
        # TODO early stopping (not just cross val, also stop training)
        if plot:
            verbose = True
            self._costs = []
            self._train_errors = []
            self._dev_errors = []
            self._updates = []

        init_lr = INIT_LR
        if method == 'rmsprop':
            init_lr = 1.E-6  # TODO REMOVE HACK
        n_seen = 0

emb_mat = w2v.get_matrix()
print("- Embedding matrix size:", emb_mat.shape)

sl.start("* Load data")
data_dict = {
    'x': np.load(dir.TRAIN + "index(4519,30).npy"),
    'y': np.load(dir.TRAIN + "y(4519,4).npy"),
    'len': [np.load(dir.TRAIN + "len(4519,).npy")],
    'tx': np.load(dir.TEST + "index(1049,30).npy"),
    'ty': np.load(dir.TEST + "y(1049,4).npy"),
    'tlen': [np.load(dir.TEST + "len(1049,).npy")],
}
sl.stop()
ef.print_shape(data_dict)

args = default_args()
args.emb_type = 'const'
args.emb_dim = w2v.vector_size
args.n_class = 4
args.n_hidden = 20
args.learning_rate = 0.001
args.l2_reg = 0.0
args.batch_size = 64
args.iter_times = 30
args.display_step = 1
args.drop_porb = 0.1

class_name = ['support', 'deny', 'query', 'comment']
nn = CNN_classify(data_dict, emb_mat, args, kernel_widths=[1, 2, 3], class_name=class_name)
nn.train_test()

rt.stop()

emb_mat = w2v.get_matrix()
print("- Embedding matrix size:", emb_mat.shape)

sl.start("* Load data")
data_dict = {
    'x': np.load(dir.TRAIN + "index(4519,30).npy"),
    'y': np.load(dir.TRAIN + "y(4519,4).npy"),
    'len': [np.load(dir.TRAIN + "len(4519,).npy")],
    'tx': np.load(dir.TEST + "index(1049,30).npy"),
    'ty': np.load(dir.TEST + "y(1049,4).npy"),
    'tlen': [np.load(dir.TEST + "len(1049,).npy")],
}
sl.stop()
ef.print_shape(data_dict)

args = default_args()
args.emb_type = 'const'
args.emb_dim = w2v.vector_size
args.n_class = 4
args.n_hidden = 50
args.learning_rate = 0.0001
args.l2_reg = 0.0
args.batch_size = 32
args.iter_times = 30
args.display_step = 1
args.drop_porb = 0.1
args.n_layer = 2
args.n_head = 8

class_name = ['support', 'deny', 'query', 'comment']
nn = transformer_classify(data_dict, emb_mat, args, class_name=class_name)
nn.train_test()

sl.start("* Load data")
data_dict = {
    'x': np.load(dir.TRAIN + "index(4519,3,30).npy"),
    'y': np.load(dir.TRAIN + "y(4519,3,4).npy"),
    'len': [np.load(dir.TRAIN + "len_sen(4519,3).npy"), np.load(dir.TRAIN + "len_seq(4519,).npy")],
    'id': ef.load_list(dir.TRAIN + "id.txt"),
    'tx': np.load(dir.TEST + "index(1049,3,30).npy"),
    'ty': np.load(dir.TEST + "y(1049,3,4).npy"),
    'tlen': [np.load(dir.TEST + "len_sen(1049,3).npy"), np.load(dir.TEST + "len_seq(1049,).npy")],
    'tid': ef.load_list(dir.TEST + "id.txt"),
}
sl.stop()
ef.print_shape(data_dict)

args = default_args()
args.emb_type = 'const'
args.emb_dim = w2v.vector_size
args.n_class = 4
args.n_hidden = 50
args.learning_rate = 0.001
args.l2_reg = 0.0
args.batch_size = 64
args.iter_times = 20
args.display_step = 1
args.drop_porb = 0.1
args.rnn_type = 'LSTM'
args.use_attention = True

class_name = ['support', 'deny', 'query', 'comment']
nn = RNN_sequence(data_dict, emb_mat, args, vote=True, class_name=class_name)
nn.train_test()

emb_mat = w2v.get_matrix()
print("- Embedding matrix size:", emb_mat.shape)

sl.start("* Load data")
data_dict = {
    'x': np.load(dir.TRAIN + "index(4519,3,30).npy"),
    'y': np.load(dir.TRAIN + "y(4519,4).npy"),
    'len': [np.load(dir.TRAIN + "len_sen(4519,3).npy"), np.load(dir.TRAIN + "len_seq(4519,).npy")],
    'tx': np.load(dir.TEST + "index(1049,3,30).npy"),
    'ty': np.load(dir.TEST + "y(1049,4).npy"),
    'tlen': [np.load(dir.TEST + "len_sen(1049,3).npy"), np.load(dir.TEST + "len_seq(1049,).npy")],
}
sl.stop()
ef.print_shape(data_dict)

args = default_args()
args.emb_type = 'const'
args.emb_dim = w2v.vector_size
args.n_class = 4
args.n_hidden = 50
args.learning_rate = 0.001
args.l2_reg = 0.0
args.batch_size = 64
args.iter_times = 20
args.display_step = 1
args.drop_porb = 0.1
args.rnn_type = 'GRU'
args.use_attention = True

class_name = ['support', 'deny', 'query', 'comment']
nn = RNN_classify(data_dict, emb_mat, args, class_name=class_name)
nn.train_test()

# allocate symbolic variables for the data
        self.x = T.matrix('x')
        self.y = T.ivector('y')
    
        # we assume that i-vectors are appended to speech features in a frame-wise manner  
        self.feat_dim = cfg_si.n_ins
        self.ivec_dim = cfg_adapt.n_ins
        self.iv = self.x[:,self.feat_dim:self.feat_dim+self.ivec_dim]
        self.feat = self.x[:,0:self.feat_dim]
        
        # the parameters 
        self.params = []            # the params to be updated in the current training
        self.delta_params = []
        
        # the i-vector network
        dnn_adapt = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_adapt, input  = self.iv)
        self.dnn_adapt = dnn_adapt

        # the final output layer which has the same dimension as the input features
        linear_func = lambda x: x
        adapt_output_layer = HiddenLayer(rng=numpy_rng,
                                 input=dnn_adapt.layers[-1].output,
                                 n_in=cfg_adapt.hidden_layers_sizes[-1],
                                 n_out=self.feat_dim,
                                 activation=linear_func)
        dnn_adapt.layers.append(adapt_output_layer)
        dnn_adapt.params.extend(adapt_output_layer.params)
        dnn_adapt.delta_params.extend(adapt_output_layer.delta_params)

        dnn_si = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = cfg_si, input = self.feat + dnn_adapt.layers[-1].output)
        self.dnn_si = dnn_si

if not theano_rng:
            theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
        # allocate symbolic variables for the data
        self.x = T.matrix('x')
        self.y = T.ivector('y') 
       
        self.input_tower1 = self.x[:,0:cfg_tower1.n_ins]
        self.input_tower2 = self.x[:,cfg_tower1.n_ins:(cfg_tower1.n_ins+cfg_tower2.n_ins)]
 
        self.dnn_tower1 = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = self.cfg_tower1,
                              input  = self.input_tower1)
        self.dnn_tower2 = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = self.cfg_tower2,
                              input  = self.input_tower2)
        concat_output = T.concatenate([self.dnn_tower1.layers[-1].output, self.dnn_tower2.layers[-1].output], axis=1)
        self.dnn = DNN(numpy_rng=numpy_rng, theano_rng = theano_rng, cfg = self.cfg, input  = concat_output)

        self.layers.extend(self.dnn_tower1.layers); self.params.extend(self.dnn_tower1.params); 
        self.delta_params.extend(self.dnn_tower1.delta_params)
        self.layers.extend(self.dnn_tower2.layers); self.params.extend(self.dnn_tower2.params);
        self.delta_params.extend(self.dnn_tower2.delta_params)
        self.layers.extend(self.dnn.layers); self.params.extend(self.dnn.params);
        self.delta_params.extend(self.dnn.delta_params)

        self.finetune_cost = self.dnn.logLayer.negative_log_likelihood(self.y)
        self.errors = self.dnn.logLayer.errors(self.y)

:param seed:
        :type seed:
        :param weight_range:
        :type weight_range:
        :return:
        :rtype:
        """
        self.mlsl_output_size = mlsl.output_sizes[-1] if outputs_from_mlsl else outputs_from_mlsl

        # Change input size of Neural net to assigned feature size plus MLSL outputs
        nnl[0]+=self.mlsl_output_size

        self.outputs_from_mlsl = outputs_from_mlsl

        self.mlsl = mlsl
        self.nnet = DNN()
        self.nnet.initialize(nnl=nnl,seed=seed, weight_range=weight_range)
        self.use_softmax = use_softmax

Returns:
        an EDEncoder class'''

    if encoder == 'listener':
        import listener
        return listener.Listener
    elif encoder == 'dummy_encoder':
        import dummy_encoder
        return dummy_encoder.DummyEncoder
    elif encoder == 'dblstm':
        import dblstm
        return dblstm.DBLSTM
    elif encoder == 'dnn':
        import dnn
        return dnn.DNN
    elif encoder == 'hotstart_encoder':
        import hotstart_encoder
        return hotstart_encoder.HotstartEncoder
    else:
        raise Exception('undefined encoder type: %s' % encoder)