Examples with MultiLayerNetwork org.deeplearning4j.nn.multilayer.MultiLayerNetwork used on opensource projects

Example 71 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class GradientCheckTests method testRbm.

@Test
public void testRbm() {
    //As above (testGradientMLP2LayerIrisSimple()) but with L2, L1, and both L2/L1 applied
    //Need to run gradient through updater, so that L2 can be applied
    RBM.HiddenUnit[] hiddenFunc = { RBM.HiddenUnit.BINARY, RBM.HiddenUnit.RECTIFIED };
    //If true: run some backprop steps first
    boolean[] characteristic = { false, true };
    LossFunction[] lossFunctions = { LossFunction.MSE, LossFunction.KL_DIVERGENCE };
    //i.e., lossFunctions[i] used with outputActivations[i] here
    String[] outputActivations = { "softmax", "sigmoid" };
    DataNormalization scaler = new NormalizerMinMaxScaler();
    DataSetIterator iter = new IrisDataSetIterator(150, 150);
    scaler.fit(iter);
    iter.setPreProcessor(scaler);
    DataSet ds = iter.next();
    INDArray input = ds.getFeatureMatrix();
    INDArray labels = ds.getLabels();
    double[] l2vals = { 0.4, 0.0, 0.4 };
    //i.e., use l2vals[i] with l1vals[i]
    double[] l1vals = { 0.0, 0.5, 0.5 };
    for (RBM.HiddenUnit hidunit : hiddenFunc) {
        for (boolean doLearningFirst : characteristic) {
            for (int i = 0; i < lossFunctions.length; i++) {
                for (int k = 0; k < l2vals.length; k++) {
                    LossFunction lf = lossFunctions[i];
                    String outputActivation = outputActivations[i];
                    double l2 = l2vals[k];
                    double l1 = l1vals[k];
                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(l2).l1(l1).learningRate(1.0).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L).list().layer(0, new RBM.Builder(hidunit, RBM.VisibleUnit.BINARY).nIn(4).nOut(3).weightInit(WeightInit.UNIFORM).updater(Updater.SGD).build()).layer(1, new OutputLayer.Builder(lf).nIn(3).nOut(3).weightInit(WeightInit.XAVIER).updater(Updater.SGD).activation(outputActivation).build()).pretrain(false).backprop(true).build();
                    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                    mln.init();
                    if (doLearningFirst) {
                        //Run a number of iterations of learning
                        mln.setInput(ds.getFeatures());
                        mln.setLabels(ds.getLabels());
                        mln.computeGradientAndScore();
                        double scoreBefore = mln.score();
                        for (int j = 0; j < 10; j++) mln.fit(ds);
                        mln.computeGradientAndScore();
                        double scoreAfter = mln.score();
                        //Can't test in 'characteristic mode of operation' if not learning
                        String msg = "testGradMLP2LayerIrisSimple() - score did not (sufficiently) decrease during learning - activationFn=" + hidunit.toString() + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l2=" + l2 + ", l1=" + l1 + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")";
                        assertTrue(msg, scoreAfter < scoreBefore);
                    }
                    if (PRINT_RESULTS) {
                        System.out.println("testGradientMLP2LayerIrisSimpleRandom() - activationFn=" + hidunit.toString() + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l2=" + l2 + ", l1=" + l1);
                        for (int j = 0; j < mln.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                    }
                    boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
                    String msg = "testGradMLP2LayerIrisSimple() - activationFn=" + hidunit.toString() + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l2=" + l2 + ", l1=" + l1;
                    assertTrue(msg, gradOK);
                }
            }
        }
    }
}

Example 72 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class NeuralNetConfigurationTest method testL1L2ByParam.

@Test
public void testL1L2ByParam() {
    double l1 = 0.01;
    double l2 = 0.07;
    int[] nIns = { 4, 3, 3 };
    int[] nOuts = { 3, 3, 3 };
    int oldScore = 1;
    int newScore = 1;
    int iteration = 3;
    INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(8).regularization(true).l1(l1).l2(l2).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).l2(0.5).build()).layer(2, new OutputLayer.Builder().nIn(nIns[2]).nOut(nOuts[2]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).backprop(true).pretrain(false).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(), new NegativeDefaultStepFunction(), null, net);
    opt.checkTerminalConditions(gradientW, oldScore, newScore, iteration);
    assertEquals(l1, net.getLayer(0).conf().getL1ByParam("W"), 1e-4);
    assertEquals(0.0, net.getLayer(0).conf().getL1ByParam("b"), 0.0);
    assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("beta"), 0.0);
    assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("gamma"), 0.0);
    assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("mean"), 0.0);
    assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("var"), 0.0);
    assertEquals(l2, net.getLayer(2).conf().getL2ByParam("W"), 1e-4);
    assertEquals(0.0, net.getLayer(2).conf().getL2ByParam("b"), 0.0);
}

Example 73 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class NeuralNetConfigurationTest method testLearningRateByParam.

@Test
public void testLearningRateByParam() {
    double lr = 0.01;
    double biasLr = 0.02;
    int[] nIns = { 4, 3, 3 };
    int[] nOuts = { 3, 3, 3 };
    int oldScore = 1;
    int newScore = 1;
    int iteration = 3;
    INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(0.3).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.SGD).learningRate(lr).biasLearningRate(biasLr).build()).layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).learningRate(0.7).build()).layer(2, new OutputLayer.Builder().nIn(nIns[2]).nOut(nOuts[2]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).backprop(true).pretrain(false).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(), new NegativeDefaultStepFunction(), null, net);
    opt.checkTerminalConditions(gradientW, oldScore, newScore, iteration);
    assertEquals(lr, net.getLayer(0).conf().getLearningRateByParam("W"), 1e-4);
    assertEquals(biasLr, net.getLayer(0).conf().getLearningRateByParam("b"), 1e-4);
    assertEquals(0.7, net.getLayer(1).conf().getLearningRateByParam("gamma"), 1e-4);
    //From global LR
    assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("W"), 1e-4);
    //From global LR
    assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("b"), 1e-4);
}

Example 74 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class LayerConfigTest method testLearningRatePolicyExponential.

@Test
public void testLearningRatePolicyExponential() {
    double lr = 2;
    double lrDecayRate = 5;
    int iterations = 1;
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr).learningRateDecayPolicy(LearningRatePolicy.Exponential).lrPolicyDecayRate(lrDecayRate).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    assertEquals(LearningRatePolicy.Exponential, conf.getConf(0).getLearningRatePolicy());
    assertEquals(LearningRatePolicy.Exponential, conf.getConf(1).getLearningRatePolicy());
    assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0);
    assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0);
}

Example 75 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class LayerConfigValidationTest method testNesterovsNotSetLocalMomentum.

@Test
public void testNesterovsNotSetLocalMomentum() {
    // Warnings only thrown
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).momentum(0.3).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
}