Examples with NormalDistribution org.deeplearning4j.nn.conf.distribution.NormalDistribution used on opensource projects

Example 86 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class CuDNNGradientChecks method testLRN.

@Test
public void testLRN() throws Exception {
    Nd4j.getRandom().setSeed(12345);
    int minibatch = 10;
    int depth = 6;
    int hw = 5;
    int nOut = 4;
    INDArray input = Nd4j.rand(new int[] { minibatch, depth, hw, hw });
    INDArray labels = Nd4j.zeros(minibatch, nOut);
    Random r = new Random(12345);
    for (int i = 0; i < minibatch; i++) {
        labels.putScalar(i, r.nextInt(nOut), 1.0);
    }
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().learningRate(1.0).regularization(false).updater(Updater.NONE).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 2)).list().layer(0, new ConvolutionLayer.Builder().nOut(6).kernelSize(2, 2).stride(1, 1).activation(Activation.TANH).build()).layer(1, new LocalResponseNormalization.Builder().build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(hw, hw, depth)).pretrain(false).backprop(true);
    MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
    mln.init();
    Field f = org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization.class.getDeclaredField("helper");
    f.setAccessible(true);
    org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization l = (org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization) mln.getLayer(1);
    LocalResponseNormalizationHelper lrn = (LocalResponseNormalizationHelper) f.get(l);
    assertTrue(lrn instanceof CudnnLocalResponseNormalizationHelper);
    if (PRINT_RESULTS) {
        for (int j = 0; j < mln.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
    }
    boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
    assertTrue(gradOK);
}

Example 87 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TransferLearningMLNTest method testNoutChanges.

@Test
public void testNoutChanges() {
    DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 2));
    NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().learningRate(0.1).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.SGD);
    FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().learningRate(0.1).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.SGD).build();
    MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(5).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build());
    modelToFineTune.init();
    MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf).nOutReplace(3, 2, WeightInit.XAVIER, WeightInit.XAVIER).nOutReplace(0, 3, WeightInit.XAVIER, new NormalDistribution(1, 1e-1)).build();
    MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(2).build()).build());
    modelExpectedArch.init();
    //Will fail - expected because of dist and weight init changes
    //assertEquals(modelExpectedArch.getLayerWiseConfigurations().toJson(), modelNow.getLayerWiseConfigurations().toJson());
    assertEquals(modelNow.getLayerWiseConfigurations().getConf(0).getLayer().getWeightInit(), WeightInit.XAVIER);
    assertEquals(modelNow.getLayerWiseConfigurations().getConf(0).getLayer().getDist(), null);
    assertEquals(modelNow.getLayerWiseConfigurations().getConf(1).getLayer().getWeightInit(), WeightInit.DISTRIBUTION);
    assertEquals(modelNow.getLayerWiseConfigurations().getConf(1).getLayer().getDist(), new NormalDistribution(1, 1e-1));
    assertEquals(modelNow.getLayerWiseConfigurations().getConf(3).getLayer().getWeightInit(), WeightInit.XAVIER);
    //modelNow should have the same architecture as modelExpectedArch
    assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape());
    modelNow.setParams(modelExpectedArch.params());
    //fit should give the same results
    modelExpectedArch.fit(randomData);
    modelNow.fit(randomData);
    assertEquals(modelExpectedArch.score(), modelNow.score(), 0.000001);
    assertEquals(modelExpectedArch.params(), modelNow.params());
}

Example 88 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TestDecayPolicies method testOriginalLearningRateUnchanged.

@Test
public void testOriginalLearningRateUnchanged() {
    // Confirm learning rate is unchanged while hash is updated
    DataSet ds = new IrisDataSetIterator(150, 150).next();
    ds.normalizeZeroMeanZeroUnitVariance();
    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).learningRate(1.0).learningRateDecayPolicy(LearningRatePolicy.Score).lrPolicyDecayRate(0.10).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(org.deeplearning4j.nn.conf.Updater.SGD).seed(12345L).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.SIGMOID).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(3).nOut(3).build()).pretrain(false).backprop(true).build();
    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
    mln.init();
    //Run a number of iterations of learning
    mln.setInput(ds.getFeatureMatrix());
    mln.setLabels(ds.getLabels());
    mln.computeGradientAndScore();
    for (int j = 0; j < 1; j++) mln.fit(ds);
    mln.computeGradientAndScore();
    double lr0 = mln.getLayer(0).conf().getLayer().getLearningRate();
    double lr1 = mln.getLayer(1).conf().getLayer().getLearningRate();
    assertEquals(1.0, lr0, 0.0);
    assertEquals(1.0, lr1, 0.0);
}

Example 89 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class RegressionTest071 method regressionTestMLP2.

@Test
public void regressionTestMLP2() throws Exception {
    File f = new ClassPathResource("regression_testing/071/071_ModelSerializer_Regression_MLP_2.zip").getTempFileFromArchive();
    MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(f, true);
    MultiLayerConfiguration conf = net.getLayerWiseConfigurations();
    assertEquals(2, conf.getConfs().size());
    assertTrue(conf.isBackprop());
    assertFalse(conf.isPretrain());
    DenseLayer l0 = (DenseLayer) conf.getConf(0).getLayer();
    assertTrue(l0.getActivationFn() instanceof ActivationLReLU);
    assertEquals(3, l0.getNIn());
    assertEquals(4, l0.getNOut());
    assertEquals(WeightInit.DISTRIBUTION, l0.getWeightInit());
    assertEquals(new NormalDistribution(0.1, 1.2), l0.getDist());
    assertEquals(Updater.RMSPROP, l0.getUpdater());
    assertEquals(0.96, l0.getRmsDecay(), 1e-6);
    assertEquals(0.15, l0.getLearningRate(), 1e-6);
    assertEquals(0.6, l0.getDropOut(), 1e-6);
    assertEquals(0.1, l0.getL1(), 1e-6);
    assertEquals(0.2, l0.getL2(), 1e-6);
    assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, l0.getGradientNormalization());
    assertEquals(1.5, l0.getGradientNormalizationThreshold(), 1e-5);
    OutputLayer l1 = (OutputLayer) conf.getConf(1).getLayer();
    assertTrue(l1.getActivationFn() instanceof ActivationIdentity);
    assertEquals(LossFunctions.LossFunction.MSE, l1.getLossFunction());
    assertTrue(l1.getLossFn() instanceof LossMSE);
    assertEquals(4, l1.getNIn());
    assertEquals(5, l1.getNOut());
    assertEquals(WeightInit.DISTRIBUTION, l0.getWeightInit());
    assertEquals(new NormalDistribution(0.1, 1.2), l0.getDist());
    assertEquals(Updater.RMSPROP, l0.getUpdater());
    assertEquals(0.96, l1.getRmsDecay(), 1e-6);
    assertEquals(0.15, l1.getLearningRate(), 1e-6);
    assertEquals(0.6, l1.getDropOut(), 1e-6);
    assertEquals(0.1, l1.getL1(), 1e-6);
    assertEquals(0.2, l1.getL2(), 1e-6);
    assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, l1.getGradientNormalization());
    assertEquals(1.5, l1.getGradientNormalizationThreshold(), 1e-5);
    int numParams = net.numParams();
    assertEquals(Nd4j.linspace(1, numParams, numParams), net.params());
    int updaterSize = net.getUpdater().stateSizeForLayer(net);
    assertEquals(Nd4j.linspace(1, updaterSize, updaterSize), net.getUpdater().getStateViewArray());
}

Example 90 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class RegressionTest050 method regressionTestMLP2.

@Test
public void regressionTestMLP2() throws Exception {
    File f = new ClassPathResource("regression_testing/050/050_ModelSerializer_Regression_MLP_2.zip").getTempFileFromArchive();
    MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(f, true);
    MultiLayerConfiguration conf = net.getLayerWiseConfigurations();
    assertEquals(2, conf.getConfs().size());
    assertTrue(conf.isBackprop());
    assertFalse(conf.isPretrain());
    DenseLayer l0 = (DenseLayer) conf.getConf(0).getLayer();
    assertTrue(l0.getActivationFn() instanceof ActivationLReLU);
    assertEquals(3, l0.getNIn());
    assertEquals(4, l0.getNOut());
    assertEquals(WeightInit.DISTRIBUTION, l0.getWeightInit());
    assertEquals(new NormalDistribution(0.1, 1.2), l0.getDist());
    assertEquals(Updater.RMSPROP, l0.getUpdater());
    assertEquals(0.96, l0.getRmsDecay(), 1e-6);
    assertEquals(0.15, l0.getLearningRate(), 1e-6);
    assertEquals(0.6, l0.getDropOut(), 1e-6);
    assertEquals(0.1, l0.getL1(), 1e-6);
    assertEquals(0.2, l0.getL2(), 1e-6);
    OutputLayer l1 = (OutputLayer) conf.getConf(1).getLayer();
    assertEquals("identity", l1.getActivationFn().toString());
    assertEquals(LossFunctions.LossFunction.MSE, l1.getLossFunction());
    assertTrue(l1.getLossFn() instanceof LossMSE);
    assertEquals(4, l1.getNIn());
    assertEquals(5, l1.getNOut());
    assertEquals(WeightInit.DISTRIBUTION, l0.getWeightInit());
    assertEquals(new NormalDistribution(0.1, 1.2), l0.getDist());
    assertEquals(Updater.RMSPROP, l0.getUpdater());
    assertEquals(0.96, l1.getRmsDecay(), 1e-6);
    assertEquals(0.15, l1.getLearningRate(), 1e-6);
    assertEquals(0.6, l1.getDropOut(), 1e-6);
    assertEquals(0.1, l1.getL1(), 1e-6);
    assertEquals(0.2, l1.getL2(), 1e-6);
    int numParams = net.numParams();
    assertEquals(Nd4j.linspace(1, numParams, numParams), net.params());
    int updaterSize = net.getUpdater().stateSizeForLayer(net);
    assertEquals(Nd4j.linspace(1, updaterSize, updaterSize), net.getUpdater().getStateViewArray());
}