Examples with NormalDistribution org.deeplearning4j.nn.conf.distribution.NormalDistribution used on opensource projects

Example 81 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TestSetGetParameters method testSetParametersRNN.

@Test
public void testSetParametersRNN() {
    //Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this.
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new GravesLSTM.Builder().nIn(9).nOut(10).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).build()).layer(1, new GravesLSTM.Builder().nIn(10).nOut(11).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).build()).layer(2, new RnnOutputLayer.Builder(LossFunction.MSE).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).nIn(11).nOut(12).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    INDArray initParams = net.params().dup();
    Map<String, INDArray> initParams2 = net.paramTable();
    net.setParams(net.params());
    INDArray initParamsAfter = net.params();
    Map<String, INDArray> initParams2After = net.paramTable();
    for (String s : initParams2.keySet()) {
        assertTrue("Params differ: " + s, initParams2.get(s).equals(initParams2After.get(s)));
    }
    assertEquals(initParams, initParamsAfter);
    //Now, try the other way: get(set(random))
    INDArray randomParams = Nd4j.rand(initParams.shape());
    net.setParams(randomParams.dup());
    assertEquals(net.params(), randomParams);
}

Example 82 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TestVAE method testReconstructionErrorSimple.

@Test
public void testReconstructionErrorSimple() {
    int inOutSize = 6;
    ReconstructionDistribution[] reconstructionDistributions = new ReconstructionDistribution[] { new LossFunctionWrapper(Activation.TANH, new LossMSE()), new LossFunctionWrapper(Activation.IDENTITY, new LossMAE()), new CompositeReconstructionDistribution.Builder().addDistribution(3, new LossFunctionWrapper(Activation.TANH, new LossMSE())).addDistribution(3, new LossFunctionWrapper(Activation.IDENTITY, new LossMAE())).build() };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        for (int i = 0; i < reconstructionDistributions.length; i++) {
            INDArray data = Nd4j.rand(minibatch, inOutSize).muli(2).subi(1);
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(0.2).l1(0.3).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).list().layer(0, new VariationalAutoencoder.Builder().nIn(inOutSize).nOut(3).encoderLayerSizes(5).decoderLayerSizes(6).pzxActivationFunction(Activation.TANH).reconstructionDistribution(reconstructionDistributions[i]).activation(new ActivationTanH()).updater(Updater.SGD).build()).pretrain(true).backprop(false).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();
            mln.initGradientsView();
            mln.fit(data);
            org.deeplearning4j.nn.layers.variational.VariationalAutoencoder layer = (org.deeplearning4j.nn.layers.variational.VariationalAutoencoder) mln.getLayer(0);
            assertTrue(layer.hasLossFunction());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionError = layer.reconstructionError(data);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionError.shape());
            for (int j = 0; j < minibatch; j++) {
                double re = reconstructionError.getDouble(j);
                assertTrue(re >= 0.0);
            }
        }
    }
}

Example 83 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GlobalPoolingMaskingTests method testMaskingRnn.

@Test
public void testMaskingRnn() {
    int timeSeriesLength = 5;
    int nIn = 5;
    int layerSize = 4;
    int nOut = 2;
    int[] minibatchSizes = new int[] { 1, 3 };
    for (int miniBatchSize : minibatchSizes) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).updater(Updater.NONE).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1.0)).seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).build()).pretrain(false).backprop(true).build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();
        Random r = new Random(12345L);
        INDArray input = Nd4j.rand(new int[] { miniBatchSize, nIn, timeSeriesLength }).subi(0.5);
        INDArray mask;
        if (miniBatchSize == 1) {
            mask = Nd4j.create(new double[] { 1, 1, 1, 1, 0 });
        } else {
            mask = Nd4j.create(new double[][] { { 1, 1, 1, 1, 1 }, { 1, 1, 1, 1, 0 }, { 1, 1, 1, 0, 0 } });
        }
        INDArray labels = Nd4j.zeros(miniBatchSize, nOut);
        for (int i = 0; i < miniBatchSize; i++) {
            int idx = r.nextInt(nOut);
            labels.putScalar(i, idx, 1.0);
        }
        net.setLayerMaskArrays(mask, null);
        INDArray outputMasked = net.output(input);
        net.clearLayerMaskArrays();
        for (int i = 0; i < miniBatchSize; i++) {
            INDArray maskRow = mask.getRow(i);
            int tsLength = maskRow.sumNumber().intValue();
            INDArray inputSubset = input.get(NDArrayIndex.interval(i, i, true), NDArrayIndex.all(), NDArrayIndex.interval(0, tsLength));
            INDArray outSubset = net.output(inputSubset);
            INDArray outputMaskedSubset = outputMasked.getRow(i);
            assertEquals(outSubset, outputMaskedSubset);
        }
    }
}

Example 84 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TestVariableLengthTS method testOutputMaskingScoreMagnitudes.

@Test
public void testOutputMaskingScoreMagnitudes() {
    //Idea: check magnitude of scores, with differeing number of values masked out
    //i.e., MSE with zero weight init and 1.0 labels: know what to expect in terms of score
    int nIn = 3;
    int[] timeSeriesLengths = { 3, 10 };
    int[] outputSizes = { 1, 2, 5 };
    int[] miniBatchSizes = { 1, 4 };
    Random r = new Random(12345);
    for (int tsLength : timeSeriesLengths) {
        for (int nOut : outputSizes) {
            for (int miniBatch : miniBatchSizes) {
                for (int nToMask = 0; nToMask < tsLength - 1; nToMask++) {
                    String msg = "tsLen=" + tsLength + ", nOut=" + nOut + ", miniBatch=" + miniBatch;
                    INDArray labelMaskArray = Nd4j.ones(miniBatch, tsLength);
                    for (int i = 0; i < miniBatch; i++) {
                        //For each example: select which outputs to mask...
                        int nMasked = 0;
                        while (nMasked < nToMask) {
                            int tryIdx = r.nextInt(tsLength);
                            if (labelMaskArray.getDouble(i, tryIdx) == 0.0)
                                continue;
                            labelMaskArray.putScalar(new int[] { i, tryIdx }, 0.0);
                            nMasked++;
                        }
                    }
                    INDArray input = Nd4j.rand(new int[] { miniBatch, nIn, tsLength });
                    INDArray labels = Nd4j.ones(miniBatch, nOut, tsLength);
                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(5).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(5).nOut(nOut).weightInit(WeightInit.ZERO).updater(Updater.NONE).build()).pretrain(false).backprop(true).build();
                    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                    mln.init();
                    //MSE loss function: 1/n * sum(squaredErrors)... but sum(squaredErrors) = n * (1-0) here -> sum(squaredErrors)
                    //Sum over minibatches, then divide by minibatch size
                    double expScore = (tsLength - nToMask);
                    mln.setLayerMaskArrays(null, labelMaskArray);
                    mln.setInput(input);
                    mln.setLabels(labels);
                    mln.computeGradientAndScore();
                    double score = mln.score();
                    assertEquals(msg, expScore, score, 0.1);
                }
            }
        }
    }
}

Example 85 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class MultiLayerTest method testFeedForwardToLayer.

@Test
public void testFeedForwardToLayer() {
    int nIn = 30;
    int nOut = 25;
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).iterations(5).learningRate(1e-3).list().layer(0, new RBM.Builder(RBM.HiddenUnit.RECTIFIED, RBM.VisibleUnit.GAUSSIAN).nIn(nIn).nOut(600).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1e-5)).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(1, new RBM.Builder(RBM.HiddenUnit.RECTIFIED, RBM.VisibleUnit.GAUSSIAN).nIn(600).nOut(250).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1e-5)).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(2, new RBM.Builder(RBM.HiddenUnit.RECTIFIED, RBM.VisibleUnit.GAUSSIAN).nIn(250).nOut(100).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1e-5)).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(100).nOut(25).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1e-5)).build()).build();
    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    INDArray input = Nd4j.rand(5, nIn);
    List<INDArray> activations = network.feedForward(input);
    //4 layers + input
    assertEquals(5, activations.size());
    List<INDArray> activationsAll = network.feedForwardToLayer(3, input);
    assertEquals(activations, activationsAll);
    for (int i = 3; i >= 0; i--) {
        List<INDArray> activationsPartial = network.feedForwardToLayer(i, input);
        //i+2: for layer 3: input + activations of {0,1,2,3} -> 5 total = 3+2
        assertEquals(i + 2, activationsPartial.size());
        for (int j = 0; j <= i; j++) {
            INDArray exp = activationsAll.get(j);
            INDArray act = activationsPartial.get(j);
            assertEquals(exp, act);
        }
    }
}