Examples with NormalDistribution org.deeplearning4j.nn.conf.distribution.NormalDistribution used on opensource projects

Example 36 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TestVariableLengthTS method testOutputMasking.

@Test
public void testOutputMasking() {
    //If labels are masked: want zero outputs for that time step.
    int nIn = 3;
    int[] timeSeriesLengths = { 3, 10 };
    int[] outputSizes = { 1, 2, 5 };
    int[] miniBatchSizes = { 1, 4 };
    Random r = new Random(12345);
    for (int tsLength : timeSeriesLengths) {
        for (int nOut : outputSizes) {
            for (int miniBatch : miniBatchSizes) {
                for (int nToMask = 0; nToMask < tsLength - 1; nToMask++) {
                    INDArray labelMaskArray = Nd4j.ones(miniBatch, tsLength);
                    for (int i = 0; i < miniBatch; i++) {
                        //For each example: select which outputs to mask...
                        int nMasked = 0;
                        while (nMasked < nToMask) {
                            int tryIdx = r.nextInt(tsLength);
                            if (labelMaskArray.getDouble(i, tryIdx) == 0.0)
                                continue;
                            labelMaskArray.putScalar(new int[] { i, tryIdx }, 0.0);
                            nMasked++;
                        }
                    }
                    INDArray input = Nd4j.rand(new int[] { miniBatch, nIn, tsLength });
                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(5).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(5).nOut(nOut).weightInit(WeightInit.XAVIER).updater(Updater.NONE).build()).pretrain(false).backprop(true).build();
                    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                    mln.init();
                    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().regularization(false).seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(5).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(5).nOut(nOut).weightInit(WeightInit.XAVIER).updater(Updater.NONE).build()).pretrain(false).backprop(true).build();
                    MultiLayerNetwork mln2 = new MultiLayerNetwork(conf2);
                    mln2.init();
                    mln.setLayerMaskArrays(null, labelMaskArray);
                    mln2.setLayerMaskArrays(null, labelMaskArray);
                    INDArray out = mln.output(input);
                    INDArray out2 = mln2.output(input);
                    for (int i = 0; i < miniBatch; i++) {
                        for (int j = 0; j < tsLength; j++) {
                            double m = labelMaskArray.getDouble(i, j);
                            if (m == 0.0) {
                                //Expect outputs to be exactly 0.0
                                INDArray outRow = out.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.point(j));
                                INDArray outRow2 = out2.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.point(j));
                                for (int k = 0; k < nOut; k++) {
                                    assertEquals(outRow.getDouble(k), 0.0, 0.0);
                                    assertEquals(outRow2.getDouble(k), 0.0, 0.0);
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 37 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TransferLearningCompGraphTest method testNoutChanges.

@Test
public void testNoutChanges() {
    DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 2));
    NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().learningRate(0.1).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.SGD).activation(Activation.IDENTITY);
    FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().learningRate(0.1).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.SGD).activation(Activation.IDENTITY).build();
    ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer2").setOutputs("layer3").build());
    modelToFineTune.init();
    ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune).fineTuneConfiguration(fineTuneConfiguration).nOutReplace("layer3", 2, WeightInit.XAVIER).nOutReplace("layer0", 3, new NormalDistribution(1, 1e-1), WeightInit.XAVIER).build();
    assertEquals(modelNow.getLayer("layer0").conf().getLayer().getWeightInit(), WeightInit.DISTRIBUTION);
    assertEquals(modelNow.getLayer("layer0").conf().getLayer().getDist(), new NormalDistribution(1, 1e-1));
    assertEquals(modelNow.getLayer("layer1").conf().getLayer().getWeightInit(), WeightInit.XAVIER);
    assertEquals(modelNow.getLayer("layer1").conf().getLayer().getDist(), null);
    assertEquals(modelNow.getLayer("layer3").conf().getLayer().getWeightInit(), WeightInit.XAVIER);
    ComputationGraph modelExpectedArch = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(2).build(), "layer2").setOutputs("layer3").build());
    modelExpectedArch.init();
    //modelNow should have the same architecture as modelExpectedArch
    assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape());
    assertArrayEquals(modelExpectedArch.getLayer("layer0").params().shape(), modelNow.getLayer("layer0").params().shape());
    assertArrayEquals(modelExpectedArch.getLayer("layer1").params().shape(), modelNow.getLayer("layer1").params().shape());
    assertArrayEquals(modelExpectedArch.getLayer("layer2").params().shape(), modelNow.getLayer("layer2").params().shape());
    assertArrayEquals(modelExpectedArch.getLayer("layer3").params().shape(), modelNow.getLayer("layer3").params().shape());
    modelNow.setParams(modelExpectedArch.params());
    //fit should give the same results
    modelExpectedArch.fit(randomData);
    modelNow.fit(randomData);
    assertEquals(modelExpectedArch.score(), modelNow.score(), 1e-8);
    assertEquals(modelExpectedArch.params(), modelNow.params());
}

Example 38 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class TestVAE method testReconstructionDistributionsSimple.

@Test
public void testReconstructionDistributionsSimple() {
    int inOutSize = 6;
    ReconstructionDistribution[] reconstructionDistributions = new ReconstructionDistribution[] { new GaussianReconstructionDistribution(Activation.IDENTITY), new GaussianReconstructionDistribution(Activation.TANH), new BernoulliReconstructionDistribution(Activation.SIGMOID), new CompositeReconstructionDistribution.Builder().addDistribution(2, new GaussianReconstructionDistribution(Activation.IDENTITY)).addDistribution(2, new BernoulliReconstructionDistribution()).addDistribution(2, new GaussianReconstructionDistribution(Activation.TANH)).build() };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        for (int i = 0; i < reconstructionDistributions.length; i++) {
            INDArray data;
            switch(i) {
                //Gaussian + identity
                case 0:
                case //Gaussian + tanh
                1:
                    data = Nd4j.rand(minibatch, inOutSize);
                    break;
                case //Bernoulli
                2:
                    data = Nd4j.create(minibatch, inOutSize);
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data, 0.5), Nd4j.getRandom());
                    break;
                case //Composite
                3:
                    data = Nd4j.create(minibatch, inOutSize);
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 2)).assign(Nd4j.rand(minibatch, 2));
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data.get(NDArrayIndex.all(), NDArrayIndex.interval(2, 4)), 0.5), Nd4j.getRandom());
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(4, 6)).assign(Nd4j.rand(minibatch, 2));
                    break;
                default:
                    throw new RuntimeException();
            }
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(0.2).l1(0.3).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).list().layer(0, new VariationalAutoencoder.Builder().nIn(inOutSize).nOut(3).encoderLayerSizes(5).decoderLayerSizes(6).pzxActivationFunction(Activation.TANH).reconstructionDistribution(reconstructionDistributions[i]).activation(new ActivationTanH()).updater(Updater.SGD).build()).pretrain(true).backprop(false).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();
            mln.initGradientsView();
            mln.fit(data);
            org.deeplearning4j.nn.layers.variational.VariationalAutoencoder layer = (org.deeplearning4j.nn.layers.variational.VariationalAutoencoder) mln.getLayer(0);
            assertFalse(layer.hasLossFunction());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionProb = layer.reconstructionProbability(data, 50);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionProb.shape());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionLogProb = layer.reconstructionLogProbability(data, 50);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionLogProb.shape());
            //                System.out.println(reconstructionDistributions[i]);
            for (int j = 0; j < minibatch; j++) {
                double p = reconstructionProb.getDouble(j);
                double logp = reconstructionLogProb.getDouble(j);
                assertTrue(p >= 0.0 && p <= 1.0);
                assertTrue(logp <= 0.0);
                double pFromLogP = Math.exp(logp);
                assertEquals(p, pFromLogP, 1e-6);
            }
        }
    }
}

Example 39 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GravesLSTMOutputTest method getNetworkConf.

private MultiLayerConfiguration getNetworkConf(int iterations, boolean useTBPTT) {
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(0.1).regularization(true).l2(0.0025).iterations(iterations).stepFunction(new NegativeDefaultStepFunction()).list().layer(0, new GravesLSTM.Builder().weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0.0, 0.01)).nIn(nIn).nOut(layerSize).updater(Updater.ADAGRAD).activation(Activation.TANH).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).updater(Updater.ADAGRAD).nIn(layerSize).nOut(nIn).activation(Activation.SOFTMAX).build()).inputPreProcessor(1, new RnnToFeedForwardPreProcessor()).backprop(true).pretrain(false);
    if (useTBPTT) {
        builder.backpropType(BackpropType.TruncatedBPTT);
        builder.tBPTTBackwardLength(window / 3);
        builder.tBPTTForwardLength(window / 3);
    }
    return builder.build();
}

Example 40 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class MultiLayerTestRNN method testRnnActivateUsingStoredState.

@Test
public void testRnnActivateUsingStoredState() {
    int timeSeriesLength = 12;
    int miniBatchSize = 7;
    int nIn = 5;
    int nOut = 4;
    int nTimeSlices = 5;
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7).activation(Activation.TANH).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 0.5)).build()).layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8).activation(Activation.TANH).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 0.5)).build()).layer(2, new RnnOutputLayer.Builder(LossFunction.MCXENT).weightInit(WeightInit.DISTRIBUTION).nIn(8).nOut(nOut).activation(Activation.SOFTMAX).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 0.5)).build()).build();
    Nd4j.getRandom().setSeed(12345);
    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
    mln.init();
    INDArray inputLong = Nd4j.rand(new int[] { miniBatchSize, nIn, nTimeSlices * timeSeriesLength });
    INDArray input = inputLong.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, timeSeriesLength));
    List<INDArray> outStandard = mln.feedForward(input, true);
    List<INDArray> outRnnAct = mln.rnnActivateUsingStoredState(input, true, true);
    //As initially state is zeros: expect these to be the same
    assertEquals(outStandard, outRnnAct);
    //Furthermore, expect multiple calls to this function to be the same:
    for (int i = 0; i < 3; i++) {
        assertEquals(outStandard, mln.rnnActivateUsingStoredState(input, true, true));
    }
    List<INDArray> outStandardLong = mln.feedForward(inputLong, true);
    BaseRecurrentLayer<?> l0 = ((BaseRecurrentLayer<?>) mln.getLayer(0));
    BaseRecurrentLayer<?> l1 = ((BaseRecurrentLayer<?>) mln.getLayer(1));
    for (int i = 0; i < nTimeSlices; i++) {
        INDArray inSlice = inputLong.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(i * timeSeriesLength, (i + 1) * timeSeriesLength));
        List<INDArray> outSlice = mln.rnnActivateUsingStoredState(inSlice, true, true);
        List<INDArray> expOut = new ArrayList<>();
        for (INDArray temp : outStandardLong) {
            expOut.add(temp.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(i * timeSeriesLength, (i + 1) * timeSeriesLength)));
        }
        for (int j = 0; j < expOut.size(); j++) {
            INDArray exp = expOut.get(j);
            INDArray act = outSlice.get(j);
            System.out.println(j);
            System.out.println(exp.sub(act));
            assertEquals(exp, act);
        }
        assertEquals(expOut, outSlice);
        //Again, expect multiple calls to give the same output
        for (int j = 0; j < 3; j++) {
            outSlice = mln.rnnActivateUsingStoredState(inSlice, true, true);
            assertEquals(expOut, outSlice);
        }
        l0.rnnSetPreviousState(l0.rnnGetTBPTTState());
        l1.rnnSetPreviousState(l1.rnnGetTBPTTState());
    }
}