Examples with NormalDistribution org.deeplearning4j.nn.conf.distribution.NormalDistribution used on opensource projects

Example 51 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testBasicL2.

@Test
public void testBasicL2() {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).activation(Activation.TANH).updater(Updater.NONE).learningRate(1.0).graphBuilder().addInputs("in1", "in2").addLayer("d0", new DenseLayer.Builder().nIn(2).nOut(2).build(), "in1").addLayer("d1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "in2").addVertex("l2", new L2Vertex(), "d0", "d1").addLayer("out", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.L2).nIn(1).nOut(1).activation(Activation.IDENTITY).build(), "l2").setOutputs("out").pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    Nd4j.getRandom().setSeed(12345);
    int nParams = graph.numParams();
    INDArray newParams = Nd4j.rand(1, nParams);
    graph.setParams(newParams);
    int[] mbSizes = new int[] { 1, 3, 10 };
    for (int minibatch : mbSizes) {
        INDArray in1 = Nd4j.rand(minibatch, 2);
        INDArray in2 = Nd4j.rand(minibatch, 2);
        INDArray labels = Nd4j.rand(minibatch, 1);
        String testName = "testBasicL2() - minibatch = " + minibatch;
        if (PRINT_RESULTS) {
            System.out.println(testName);
            for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }
        boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { in1, in2 }, new INDArray[] { labels });
        assertTrue(testName, gradOK);
    }
}

Example 52 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testMultipleOutputsMergeCnn.

@Test
public void testMultipleOutputsMergeCnn() {
    int inH = 7;
    int inW = 7;
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).learningRate(1.0).activation(Activation.TANH).graphBuilder().addInputs("input").addLayer("l0", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(2).nOut(2).activation(Activation.TANH).build(), "input").addLayer("l1", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(2).nOut(2).activation(Activation.TANH).build(), "l0").addLayer("l2", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(2).nOut(2).activation(Activation.TANH).build(), "l0").addVertex("m", new MergeVertex(), "l1", "l2").addLayer("l3", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(4).nOut(2).activation(Activation.TANH).build(), "m").addLayer("l4", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(4).nOut(2).activation(Activation.TANH).build(), "m").addLayer("out", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(2).build(), "l3", "l4").setOutputs("out").setInputTypes(InputType.convolutional(inH, inW, 2)).pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    int[] minibatchSizes = { 1, 3 };
    for (int mb : minibatchSizes) {
        //Order: examples, channels, height, width
        INDArray input = Nd4j.rand(new int[] { mb, 2, inH, inW });
        INDArray out = Nd4j.rand(mb, 2);
        String msg = "testMultipleOutputsMergeVertex() - minibatchSize = " + mb;
        if (PRINT_RESULTS) {
            System.out.println(msg);
            for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }
        boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { input }, new INDArray[] { out });
        assertTrue(msg, gradOK);
    }
}

Example 53 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testL2NormalizeVertex2d.

@Test
public void testL2NormalizeVertex2d() {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).activation(Activation.TANH).updater(Updater.NONE).learningRate(1.0).graphBuilder().addInputs("in1").addLayer("d1", new DenseLayer.Builder().nIn(2).nOut(3).build(), "in1").addVertex("norm", new L2NormalizeVertex(), "d1").addLayer("out1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.L2).nIn(3).nOut(2).activation(Activation.IDENTITY).build(), "norm").setOutputs("out1").build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    int[] mbSizes = new int[] { 1, 3, 10 };
    for (int minibatch : mbSizes) {
        INDArray in1 = Nd4j.rand(minibatch, 2);
        INDArray labels1 = Nd4j.rand(minibatch, 2);
        String testName = "testL2NormalizeVertex2d() - minibatch = " + minibatch;
        if (PRINT_RESULTS) {
            System.out.println(testName);
            for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }
        boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { in1 }, new INDArray[] { labels1 });
        assertTrue(testName, gradOK);
    }
}

Example 54 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testCnnPoolCenterLoss.

@Test
public void testCnnPoolCenterLoss() {
    Nd4j.getRandom().setSeed(12345);
    int numLabels = 2;
    boolean[] trainFirst = new boolean[] { false, true };
    int inputH = 5;
    int inputW = 4;
    int inputDepth = 3;
    for (boolean train : trainFirst) {
        for (double lambda : new double[] { 0.0, 0.5, 2.0 }) {
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).updater(Updater.NONE).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1.0)).seed(12345L).list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nOut(3).build()).layer(1, new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build()).layer(2, new CenterLossOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).nOut(numLabels).alpha(1.0).lambda(lambda).gradientCheck(true).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).setInputType(InputType.convolutional(inputH, inputW, inputDepth)).build();
            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();
            INDArray example = Nd4j.rand(new int[] { 150, inputDepth, inputH, inputW });
            INDArray labels = Nd4j.zeros(150, numLabels);
            Random r = new Random(12345);
            for (int i = 0; i < 150; i++) {
                labels.putScalar(i, r.nextInt(numLabels), 1.0);
            }
            if (train) {
                for (int i = 0; i < 10; i++) {
                    INDArray f = Nd4j.rand(new int[] { 10, inputDepth, inputH, inputW });
                    INDArray l = Nd4j.zeros(10, numLabels);
                    for (int j = 0; j < 10; j++) {
                        l.putScalar(j, r.nextInt(numLabels), 1.0);
                    }
                    net.fit(f, l);
                }
            }
            String msg = "testBasicCenterLoss() - trainFirst = " + train;
            if (PRINT_RESULTS) {
                System.out.println(msg);
                for (int j = 0; j < net.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
            }
            boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, example, labels);
            assertTrue(msg, gradOK);
        }
    }
}

Example 55 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsMasking method gradientCheckMaskingOutputSimple.

@Test
public void gradientCheckMaskingOutputSimple() {
    int timeSeriesLength = 5;
    boolean[][] mask = new boolean[5][0];
    //No masking
    mask[0] = new boolean[] { true, true, true, true, true };
    //mask first output time step
    mask[1] = new boolean[] { false, true, true, true, true };
    //time series classification: mask all but last
    mask[2] = new boolean[] { false, false, false, false, true };
    //time series classification w/ variable length TS
    mask[3] = new boolean[] { false, false, true, false, true };
    //variable length TS
    mask[4] = new boolean[] { true, true, true, false, true };
    int nIn = 4;
    int layerSize = 3;
    int nOut = 2;
    Random r = new Random(12345L);
    INDArray input = Nd4j.zeros(1, nIn, timeSeriesLength);
    for (int m = 0; m < 1; m++) {
        for (int j = 0; j < nIn; j++) {
            for (int k = 0; k < timeSeriesLength; k++) {
                input.putScalar(new int[] { m, j, k }, r.nextDouble() - 0.5);
            }
        }
    }
    INDArray labels = Nd4j.zeros(1, nOut, timeSeriesLength);
    for (int m = 0; m < 1; m++) {
        for (int j = 0; j < timeSeriesLength; j++) {
            int idx = r.nextInt(nOut);
            labels.putScalar(new int[] { m, idx, j }, 1.0f);
        }
    }
    for (int i = 0; i < mask.length; i++) {
        //Create mask array:
        INDArray maskArr = Nd4j.create(1, timeSeriesLength);
        for (int j = 0; j < mask[i].length; j++) {
            maskArr.putScalar(new int[] { 0, j }, mask[i][j] ? 1.0 : 0.0);
        }
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).pretrain(false).backprop(true).build();
        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();
        mln.setLayerMaskArrays(null, maskArr);
        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
        String msg = "gradientCheckMaskingOutputSimple() - timeSeriesLength=" + timeSeriesLength + ", miniBatchSize=" + 1;
        assertTrue(msg, gradOK);
    }
}