Examples with NormalDistribution org.deeplearning4j.nn.conf.distribution.NormalDistribution used on opensource projects

Example 21 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class VaeGradientCheckTests method testVaeAsMLP.

@Test
public void testVaeAsMLP() {
    //Post pre-training: a VAE can be used as a MLP, by taking the mean value from p(z|x) as the output
    //This gradient check tests this part
    //activation functions such as relu and hardtanh: may randomly fail due to discontinuities
    String[] activFns = { "identity", "tanh" };
    LossFunction[] lossFunctions = { LossFunction.MCXENT, LossFunction.MSE };
    //i.e., lossFunctions[i] used with outputActivations[i] here
    String[] outputActivations = { "softmax", "tanh" };
    //use l2vals[i] with l1vals[i]
    double[] l2vals = { 0.4, 0.0, 0.4, 0.4 };
    double[] l1vals = { 0.0, 0.0, 0.5, 0.0 };
    double[] biasL2 = { 0.0, 0.0, 0.0, 0.2 };
    double[] biasL1 = { 0.0, 0.0, 0.6, 0.0 };
    int[][] encoderLayerSizes = new int[][] { { 5 }, { 5, 6 } };
    int[][] decoderLayerSizes = new int[][] { { 6 }, { 7, 8 } };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        INDArray input = Nd4j.rand(minibatch, 4);
        INDArray labels = Nd4j.create(minibatch, 3);
        for (int i = 0; i < minibatch; i++) {
            labels.putScalar(i, i % 3, 1.0);
        }
        for (int ls = 0; ls < encoderLayerSizes.length; ls++) {
            int[] encoderSizes = encoderLayerSizes[ls];
            int[] decoderSizes = decoderLayerSizes[ls];
            for (String afn : activFns) {
                for (int i = 0; i < lossFunctions.length; i++) {
                    for (int k = 0; k < l2vals.length; k++) {
                        LossFunction lf = lossFunctions[i];
                        String outputActivation = outputActivations[i];
                        double l2 = l2vals[k];
                        double l1 = l1vals[k];
                        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(l2).l1(l1).l2Bias(biasL2[k]).l1Bias(biasL1[k]).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).list().layer(0, new VariationalAutoencoder.Builder().nIn(4).nOut(3).encoderLayerSizes(encoderSizes).decoderLayerSizes(decoderSizes).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).activation(afn).updater(Updater.SGD).build()).layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nIn(3).nOut(3).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.SGD).build()).pretrain(false).backprop(true).build();
                        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                        mln.init();
                        String msg = "testVaeAsMLP() - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", encLayerSizes = " + Arrays.toString(encoderSizes) + ", decLayerSizes = " + Arrays.toString(decoderSizes) + ", l2=" + l2 + ", l1=" + l1;
                        if (PRINT_RESULTS) {
                            System.out.println(msg);
                            for (int j = 0; j < mln.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                        }
                        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
                        assertTrue(msg, gradOK);
                    }
                }
            }
        }
    }
}

Example 22 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class MultiLayerNeuralNetConfigurationTest method testYaml.

@Test
public void testYaml() throws Exception {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new RBM.Builder().dist(new NormalDistribution(1, 1e-1)).build()).inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build();
    String json = conf.toYaml();
    MultiLayerConfiguration from = MultiLayerConfiguration.fromYaml(json);
    assertEquals(conf.getConf(0), from.getConf(0));
    Properties props = new Properties();
    props.put("json", json);
    String key = props.getProperty("json");
    assertEquals(json, key);
    File f = new File("props");
    f.deleteOnExit();
    BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f));
    props.store(bos, "");
    bos.flush();
    bos.close();
    BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f));
    Properties props2 = new Properties();
    props2.load(bis);
    bis.close();
    assertEquals(props2.getProperty("json"), props.getProperty("json"));
    String yaml = props2.getProperty("json");
    MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromYaml(yaml);
    assertEquals(conf.getConf(0), conf3.getConf(0));
}

Example 23 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class MultiLayerNeuralNetConfigurationTest method testJson.

@Test
public void testJson() throws Exception {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new RBM.Builder().dist(new NormalDistribution(1, 1e-1)).build()).inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build();
    String json = conf.toJson();
    MultiLayerConfiguration from = MultiLayerConfiguration.fromJson(json);
    assertEquals(conf.getConf(0), from.getConf(0));
    Properties props = new Properties();
    props.put("json", json);
    String key = props.getProperty("json");
    assertEquals(json, key);
    File f = new File("props");
    f.deleteOnExit();
    BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f));
    props.store(bos, "");
    bos.flush();
    bos.close();
    BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f));
    Properties props2 = new Properties();
    props2.load(bis);
    bis.close();
    assertEquals(props2.getProperty("json"), props.getProperty("json"));
    String json2 = props2.getProperty("json");
    MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromJson(json2);
    assertEquals(conf.getConf(0), conf3.getConf(0));
}

Example 24 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GradientCheckTests method testGradientGravesBidirectionalLSTMEdgeCases.

@Test
public void testGradientGravesBidirectionalLSTMEdgeCases() {
    //Edge cases: T=1, miniBatchSize=1, both
    int[] timeSeriesLength = { 1, 5, 1 };
    int[] miniBatchSize = { 7, 1, 1 };
    int nIn = 7;
    int layerSize = 9;
    int nOut = 4;
    for (int i = 0; i < timeSeriesLength.length; i++) {
        Random r = new Random(12345L);
        INDArray input = Nd4j.zeros(miniBatchSize[i], nIn, timeSeriesLength[i]);
        for (int m = 0; m < miniBatchSize[i]; m++) {
            for (int j = 0; j < nIn; j++) {
                for (int k = 0; k < timeSeriesLength[i]; k++) {
                    input.putScalar(new int[] { m, j, k }, r.nextDouble() - 0.5);
                }
            }
        }
        INDArray labels = Nd4j.zeros(miniBatchSize[i], nOut, timeSeriesLength[i]);
        for (int m = 0; m < miniBatchSize[i]; m++) {
            for (int j = 0; j < timeSeriesLength[i]; j++) {
                int idx = r.nextInt(nOut);
                labels.putScalar(new int[] { m, idx, j }, 1.0f);
            }
        }
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).seed(12345L).list().layer(0, new GravesBidirectionalLSTM.Builder().nIn(nIn).nOut(layerSize).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).layer(1, new RnnOutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).pretrain(false).backprop(true).build();
        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();
        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
        String msg = "testGradientGravesLSTMEdgeCases() - timeSeriesLength=" + timeSeriesLength[i] + ", miniBatchSize=" + miniBatchSize[i];
        assertTrue(msg, gradOK);
    }
}

Example 25 with NormalDistribution

use of org.deeplearning4j.nn.conf.distribution.NormalDistribution in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testCnnDepthMerge.

@Test
public void testCnnDepthMerge() {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 0.1)).updater(Updater.NONE).learningRate(1.0).graphBuilder().addInputs("input").addLayer("l1", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(2).nOut(2).activation(Activation.TANH).build(), "input").addLayer("l2", new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(2).nOut(2).activation(Activation.TANH).build(), "input").addVertex("merge", new MergeVertex(), "l1", "l2").addLayer("outputLayer", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(5 * 5 * (2 + 2)).nOut(3).build(), "merge").setOutputs("outputLayer").inputPreProcessor("outputLayer", new CnnToFeedForwardPreProcessor(5, 5, 4)).pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    Random r = new Random(12345);
    //Order: examples, channels, height, width
    INDArray input = Nd4j.rand(new int[] { 5, 2, 6, 6 });
    INDArray labels = Nd4j.zeros(5, 3);
    for (int i = 0; i < 5; i++) labels.putScalar(new int[] { i, r.nextInt(3) }, 1.0);
    if (PRINT_RESULTS) {
        System.out.println("testCnnDepthMerge()");
        for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
    }
    boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { input }, new INDArray[] { labels });
    String msg = "testCnnDepthMerge()";
    assertTrue(msg, gradOK);
}