Examples with MultiLayerNetwork org.deeplearning4j.nn.multilayer.MultiLayerNetwork used on opensource projects

Example 61 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class LossFunctionGradientCheck method lossFunctionWeightedGradientCheck.

@Test
public void lossFunctionWeightedGradientCheck() {
    INDArray[] weights = new INDArray[] { Nd4j.create(new double[] { 0.2, 0.3, 0.5 }), Nd4j.create(new double[] { 1.0, 0.5, 2.0 }) };
    List<String> passed = new ArrayList<>();
    List<String> failed = new ArrayList<>();
    for (INDArray w : weights) {
        ILossFunction[] lossFunctions = new ILossFunction[] { new LossBinaryXENT(w), new LossL1(w), new LossL1(w), new LossL2(w), new LossL2(w), new LossMAE(w), new LossMAE(w), new LossMAPE(w), new LossMAPE(w), new LossMCXENT(w), new LossMSE(w), new LossMSE(w), new LossMSLE(w), new LossMSLE(w), new LossNegativeLogLikelihood(w), new LossNegativeLogLikelihood(w) };
        String[] outputActivationFn = new String[] { //xent
        "sigmoid", //l1
        "tanh", //l1 + softmax
        "softmax", //l2
        "tanh", //l2 + softmax
        "softmax", //mae
        "identity", //mae + softmax
        "softmax", //mape
        "identity", //mape + softmax
        "softmax", //mcxent
        "softmax", //mse
        "identity", //mse + softmax
        "softmax", //msle  -   requires positive labels/activations due to log
        "sigmoid", //msle + softmax
        "softmax", //nll
        "sigmoid", //nll + softmax
        "softmax" };
        int[] minibatchSizes = new int[] { 1, 3 };
        for (int i = 0; i < lossFunctions.length; i++) {
            for (int j = 0; j < minibatchSizes.length; j++) {
                String testName = lossFunctions[i] + " - " + outputActivationFn[i] + " - minibatchSize = " + minibatchSizes[j] + "; weights = " + w;
                Nd4j.getRandom().setSeed(12345);
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(1).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345).updater(Updater.NONE).regularization(false).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(-3, 3)).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1, new OutputLayer.Builder().lossFunction(lossFunctions[i]).activation(outputActivationFn[i]).nIn(4).nOut(3).build()).pretrain(false).backprop(true).build();
                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();
                INDArray[] inOut = getFeaturesAndLabels(lossFunctions[i], minibatchSizes[j], 4, 3, 12345);
                INDArray input = inOut[0];
                INDArray labels = inOut[1];
                log.info(" ***** Starting test: {} *****", testName);
                //                System.out.println(Arrays.toString(labels.data().asDouble()));
                //                System.out.println(Arrays.toString(net.output(input,false).data().asDouble()));
                //                System.out.println(net.score(new DataSet(input,labels)));
                boolean gradOK;
                try {
                    gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
                } catch (Exception e) {
                    e.printStackTrace();
                    failed.add(testName + "\t" + "EXCEPTION");
                    continue;
                }
                if (gradOK) {
                    passed.add(testName);
                } else {
                    failed.add(testName);
                }
                System.out.println("\n\n");
            }
        }
    }
    System.out.println("---- Passed ----");
    for (String s : passed) {
        System.out.println(s);
    }
    System.out.println("---- Failed ----");
    for (String s : failed) {
        System.out.println(s);
    }
    assertEquals("Tests failed", 0, failed.size());
}

Example 62 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class VaeGradientCheckTests method testVaePretrain.

@Test
public void testVaePretrain() {
    //activation functions such as relu and hardtanh: may randomly fail due to discontinuities
    String[] activFns = { "identity", "identity", "tanh", "tanh" };
    String[] pzxAfns = { "identity", "tanh", "identity", "tanh" };
    String[] pxzAfns = { "tanh", "identity", "tanh", "identity" };
    //use l2vals[i] with l1vals[i]
    double[] l2vals = { 0.4, 0.0, 0.4, 0.4 };
    double[] l1vals = { 0.0, 0.0, 0.5, 0.0 };
    double[] biasL2 = { 0.0, 0.0, 0.0, 0.2 };
    double[] biasL1 = { 0.0, 0.0, 0.6, 0.0 };
    int[][] encoderLayerSizes = new int[][] { { 5 }, { 5, 6 } };
    int[][] decoderLayerSizes = new int[][] { { 6 }, { 7, 8 } };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        INDArray features = Nd4j.rand(minibatch, 4);
        for (int ls = 0; ls < encoderLayerSizes.length; ls++) {
            int[] encoderSizes = encoderLayerSizes[ls];
            int[] decoderSizes = decoderLayerSizes[ls];
            for (int j = 0; j < activFns.length; j++) {
                String afn = activFns[j];
                String pzxAfn = pzxAfns[j];
                String pxzAfn = pxzAfns[j];
                //Ideally we'd do the cartesian product of l1/l2 and the activation functions, but that takes too long...
                double l2 = l2vals[j];
                double l1 = l1vals[j];
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(l2).l1(l1).l2Bias(biasL2[j]).l1Bias(biasL1[j]).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).weightInit(WeightInit.XAVIER).list().layer(0, new VariationalAutoencoder.Builder().nIn(4).nOut(3).encoderLayerSizes(encoderSizes).decoderLayerSizes(decoderSizes).pzxActivationFunction(pzxAfn).reconstructionDistribution(new GaussianReconstructionDistribution(pxzAfn)).activation(afn).updater(Updater.SGD).build()).pretrain(true).backprop(false).build();
                MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                mln.init();
                mln.initGradientsView();
                org.deeplearning4j.nn.api.Layer layer = mln.getLayer(0);
                String msg = "testVaePretrain() - activationFn=" + afn + ", p(z|x) afn = " + pzxAfn + ", p(x|z) afn = " + pxzAfn + ", encLayerSizes = " + Arrays.toString(encoderSizes) + ", decLayerSizes = " + Arrays.toString(decoderSizes) + ", l2=" + l2 + ", l1=" + l1;
                if (PRINT_RESULTS) {
                    System.out.println(msg);
                    for (int l = 0; l < mln.getnLayers(); l++) System.out.println("Layer " + l + " # params: " + mln.getLayer(l).numParams());
                }
                boolean gradOK = GradientCheckUtil.checkGradientsPretrainLayer(layer, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, features, 12345);
                assertTrue(msg, gradOK);
            }
        }
    }
}

Example 63 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class TestComputationGraphNetwork method testForwardBasicIris.

@Test
public void testForwardBasicIris() {
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();
    MultiLayerConfiguration mlc = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();
    DataSetIterator iris = new IrisDataSetIterator(150, 150);
    DataSet ds = iris.next();
    graph.setInput(0, ds.getFeatureMatrix());
    Map<String, INDArray> activations = graph.feedForward(false);
    //2 layers + 1 input node
    assertEquals(3, activations.size());
    assertTrue(activations.containsKey("input"));
    assertTrue(activations.containsKey("firstLayer"));
    assertTrue(activations.containsKey("outputLayer"));
    //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same outputs
    Nd4j.getRandom().setSeed(12345);
    int nParams = getNumParams();
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());
    List<INDArray> mlnAct = net.feedForward(ds.getFeatureMatrix(), false);
    activations = graph.feedForward(ds.getFeatureMatrix(), false);
    assertEquals(mlnAct.get(0), activations.get("input"));
    assertEquals(mlnAct.get(1), activations.get("firstLayer"));
    assertEquals(mlnAct.get(2), activations.get("outputLayer"));
}

Example 64 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class TestComputationGraphNetwork method testIrisFit.

@Test
public void testIrisFit() {
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();
    MultiLayerConfiguration mlnConfig = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlnConfig);
    net.init();
    Nd4j.getRandom().setSeed(12345);
    int nParams = getNumParams();
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());
    DataSetIterator iris = new IrisDataSetIterator(75, 150);
    net.fit(iris);
    iris.reset();
    graph.fit(iris);
    //Check that parameters are equal for both models after fitting:
    INDArray paramsMLN = net.params();
    INDArray paramsGraph = graph.params();
    assertNotEquals(params, paramsGraph);
    assertEquals(paramsMLN, paramsGraph);
}

Example 65 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class TestComputationGraphNetwork method testScoringDataSet.

@Test
public void testScoringDataSet() {
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();
    MultiLayerConfiguration mlc = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();
    DataSetIterator iris = new IrisDataSetIterator(150, 150);
    DataSet ds = iris.next();
    //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same score
    Nd4j.getRandom().setSeed(12345);
    int nParams = getNumParams();
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());
    double scoreMLN = net.score(ds, false);
    double scoreCG = graph.score(ds, false);
    assertEquals(scoreMLN, scoreCG, 1e-4);
}