Examples with ComputationGraph org.deeplearning4j.nn.graph.ComputationGraph used on opensource projects

Example 56 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class TestEarlyStoppingCompGraph method testBadTuning.

@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).learningRate(//Intentionally huge LR
    5.0).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in").setOutputs("0").pretrain(false).backprop(true).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));
    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>().epochTerminationConditions(new MaxEpochsTerminationCondition(5000)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES), //Initial score is ~2.5
    new MaxScoreIterationTerminationCondition(10)).scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();
    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition, result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(10).toString();
    assertEquals(expDetails, result.getTerminationDetails());
    assertEquals(0, result.getBestModelEpoch());
    assertNotNull(result.getBestModel());
}

Example 57 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testBasicCenterLoss.

@Test
public void testBasicCenterLoss() {
    Nd4j.getRandom().setSeed(12345);
    int numLabels = 2;
    boolean[] trainFirst = new boolean[] { false, true };
    for (boolean train : trainFirst) {
        for (double lambda : new double[] { 0.0, 0.5, 2.0 }) {
            ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new GaussianDistribution(0, 1)).updater(Updater.NONE).learningRate(1.0).graphBuilder().addInputs("input1").addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(), "input1").addLayer("cl", new CenterLossOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(numLabels).alpha(1.0).lambda(lambda).gradientCheck(true).activation(Activation.SOFTMAX).build(), "l1").setOutputs("cl").pretrain(false).backprop(true).build();
            ComputationGraph graph = new ComputationGraph(conf);
            graph.init();
            INDArray example = Nd4j.rand(150, 4);
            INDArray labels = Nd4j.zeros(150, numLabels);
            Random r = new Random(12345);
            for (int i = 0; i < 150; i++) {
                labels.putScalar(i, r.nextInt(numLabels), 1.0);
            }
            if (train) {
                for (int i = 0; i < 10; i++) {
                    INDArray f = Nd4j.rand(10, 4);
                    INDArray l = Nd4j.zeros(10, numLabels);
                    for (int j = 0; j < 10; j++) {
                        l.putScalar(j, r.nextInt(numLabels), 1.0);
                    }
                    graph.fit(new INDArray[] { f }, new INDArray[] { l });
                }
            }
            String msg = "testBasicCenterLoss() - lambda = " + lambda + ", trainFirst = " + train;
            if (PRINT_RESULTS) {
                System.out.println(msg);
                for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
            }
            boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { example }, new INDArray[] { labels });
            assertTrue(msg, gradOK);
        }
    }
}

Example 58 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testLSTMWithDuplicateToTimeSeries.

@Test
public void testLSTMWithDuplicateToTimeSeries() {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).learningRate(1.0).graphBuilder().addInputs("input1", "input2").setOutputs("out").addLayer("lstm1", new GravesLSTM.Builder().nIn(3).nOut(4).activation(Activation.TANH).build(), "input1").addLayer("lstm2", new GravesLSTM.Builder().nIn(4).nOut(5).activation(Activation.SOFTSIGN).build(), "input2").addVertex("lastTS", new LastTimeStepVertex("input2"), "lstm2").addVertex("duplicate", new DuplicateToTimeSeriesVertex("input2"), "lastTS").addLayer("out", new RnnOutputLayer.Builder().nIn(5 + 4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "lstm1", "duplicate").pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    Random r = new Random(12345);
    INDArray input1 = Nd4j.rand(new int[] { 3, 3, 5 });
    INDArray input2 = Nd4j.rand(new int[] { 3, 4, 5 });
    INDArray labels = Nd4j.zeros(3, 3, 5);
    for (int i = 0; i < 3; i++) {
        for (int j = 0; j < 5; j++) {
            labels.putScalar(new int[] { i, r.nextInt(3), j }, 1.0);
        }
    }
    if (PRINT_RESULTS) {
        System.out.println("testLSTMWithDuplicateToTimeSeries()");
        for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
    }
    boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { input1, input2 }, new INDArray[] { labels });
    String msg = "testLSTMWithDuplicateToTimeSeries()";
    assertTrue(msg, gradOK);
}

Example 59 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testBasicIrisWithMerging.

@Test
public void testBasicIrisWithMerging() {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).learningRate(1.0).graphBuilder().addInputs("input").addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(), "input").addLayer("l2", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build(), "input").addVertex("merge", new MergeVertex(), "l1", "l2").addLayer("outputLayer", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(5 + 5).nOut(3).build(), "merge").setOutputs("outputLayer").pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    int numParams = (4 * 5 + 5) + (4 * 5 + 5) + (10 * 3 + 3);
    assertEquals(numParams, graph.numParams());
    Nd4j.getRandom().setSeed(12345);
    int nParams = graph.numParams();
    INDArray newParams = Nd4j.rand(1, nParams);
    graph.setParams(newParams);
    DataSet ds = new IrisDataSetIterator(150, 150).next();
    INDArray min = ds.getFeatureMatrix().min(0);
    INDArray max = ds.getFeatureMatrix().max(0);
    ds.getFeatureMatrix().subiRowVector(min).diviRowVector(max.sub(min));
    INDArray input = ds.getFeatureMatrix();
    INDArray labels = ds.getLabels();
    if (PRINT_RESULTS) {
        System.out.println("testBasicIrisWithMerging()");
        for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
    }
    boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { input }, new INDArray[] { labels });
    String msg = "testBasicIrisWithMerging()";
    assertTrue(msg, gradOK);
}

Example 60 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testMultipleOutputsMergeVertex.

@Test
public void testMultipleOutputsMergeVertex() {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).learningRate(1.0).activation(Activation.TANH).graphBuilder().addInputs("i0", "i1", "i2").addLayer("d0", new DenseLayer.Builder().nIn(2).nOut(2).build(), "i0").addLayer("d1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "i1").addLayer("d2", new DenseLayer.Builder().nIn(2).nOut(2).build(), "i2").addVertex("m", new MergeVertex(), "d0", "d1", "d2").addLayer("D0", new DenseLayer.Builder().nIn(6).nOut(2).build(), "m").addLayer("D1", new DenseLayer.Builder().nIn(6).nOut(2).build(), "m").addLayer("D2", new DenseLayer.Builder().nIn(6).nOut(2).build(), "m").addLayer("out", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(6).nOut(2).build(), "D0", "D1", "D2").setOutputs("out").pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    int[] minibatchSizes = { 1, 3 };
    for (int mb : minibatchSizes) {
        INDArray[] input = new INDArray[3];
        for (int i = 0; i < 3; i++) {
            input[i] = Nd4j.rand(mb, 2);
        }
        INDArray out = Nd4j.rand(mb, 2);
        String msg = "testMultipleOutputsMergeVertex() - minibatchSize = " + mb;
        if (PRINT_RESULTS) {
            System.out.println(msg);
            for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }
        boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, new INDArray[] { out });
        assertTrue(msg, gradOK);
    }
}