Examples with Gradient org.deeplearning4j.nn.gradient.Gradient used on opensource projects

Example 11 with Gradient

use of org.deeplearning4j.nn.gradient.Gradient in project deeplearning4j by deeplearning4j.

the class TestGraphNodes method testStackNode.

@Test
public void testStackNode() {
    Nd4j.getRandom().setSeed(12345);
    GraphVertex unstack = new StackVertex(null, "", -1);
    INDArray in1 = Nd4j.rand(5, 2);
    INDArray in2 = Nd4j.rand(5, 2);
    INDArray in3 = Nd4j.rand(5, 2);
    unstack.setInputs(in1, in2, in3);
    INDArray out = unstack.doForward(false);
    assertEquals(in1, out.get(NDArrayIndex.interval(0, 5), NDArrayIndex.all()));
    assertEquals(in2, out.get(NDArrayIndex.interval(5, 10), NDArrayIndex.all()));
    assertEquals(in3, out.get(NDArrayIndex.interval(10, 15), NDArrayIndex.all()));
    unstack.setEpsilon(out);
    Pair<Gradient, INDArray[]> b = unstack.doBackward(false);
    assertEquals(in1, b.getSecond()[0]);
    assertEquals(in2, b.getSecond()[1]);
    assertEquals(in3, b.getSecond()[2]);
}

Example 12 with Gradient

use of org.deeplearning4j.nn.gradient.Gradient in project deeplearning4j by deeplearning4j.

the class SubsamplingLayerTest method testSubSampleLayerAvgBackprop.

@Test
public void testSubSampleLayerAvgBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput = Nd4j.create(new double[] { 1., 2., 3., 4., 5., 6., 7., 8. }, new int[] { 1, 2, 2, 2 });
    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] { 0.25, 0.25, 0.5, 0.5, 0.25, 0.25, 0.5, 0.5, 0.75, 0.75, 1., 1., 0.75, 0.75, 1., 1., 1.25, 1.25, 1.5, 1.5, 1.25, 1.25, 1.5, 1.5, 1.75, 1.75, 2., 2., 1.75, 1.75, 2., 2. }, new int[] { 1, 2, 4, 4 });
    INDArray input = getContainedData();
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG);
    layer.activate(input);
    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput);
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertArrayEquals(expectedContainedEpsilonResult.shape(), containedOutput.getSecond().shape());
}

Example 13 with Gradient

use of org.deeplearning4j.nn.gradient.Gradient in project deeplearning4j by deeplearning4j.

the class SubsamplingLayerTest method testSubSampleLayerMaxBackprop.

//////////////////////////////////////////////////////////////////////////////////
@Test
public void testSubSampleLayerMaxBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput = Nd4j.create(new double[] { 1., 1., 1., 1., 1., 1., 1., 1. }, new int[] { 1, 2, 2, 2 });
    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] { 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0. }, new int[] { 1, 2, 4, 4 });
    INDArray input = getContainedData();
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX);
    layer.activate(input);
    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput);
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);
    INDArray input2 = getData();
    layer.activate(input2);
    int depth = input2.size(1);
    epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth);
    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon);
    assertEquals(input.shape().length, out.getSecond().shape().length);
    // depth retained
    assertEquals(depth, out.getSecond().size(1));
}

Example 14 with Gradient

use of org.deeplearning4j.nn.gradient.Gradient in project deeplearning4j by deeplearning4j.

the class TestDecayPolicies method testLearningRateSigmoidDecaySingleLayer.

@Test
public void testLearningRateSigmoidDecaySingleLayer() {
    int iterations = 2;
    double lr = 1e-2;
    double decayRate = 2;
    double steps = 3;
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr).learningRateDecayPolicy(LearningRatePolicy.Sigmoid).lrPolicyDecayRate(decayRate).lrPolicySteps(steps).iterations(iterations).layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).build();
    int numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true);
    Updater updater = UpdaterCreator.getUpdater(layer);
    Gradient gradientActual = new DefaultGradient();
    gradientActual.setGradientFor(DefaultParamInitializer.WEIGHT_KEY, weightGradient);
    gradientActual.setGradientFor(DefaultParamInitializer.BIAS_KEY, biasGradient);
    for (int i = 0; i < iterations; i++) {
        updater.update(layer, gradientActual, i, 1);
        double expectedLr = calcSigmoidDecay(layer.conf().getLearningRateByParam("W"), decayRate, i, steps);
        assertEquals(expectedLr, layer.conf().getLearningRateByParam("W"), 1e-4);
        assertEquals(expectedLr, layer.conf().getLearningRateByParam("b"), 1e-4);
    }
}

Example 15 with Gradient

use of org.deeplearning4j.nn.gradient.Gradient in project deeplearning4j by deeplearning4j.

the class TestDecayPolicies method testMomentumScheduleMLN.

@Test
public void testMomentumScheduleMLN() {
    double lr = 1e-2;
    double mu = 0.6;
    Map<Integer, Double> momentumAfter = new HashMap<>();
    momentumAfter.put(1, 0.2);
    int iterations = 2;
    int[] nIns = { 4, 2 };
    int[] nOuts = { 2, 3 };
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr).momentum(mu).momentumAfter(momentumAfter).iterations(iterations).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.NESTEROVS).build()).layer(1, new OutputLayer.Builder().nIn(nIns[1]).nOut(nOuts[1]).updater(org.deeplearning4j.nn.conf.Updater.NESTEROVS).build()).backprop(true).pretrain(false).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    Updater updater = UpdaterCreator.getUpdater(net);
    int stateSize = updater.stateSizeForLayer(net);
    updater.setStateViewArray(net, Nd4j.create(1, stateSize), true);
    String wKey, bKey;
    Gradient gradientExpected = new DefaultGradient();
    for (int k = 0; k < net.getnLayers(); k++) {
        wKey = String.valueOf(k) + "_" + DefaultParamInitializer.WEIGHT_KEY;
        gradientExpected.setGradientFor(wKey, Nd4j.ones(nIns[k], nOuts[k]));
        bKey = String.valueOf(k) + "_" + DefaultParamInitializer.BIAS_KEY;
        gradientExpected.setGradientFor(bKey, Nd4j.ones(1, nOuts[k]));
    }
    Gradient gradientMLN = new DefaultGradient();
    for (int j = 0; j < 2; j++) {
        wKey = String.valueOf(j) + "_" + DefaultParamInitializer.WEIGHT_KEY;
        gradientMLN.setGradientFor(wKey, Nd4j.ones(nIns[j], nOuts[j]));
        bKey = String.valueOf(j) + "_" + DefaultParamInitializer.BIAS_KEY;
        gradientMLN.setGradientFor(bKey, Nd4j.ones(1, nOuts[j]));
    }
    for (int i = 0; i < 2; i++) {
        updater.update(net, gradientMLN, i, 1);
        mu = testNesterovsComputation(gradientMLN, gradientExpected, lr, mu, momentumAfter, i);
        assertEquals(mu, net.getLayer(1).conf().getLayer().getMomentum(), 1e-4);
    }
}