Examples with DefaultGradient org.deeplearning4j.nn.gradient.DefaultGradient used on opensource projects

Example 26 with DefaultGradient

use of org.deeplearning4j.nn.gradient.DefaultGradient in project deeplearning4j by deeplearning4j.

the class AutoEncoderTest method testBackProp.

@Test
public void testBackProp() throws Exception {
    MnistDataFetcher fetcher = new MnistDataFetcher(true);
    //        LayerFactory layerFactory = LayerFactories.getFactory(new org.deeplearning4j.nn.conf.layers.AutoEncoder());
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().momentum(0.9f).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).iterations(100).learningRate(1e-1f).layer(new org.deeplearning4j.nn.conf.layers.AutoEncoder.Builder().nIn(784).nOut(600).corruptionLevel(0.6).lossFunction(LossFunctions.LossFunction.RECONSTRUCTION_CROSSENTROPY).build()).build();
    fetcher.fetch(100);
    DataSet d2 = fetcher.next();
    INDArray input = d2.getFeatureMatrix();
    int numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    AutoEncoder da = (AutoEncoder) conf.getLayer().instantiate(conf, null, 0, params, true);
    Gradient g = new DefaultGradient();
    g.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, da.decode(da.activate(input)).sub(input));
}

Example 27 with DefaultGradient

use of org.deeplearning4j.nn.gradient.DefaultGradient in project deeplearning4j by deeplearning4j.

the class TestComputationGraphNetwork method testGradientUpdate.

@Test
public void testGradientUpdate() {
    DataSetIterator iter = new IrisDataSetIterator(1, 1);
    Gradient expectedGradient = new DefaultGradient();
    expectedGradient.setGradientFor("first_W", Nd4j.ones(4, 5));
    expectedGradient.setGradientFor("first_b", Nd4j.ones(1, 5));
    expectedGradient.setGradientFor("output_W", Nd4j.ones(5, 3));
    expectedGradient.setGradientFor("output_b", Nd4j.ones(1, 3));
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input").addLayer("first", new DenseLayer.Builder().nIn(4).nOut(5).build(), "input").addLayer("output", new OutputLayer.Builder().nIn(5).nOut(3).build(), "first").setOutputs("output").pretrain(false).backprop(true).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.init();
    net.fit(iter.next());
    Gradient actualGradient = net.gradient;
    assertNotEquals(expectedGradient.getGradientFor("first_W"), actualGradient.getGradientFor("first_W"));
    net.update(expectedGradient);
    actualGradient = net.gradient;
    assertEquals(expectedGradient.getGradientFor("first_W"), actualGradient.getGradientFor("first_W"));
    // Update params with set
    net.setParam("first_W", Nd4j.ones(4, 5));
    net.setParam("first_b", Nd4j.ones(1, 5));
    net.setParam("output_W", Nd4j.ones(5, 3));
    net.setParam("output_b", Nd4j.ones(1, 3));
    INDArray actualParams = net.params();
    // Confirm params
    assertEquals(Nd4j.ones(1, 43), actualParams);
    net.update(expectedGradient);
    actualParams = net.params();
    assertEquals(Nd4j.ones(1, 43).addi(1), actualParams);
}

Example 28 with DefaultGradient

use of org.deeplearning4j.nn.gradient.DefaultGradient in project deeplearning4j by deeplearning4j.

the class SubsamplingLayerTest method createPrevGradient.

private Gradient createPrevGradient() {
    Gradient gradient = new DefaultGradient();
    INDArray pseudoGradients = Nd4j.ones(nExamples, nChannelsIn, inputHeight, inputWidth);
    gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, pseudoGradients);
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, pseudoGradients);
    return gradient;
}

Example 29 with DefaultGradient

use of org.deeplearning4j.nn.gradient.DefaultGradient in project deeplearning4j by deeplearning4j.

the class MultiLayerTest method testGradientUpdate.

@Test
public void testGradientUpdate() throws Exception {
    DataSetIterator iter = new IrisDataSetIterator(1, 1);
    Gradient expectedGradient = new DefaultGradient();
    expectedGradient.setGradientFor("0_W", Nd4j.ones(4, 5));
    expectedGradient.setGradientFor("0_b", Nd4j.ones(1, 5));
    expectedGradient.setGradientFor("1_W", Nd4j.ones(5, 3));
    expectedGradient.setGradientFor("1_b", Nd4j.ones(1, 3));
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(org.deeplearning4j.nn.conf.Updater.SGD).learningRate(1).activation(Activation.RELU).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().name("dnn1").nIn(4).nOut(5).build()).layer(1, new OutputLayer.Builder().name("output").nIn(5).nOut(3).activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build()).backprop(true).pretrain(false).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    net.fit(iter.next());
    // TODO validate actual layer gradientView - issue getting var out of BaseLayer w/o adding MLN getter that gets confused with local gradient vars
    Gradient actualGradient = net.gradient;
    assertNotEquals(expectedGradient.getGradientFor("0_W"), actualGradient.getGradientFor("0_W"));
    net.update(expectedGradient);
    actualGradient = net.gradient;
    assertEquals(expectedGradient.getGradientFor("0_W"), actualGradient.getGradientFor("0_W"));
    // Update params with set
    net.setParam("0_W", Nd4j.ones(4, 5));
    net.setParam("0_b", Nd4j.ones(1, 5));
    net.setParam("1_W", Nd4j.ones(5, 3));
    net.setParam("1_b", Nd4j.ones(1, 3));
    INDArray actualParams = net.params();
    // Confirm params
    assertEquals(expectedGradient.gradient(), actualParams);
    net.update(expectedGradient);
    actualParams = net.params();
    assertEquals(Nd4j.ones(1, 43).addi(1), actualParams);
}

Example 30 with DefaultGradient

use of org.deeplearning4j.nn.gradient.DefaultGradient in project deeplearning4j by deeplearning4j.

the class TestDecayPolicies method testMomentumScheduleSingleLayer.

@Test
public void testMomentumScheduleSingleLayer() {
    double lr = 1e-2;
    double mu = 0.6;
    Map<Integer, Double> momentumAfter = new HashMap<>();
    momentumAfter.put(1, 0.2);
    int iterations = 2;
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr).momentum(mu).momentumAfter(momentumAfter).iterations(iterations).layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).updater(org.deeplearning4j.nn.conf.Updater.NESTEROVS).build()).build();
    int numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true);
    Updater updater = UpdaterCreator.getUpdater(layer);
    int stateSize = updater.stateSizeForLayer(layer);
    updater.setStateViewArray(layer, Nd4j.create(1, stateSize), true);
    Gradient gradientExpected = new DefaultGradient();
    gradientExpected.setGradientFor(DefaultParamInitializer.WEIGHT_KEY, weightGradient.dup());
    gradientExpected.setGradientFor(DefaultParamInitializer.BIAS_KEY, biasGradient.dup());
    for (int i = 0; i < 2; i++) {
        updater.update(layer, gradientSingle, i, 1);
        mu = testNesterovsComputation(gradientSingle, gradientExpected, lr, mu, momentumAfter, i);
        assertEquals(mu, layer.conf().getLayer().getMomentum(), 1e-4);
    }
}