use of org.deeplearning4j.optimize.solvers.StochasticGradientDescent in project deeplearning4j by deeplearning4j.
the class NeuralNetConfigurationTest method testL1L2ByParam.
@Test
public void testL1L2ByParam() {
double l1 = 0.01;
double l2 = 0.07;
int[] nIns = { 4, 3, 3 };
int[] nOuts = { 3, 3, 3 };
int oldScore = 1;
int newScore = 1;
int iteration = 3;
INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]);
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(8).regularization(true).l1(l1).l2(l2).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).l2(0.5).build()).layer(2, new OutputLayer.Builder().nIn(nIns[2]).nOut(nOuts[2]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).backprop(true).pretrain(false).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(), new NegativeDefaultStepFunction(), null, net);
opt.checkTerminalConditions(gradientW, oldScore, newScore, iteration);
assertEquals(l1, net.getLayer(0).conf().getL1ByParam("W"), 1e-4);
assertEquals(0.0, net.getLayer(0).conf().getL1ByParam("b"), 0.0);
assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("beta"), 0.0);
assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("gamma"), 0.0);
assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("mean"), 0.0);
assertEquals(0.0, net.getLayer(1).conf().getL2ByParam("var"), 0.0);
assertEquals(l2, net.getLayer(2).conf().getL2ByParam("W"), 1e-4);
assertEquals(0.0, net.getLayer(2).conf().getL2ByParam("b"), 0.0);
}
use of org.deeplearning4j.optimize.solvers.StochasticGradientDescent in project deeplearning4j by deeplearning4j.
the class NeuralNetConfigurationTest method testLearningRateByParam.
@Test
public void testLearningRateByParam() {
double lr = 0.01;
double biasLr = 0.02;
int[] nIns = { 4, 3, 3 };
int[] nOuts = { 3, 3, 3 };
int oldScore = 1;
int newScore = 1;
int iteration = 3;
INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]);
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(0.3).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.SGD).learningRate(lr).biasLearningRate(biasLr).build()).layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).learningRate(0.7).build()).layer(2, new OutputLayer.Builder().nIn(nIns[2]).nOut(nOuts[2]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).backprop(true).pretrain(false).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(), new NegativeDefaultStepFunction(), null, net);
opt.checkTerminalConditions(gradientW, oldScore, newScore, iteration);
assertEquals(lr, net.getLayer(0).conf().getLearningRateByParam("W"), 1e-4);
assertEquals(biasLr, net.getLayer(0).conf().getLearningRateByParam("b"), 1e-4);
assertEquals(0.7, net.getLayer(1).conf().getLearningRateByParam("gamma"), 1e-4);
//From global LR
assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("W"), 1e-4);
//From global LR
assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("b"), 1e-4);
}
use of org.deeplearning4j.optimize.solvers.StochasticGradientDescent in project deeplearning4j by deeplearning4j.
the class TestDecayPolicies method testLearningRateScoreDecay.
@Test
public void testLearningRateScoreDecay() {
double lr = 0.01;
double lrScoreDecay = 0.10;
int[] nIns = { 4, 2 };
int[] nOuts = { 2, 3 };
int oldScore = 1;
int newScore = 1;
int iteration = 3;
INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]);
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr).learningRateDecayPolicy(LearningRatePolicy.Score).lrPolicyDecayRate(lrScoreDecay).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).layer(1, new OutputLayer.Builder().nIn(nIns[1]).nOut(nOuts[1]).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).backprop(true).pretrain(false).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(), new NegativeDefaultStepFunction(), null, net);
opt.checkTerminalConditions(gradientW, oldScore, newScore, iteration);
assertEquals(lrScoreDecay, net.getLayer(0).conf().getLrPolicyDecayRate(), 1e-4);
assertEquals(lr * (lrScoreDecay + Nd4j.EPS_THRESHOLD), net.getLayer(0).conf().getLearningRateByParam("W"), 1e-4);
}
Aggregations