Examples with InMemoryModelSaver org.deeplearning4j.earlystopping.saver.InMemoryModelSaver used on opensource projects

Example 16 with InMemoryModelSaver

use of org.deeplearning4j.earlystopping.saver.InMemoryModelSaver in project deeplearning4j by deeplearning4j.

the class TestEarlyStopping method testMinImprovementNEpochsTermination.

@Test
public void testMinImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve more than minImprovement for 5 consecutive epochs
    //Simulate this by setting LR = 0.0
    Random rng = new Random(123);
    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).iterations(10).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(0.0).updater(Updater.NESTEROVS).momentum(0.9).list().layer(0, new DenseLayer.Builder().nIn(1).nOut(20).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER).nIn(20).nOut(1).build()).pretrain(false).backprop(true).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));
    int nSamples = 100;
    //Generate the training data
    INDArray x = Nd4j.linspace(-10, 10, nSamples).reshape(nSamples, 1);
    INDArray y = Nd4j.getExecutioner().execAndReturn(new Sin(x.dup()));
    DataSet allData = new DataSet(x, y);
    List<DataSet> list = allData.asList();
    Collections.shuffle(list, rng);
    DataSetIterator training = new ListDataSetIterator(list, nSamples);
    double minImprovement = 0.0009;
    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf = new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>().epochTerminationConditions(new MaxEpochsTerminationCondition(1000), //Go on for max 5 epochs without any improvements that are greater than minImprovement
    new ScoreImprovementEpochTerminationCondition(5, minImprovement)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(3, TimeUnit.MINUTES)).scoreCalculator(new DataSetLossCalculator(training, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConf, net, training);
    EarlyStoppingResult result = trainer.fit();
    assertEquals(6, result.getTotalEpochs());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5, minImprovement).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}

Example 17 with InMemoryModelSaver

use of org.deeplearning4j.earlystopping.saver.InMemoryModelSaver in project deeplearning4j by deeplearning4j.

the class TestEarlyStopping method testEarlyStoppingIris.

@Test
public void testEarlyStoppingIris() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).weightInit(WeightInit.XAVIER).list().layer(0, new OutputLayer.Builder().nIn(4).nOut(3).lossFunction(LossFunctions.LossFunction.MCXENT).build()).pretrain(false).backprop(true).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));
    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf = new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>().epochTerminationConditions(new MaxEpochsTerminationCondition(5)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES)).scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer<MultiLayerNetwork> trainer = new EarlyStoppingTrainer(esConf, net, irisIter);
    EarlyStoppingResult<MultiLayerNetwork> result = trainer.fit();
    System.out.println(result);
    assertEquals(5, result.getTotalEpochs());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    Map<Integer, Double> scoreVsIter = result.getScoreVsEpoch();
    assertEquals(5, scoreVsIter.size());
    String expDetails = esConf.getEpochTerminationConditions().get(0).toString();
    assertEquals(expDetails, result.getTerminationDetails());
    MultiLayerNetwork out = result.getBestModel();
    assertNotNull(out);
    //Check that best score actually matches (returned model vs. manually calculated score)
    MultiLayerNetwork bestNetwork = result.getBestModel();
    irisIter.reset();
    double score = bestNetwork.score(irisIter.next());
    assertEquals(result.getBestModelScore(), score, 1e-2);
}

Example 18 with InMemoryModelSaver

use of org.deeplearning4j.earlystopping.saver.InMemoryModelSaver in project deeplearning4j by deeplearning4j.

the class TestEarlyStoppingCompGraph method testNoImprovementNEpochsTermination.

@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).learningRate(0.0).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in").setOutputs("0").pretrain(false).backprop(true).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));
    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>().epochTerminationConditions(new MaxEpochsTerminationCondition(100), new ScoreImprovementEpochTerminationCondition(5)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(3, TimeUnit.SECONDS), //Initial score is ~2.5
    new MaxScoreIterationTerminationCondition(7.5)).scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();
    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertEquals(6, result.getTotalEpochs());
    assertEquals(0, result.getBestModelEpoch());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}

Example 19 with InMemoryModelSaver

use of org.deeplearning4j.earlystopping.saver.InMemoryModelSaver in project deeplearning4j by deeplearning4j.

the class TestEarlyStoppingCompGraph method testEarlyStoppingIris.

@Test
public void testEarlyStoppingIris() {
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in").setOutputs("0").pretrain(false).backprop(true).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));
    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>().epochTerminationConditions(new MaxEpochsTerminationCondition(5)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES)).scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer<ComputationGraph> trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult<ComputationGraph> result = trainer.fit();
    System.out.println(result);
    assertEquals(5, result.getTotalEpochs());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    Map<Integer, Double> scoreVsIter = result.getScoreVsEpoch();
    assertEquals(5, scoreVsIter.size());
    String expDetails = esConf.getEpochTerminationConditions().get(0).toString();
    assertEquals(expDetails, result.getTerminationDetails());
    ComputationGraph out = result.getBestModel();
    assertNotNull(out);
    //Check that best score actually matches (returned model vs. manually calculated score)
    ComputationGraph bestNetwork = result.getBestModel();
    irisIter.reset();
    double score = bestNetwork.score(irisIter.next());
    assertEquals(result.getBestModelScore(), score, 1e-2);
}

Example 20 with InMemoryModelSaver

use of org.deeplearning4j.earlystopping.saver.InMemoryModelSaver in project deeplearning4j by deeplearning4j.

the class TestEarlyStoppingCompGraph method testBadTuning.

@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).learningRate(//Intentionally huge LR
    5.0).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in").setOutputs("0").pretrain(false).backprop(true).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));
    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>().epochTerminationConditions(new MaxEpochsTerminationCondition(5000)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES), //Initial score is ~2.5
    new MaxScoreIterationTerminationCondition(10)).scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();
    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition, result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(10).toString();
    assertEquals(expDetails, result.getTerminationDetails());
    assertEquals(0, result.getBestModelEpoch());
    assertNotNull(result.getBestModel());
}