Examples with IterationListener org.deeplearning4j.optimize.api.IterationListener used on opensource projects

Example 1 with IterationListener

use of org.deeplearning4j.optimize.api.IterationListener in project deeplearning4j by deeplearning4j.

the class GravesBidirectionalLSTMTest method testConvergence.

@Test
@Ignore
public void testConvergence() {
    Nd4j.getRandom().setSeed(12345);
    final int state1Len = 100;
    final int state2Len = 30;
    //segment by signal mean
    //Data: has shape [miniBatchSize,nIn,timeSeriesLength];
    final INDArray sig1 = Nd4j.randn(new int[] { 1, 2, state1Len }).mul(0.1);
    final INDArray sig2 = Nd4j.randn(new int[] { 1, 2, state2Len }).mul(0.1).add(Nd4j.ones(new int[] { 1, 2, state2Len }).mul(1.0));
    INDArray sig = Nd4j.concat(2, sig1, sig2);
    INDArray labels = Nd4j.zeros(new int[] { 1, 2, state1Len + state2Len });
    for (int t = 0; t < state1Len; t++) {
        labels.putScalar(new int[] { 0, 0, t }, 1.0);
    }
    for (int t = state1Len; t < state1Len + state2Len; t++) {
        labels.putScalar(new int[] { 0, 1, t }, 1.0);
    }
    for (int i = 0; i < 3; i++) {
        sig = Nd4j.concat(2, sig, sig);
        labels = Nd4j.concat(2, labels, labels);
    }
    final DataSet ds = new DataSet(sig, labels);
    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(5).learningRate(0.1).rmsDecay(0.95).regularization(true).l2(0.001).updater(Updater.ADAGRAD).seed(12345).list().pretrain(false).layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(2).activation(Activation.TANH).build()).backprop(true).build();
    final MultiLayerNetwork net = new MultiLayerNetwork(conf);
    final IterationListener scoreSaver = new IterationListener() {

        @Override
        public boolean invoked() {
            return false;
        }

        @Override
        public void invoke() {
        }

        @Override
        public void iterationDone(Model model, int iteration) {
            score = model.score();
        }
    };
    net.setListeners(scoreSaver, new ScoreIterationListener(1));
    double oldScore = Double.POSITIVE_INFINITY;
    net.init();
    for (int iEpoch = 0; iEpoch < 3; iEpoch++) {
        net.fit(ds);
        System.out.print(String.format("score is %f%n", score));
        assertTrue(!Double.isNaN(score));
        assertTrue(score < 0.9 * oldScore);
        oldScore = score;
        final INDArray output = net.output(ds.getFeatureMatrix());
        Evaluation evaluation = new Evaluation();
        evaluation.evalTimeSeries(ds.getLabels(), output);
        System.out.print(evaluation.stats() + "\n");
    }
}

Example 2 with IterationListener

use of org.deeplearning4j.optimize.api.IterationListener in project deeplearning4j by deeplearning4j.

the class OutputLayerTest method testSetParams.

@Test
public void testSetParams() {
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).iterations(100).learningRate(1e-1).layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.ZERO).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).build();
    int numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    OutputLayer l = (OutputLayer) conf.getLayer().instantiate(conf, Collections.<IterationListener>singletonList(new ScoreIterationListener(1)), 0, params, true);
    params = l.params();
    l.setParams(params);
    assertEquals(params, l.params());
}

Example 3 with IterationListener

use of org.deeplearning4j.optimize.api.IterationListener in project deeplearning4j by deeplearning4j.

the class ParallelWrapper method setListeners.

/**
     * Set the listeners, along with a StatsStorageRouter that the results will be shuffled to (in the case of any listeners
     * that implement the {@link RoutingIterationListener} interface)
     *
     * @param statsStorage Stats storage router to place the results into
     * @param listeners    Listeners to set
     */
public void setListeners(StatsStorageRouter statsStorage, Collection<? extends IterationListener> listeners) {
    //Check if we have any RoutingIterationListener instances that need a StatsStorage implementation...
    StatsStorageRouterProvider routerProvider = null;
    if (listeners != null) {
        for (IterationListener l : listeners) {
            if (l instanceof RoutingIterationListener) {
                RoutingIterationListener rl = (RoutingIterationListener) l;
                if (rl.getStorageRouter() == null) {
                    log.warn("RoutingIterationListener provided without providing any StatsStorage instance. Iterator may not function without one. Listener: {}", l);
                } else if (!(rl.getStorageRouter() instanceof Serializable)) {
                    //Spark would throw a (probably cryptic) serialization exception later anyway...
                    throw new IllegalStateException("RoutingIterationListener provided with non-serializable storage router");
                }
            }
        }
    }
    this.storageRouter = statsStorage;
    this.listeners.addAll(listeners);
}

Example 4 with IterationListener

use of org.deeplearning4j.optimize.api.IterationListener in project deeplearning4j by deeplearning4j.

the class MultiLayerTest method testSelectedActivations.

/**
     *  This test intended only to test activateSelectedLayers method, it does not involves fully-working AutoEncoder.
     */
@Test
public void testSelectedActivations() {
    // Train DeepAutoEncoder on very limited trainset
    final int numRows = 28;
    final int numColumns = 28;
    int seed = 123;
    int numSamples = 3;
    int iterations = 1;
    int listenerFreq = iterations / 5;
    log.info("Load data....");
    float[][] trainingData = new float[numSamples][numColumns * numRows];
    Arrays.fill(trainingData[0], 0.95f);
    Arrays.fill(trainingData[1], 0.5f);
    Arrays.fill(trainingData[2], 0.05f);
    log.info("Build model....");
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list().layer(0, new RBM.Builder().nIn(numRows * numColumns).nOut(1000).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(1, new RBM.Builder().nIn(1000).nOut(500).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(2, new RBM.Builder().nIn(500).nOut(250).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(3, new RBM.Builder().nIn(250).nOut(100).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(4, new RBM.Builder().nIn(100).nOut(30).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(5, new RBM.Builder().nIn(30).nOut(100).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(6, new RBM.Builder().nIn(100).nOut(250).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(7, new RBM.Builder().nIn(250).nOut(500).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(8, new RBM.Builder().nIn(500).nOut(1000).lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).build()).layer(9, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(1000).nOut(numRows * numColumns).build()).pretrain(true).backprop(true).build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    model.setListeners(Arrays.asList((IterationListener) new ScoreIterationListener(listenerFreq)));
    log.info("Train model....");
    int cnt = 0;
    while (cnt < numSamples) {
        INDArray input = Nd4j.create(trainingData[cnt]);
        model.fit(new DataSet(input, input));
        cnt++;
    }
    // Make two separate selective calls
    log.info("Testing full cycle...");
    List<INDArray> comparableResult = model.feedForward(Nd4j.create(trainingData[0]));
    INDArray encodeResult = model.activateSelectedLayers(0, 4, Nd4j.create(trainingData[0]));
    log.info("Compare feedForward results with selectedActivation");
    assertEquals(comparableResult.get(5), encodeResult);
    INDArray decodeResults = model.activateSelectedLayers(5, 9, encodeResult);
    log.info("Decode results: " + decodeResults.columns() + " " + decodeResults);
    log.info("Comparable  results: " + comparableResult.get(10).columns() + " " + comparableResult.get(10));
    assertEquals(comparableResult.get(10), decodeResults);
}

Example 5 with IterationListener

use of org.deeplearning4j.optimize.api.IterationListener in project deeplearning4j by deeplearning4j.

the class BackTrackLineSearchTest method testBackTrackLineLBFGS.

@Test
public void testBackTrackLineLBFGS() {
    OptimizationAlgorithm optimizer = OptimizationAlgorithm.LBFGS;
    DataSet data = irisIter.next();
    data.normalizeZeroMeanZeroUnitVariance();
    MultiLayerNetwork network = new MultiLayerNetwork(getIrisMultiLayerConfig(Activation.RELU, 5, optimizer));
    network.init();
    IterationListener listener = new ScoreIterationListener(1);
    network.setListeners(Collections.singletonList(listener));
    double oldScore = network.score(data);
    network.fit(data.getFeatureMatrix(), data.getLabels());
    double score = network.score();
    assertTrue(score < oldScore);
}