Examples with Evaluation org.deeplearning4j.eval.Evaluation used on opensource projects

Example 1 with Evaluation

use of org.deeplearning4j.eval.Evaluation in project deeplearning4j by deeplearning4j.

the class DataSetIteratorTest method testLfwModel.

@Test
public void testLfwModel() throws Exception {
    final int numRows = 28;
    final int numColumns = 28;
    int numChannels = 3;
    int outputNum = LFWLoader.SUB_NUM_LABELS;
    int numSamples = 4;
    int batchSize = 2;
    int iterations = 1;
    int seed = 123;
    int listenerFreq = iterations;
    LFWDataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples, new int[] { numRows, numColumns, numChannels }, outputNum, true, true, 1.0, new Random(seed));
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations).gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list().layer(0, new ConvolutionLayer.Builder(10, 10).nIn(numChannels).nOut(6).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).stride(1, 1).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(numRows, numColumns, numChannels)).backprop(true).pretrain(false);
    MultiLayerNetwork model = new MultiLayerNetwork(builder.build());
    model.init();
    model.setListeners(new ScoreIterationListener(listenerFreq));
    model.fit(lfw.next());
    DataSet dataTest = lfw.next();
    INDArray output = model.output(dataTest.getFeatureMatrix());
    Evaluation eval = new Evaluation(outputNum);
    eval.eval(dataTest.getLabels(), output);
    System.out.println(eval.stats());
}

Example 2 with Evaluation

use of org.deeplearning4j.eval.Evaluation in project deeplearning4j by deeplearning4j.

the class ParallelWrapperTest method testParallelWrapperRun.

@Test
public void testParallelWrapperRun() throws Exception {
    int nChannels = 1;
    int outputNum = 10;
    // for GPU you usually want to have higher batchSize
    int batchSize = 128;
    int nEpochs = 10;
    int iterations = 1;
    int seed = 123;
    log.info("Load data....");
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345);
    DataSetIterator mnistTest = new MnistDataSetIterator(batchSize, false, 12345);
    log.info("Build model....");
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations).regularization(true).l2(0.0005).learningRate(//.biasLearningRate(0.02)
    0.01).weightInit(WeightInit.XAVIER).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.NESTEROVS).momentum(0.9).list().layer(0, new ConvolutionLayer.Builder(5, 5).nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(2, new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()).layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()).layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).activation(Activation.SOFTMAX).build()).backprop(true).pretrain(false);
    // The builder needs the dimensions of the image along with the number of channels. these are 28x28 images in one channel
    new ConvolutionLayerSetup(builder, 28, 28, 1);
    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model).prefetchBuffer(24).workers(2).averagingFrequency(3).reportScoreAfterAveraging(true).useLegacyAveraging(true).build();
    log.info("Train model....");
    model.setListeners(new ScoreIterationListener(100));
    long timeX = System.currentTimeMillis();
    for (int i = 0; i < nEpochs; i++) {
        long time1 = System.currentTimeMillis();
        // Please note: we're feeding ParallelWrapper with iterator, not model directly
        //            wrapper.fit(mnistMultiEpochIterator);
        wrapper.fit(mnistTrain);
        long time2 = System.currentTimeMillis();
        log.info("*** Completed epoch {}, time: {} ***", i, (time2 - time1));
    }
    long timeY = System.currentTimeMillis();
    log.info("*** Training complete, time: {} ***", (timeY - timeX));
    log.info("Evaluate model....");
    Evaluation eval = new Evaluation(outputNum);
    while (mnistTest.hasNext()) {
        DataSet ds = mnistTest.next();
        INDArray output = model.output(ds.getFeatureMatrix(), false);
        eval.eval(ds.getLabels(), output);
    }
    log.info(eval.stats());
    mnistTest.reset();
    log.info("****************Example finished********************");
    wrapper.shutdown();
}

Example 3 with Evaluation

use of org.deeplearning4j.eval.Evaluation in project deeplearning4j by deeplearning4j.

the class ComputationGraph method evaluate.

/**
     * Evaluate the network (for classification) on the provided data set, with top N accuracy in addition to standard accuracy.
     * For 'standard' accuracy evaluation only, use topN = 1
     *
     * @param iterator   Iterator (data) to evaluate on
     * @param labelsList List of labels. May be null.
     * @param topN       N value for top N accuracy evaluation
     * @return Evaluation object, summarizing the results of the evaluation on the provided DataSetIterator
     */
public Evaluation evaluate(DataSetIterator iterator, List<String> labelsList, int topN) {
    if (layers == null || !(getOutputLayer(0) instanceof IOutputLayer)) {
        throw new IllegalStateException("Cannot evaluate network with no output layer");
    }
    if (labelsList == null)
        labelsList = iterator.getLabels();
    Evaluation e = new Evaluation(labelsList, topN);
    while (iterator.hasNext()) {
        org.nd4j.linalg.dataset.DataSet next = iterator.next();
        if (next.getFeatureMatrix() == null || next.getLabels() == null)
            break;
        INDArray features = next.getFeatures();
        INDArray labels = next.getLabels();
        INDArray[] out;
        out = output(false, features);
        if (labels.rank() == 3)
            e.evalTimeSeries(labels, out[0]);
        else
            e.eval(labels, out[0]);
    }
    return e;
}

Example 4 with Evaluation

use of org.deeplearning4j.eval.Evaluation in project deeplearning4j by deeplearning4j.

the class GravesBidirectionalLSTMTest method testConvergence.

@Test
@Ignore
public void testConvergence() {
    Nd4j.getRandom().setSeed(12345);
    final int state1Len = 100;
    final int state2Len = 30;
    //segment by signal mean
    //Data: has shape [miniBatchSize,nIn,timeSeriesLength];
    final INDArray sig1 = Nd4j.randn(new int[] { 1, 2, state1Len }).mul(0.1);
    final INDArray sig2 = Nd4j.randn(new int[] { 1, 2, state2Len }).mul(0.1).add(Nd4j.ones(new int[] { 1, 2, state2Len }).mul(1.0));
    INDArray sig = Nd4j.concat(2, sig1, sig2);
    INDArray labels = Nd4j.zeros(new int[] { 1, 2, state1Len + state2Len });
    for (int t = 0; t < state1Len; t++) {
        labels.putScalar(new int[] { 0, 0, t }, 1.0);
    }
    for (int t = state1Len; t < state1Len + state2Len; t++) {
        labels.putScalar(new int[] { 0, 1, t }, 1.0);
    }
    for (int i = 0; i < 3; i++) {
        sig = Nd4j.concat(2, sig, sig);
        labels = Nd4j.concat(2, labels, labels);
    }
    final DataSet ds = new DataSet(sig, labels);
    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(5).learningRate(0.1).rmsDecay(0.95).regularization(true).l2(0.001).updater(Updater.ADAGRAD).seed(12345).list().pretrain(false).layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(2).activation(Activation.TANH).build()).backprop(true).build();
    final MultiLayerNetwork net = new MultiLayerNetwork(conf);
    final IterationListener scoreSaver = new IterationListener() {

        @Override
        public boolean invoked() {
            return false;
        }

        @Override
        public void invoke() {
        }

        @Override
        public void iterationDone(Model model, int iteration) {
            score = model.score();
        }
    };
    net.setListeners(scoreSaver, new ScoreIterationListener(1));
    double oldScore = Double.POSITIVE_INFINITY;
    net.init();
    for (int iEpoch = 0; iEpoch < 3; iEpoch++) {
        net.fit(ds);
        System.out.print(String.format("score is %f%n", score));
        assertTrue(!Double.isNaN(score));
        assertTrue(score < 0.9 * oldScore);
        oldScore = score;
        final INDArray output = net.output(ds.getFeatureMatrix());
        Evaluation evaluation = new Evaluation();
        evaluation.evalTimeSeries(ds.getLabels(), output);
        System.out.print(evaluation.stats() + "\n");
    }
}

Example 5 with Evaluation

use of org.deeplearning4j.eval.Evaluation in project deeplearning4j by deeplearning4j.

the class ConvolutionLayerTest method testCNNMLNPretrain.

//////////////////////////////////////////////////////////////////////////////////
@Test
public void testCNNMLNPretrain() throws Exception {
    // Note CNN does not do pretrain
    int numSamples = 10;
    int batchSize = 10;
    DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true);
    MultiLayerNetwork model = getCNNMLNConfig(false, true);
    model.fit(mnistIter);
    mnistIter.reset();
    MultiLayerNetwork model2 = getCNNMLNConfig(false, true);
    model2.fit(mnistIter);
    mnistIter.reset();
    DataSet test = mnistIter.next();
    Evaluation eval = new Evaluation();
    INDArray output = model.output(test.getFeatureMatrix());
    eval.eval(test.getLabels(), output);
    double f1Score = eval.f1();
    Evaluation eval2 = new Evaluation();
    INDArray output2 = model2.output(test.getFeatureMatrix());
    eval2.eval(test.getLabels(), output2);
    double f1Score2 = eval2.f1();
    assertEquals(f1Score, f1Score2, 1e-4);
}