Examples with DataSetIterator org.nd4j.linalg.dataset.api.iterator.DataSetIterator used on opensource projects

Example 91 with DataSetIterator

use of org.nd4j.linalg.dataset.api.iterator.DataSetIterator in project deeplearning4j by deeplearning4j.

the class DataSetIteratorTest method testIteratorDataSetIteratorCombining.

@Test
public void testIteratorDataSetIteratorCombining() {
    //Test combining of a bunch of small (size 1) data sets together
    int batchSize = 3;
    int numBatches = 4;
    int featureSize = 5;
    int labelSize = 6;
    Nd4j.getRandom().setSeed(12345);
    List<DataSet> orig = new ArrayList<>();
    for (int i = 0; i < batchSize * numBatches; i++) {
        INDArray features = Nd4j.rand(1, featureSize);
        INDArray labels = Nd4j.rand(1, labelSize);
        orig.add(new DataSet(features, labels));
    }
    DataSetIterator iter = new IteratorDataSetIterator(orig.iterator(), batchSize);
    int count = 0;
    while (iter.hasNext()) {
        DataSet ds = iter.next();
        assertArrayEquals(new int[] { batchSize, featureSize }, ds.getFeatureMatrix().shape());
        assertArrayEquals(new int[] { batchSize, labelSize }, ds.getLabels().shape());
        List<INDArray> fList = new ArrayList<>();
        List<INDArray> lList = new ArrayList<>();
        for (int i = 0; i < batchSize; i++) {
            DataSet dsOrig = orig.get(count * batchSize + i);
            fList.add(dsOrig.getFeatureMatrix());
            lList.add(dsOrig.getLabels());
        }
        INDArray fExp = Nd4j.vstack(fList);
        INDArray lExp = Nd4j.vstack(lList);
        assertEquals(fExp, ds.getFeatureMatrix());
        assertEquals(lExp, ds.getLabels());
        count++;
    }
    assertEquals(count, numBatches);
}

Example 92 with DataSetIterator

use of org.nd4j.linalg.dataset.api.iterator.DataSetIterator in project deeplearning4j by deeplearning4j.

the class DataSetIteratorTest method testBatchSizeOfOneMnist.

@Test
public void testBatchSizeOfOneMnist() throws Exception {
    //MNIST:
    DataSetIterator mnist = new MnistDataSetIterator(1, 5);
    int mnistC = 0;
    while (mnist.hasNext()) {
        mnistC++;
        DataSet ds = mnist.next();
        assertTrue(ds.getLabels().sum(Integer.MAX_VALUE).getDouble(0) == 1.0);
    }
    assertEquals(5, mnistC);
}

Example 93 with DataSetIterator

use of org.nd4j.linalg.dataset.api.iterator.DataSetIterator in project deeplearning4j by deeplearning4j.

the class MultipleEpochsIteratorTest method testLoadBatchDataSet.

@Test
public void testLoadBatchDataSet() throws Exception {
    int epochs = 2;
    RecordReader rr = new CSVRecordReader();
    rr.initialize(new FileSplit(new ClassPathResource("iris.txt").getFile()));
    DataSetIterator iter = new RecordReaderDataSetIterator(rr, 150);
    DataSet ds = iter.next(20);
    MultipleEpochsIterator multiIter = new MultipleEpochsIterator(epochs, ds);
    while (multiIter.hasNext()) {
        DataSet path = multiIter.next(10);
        assertEquals(path.numExamples(), 10, 0.0);
        assertFalse(path == null);
    }
    assertEquals(epochs, multiIter.epochs);
}

Example 94 with DataSetIterator

use of org.nd4j.linalg.dataset.api.iterator.DataSetIterator in project deeplearning4j by deeplearning4j.

the class SamplingTest method testSample.

@Test
public void testSample() throws Exception {
    DataSetIterator iter = new MnistDataSetIterator(10, 10);
    //batch size and total
    DataSetIterator sampling = new SamplingDataSetIterator(iter.next(), 10, 10);
    assertEquals(sampling.next().numExamples(), 10);
}

Example 95 with DataSetIterator

use of org.nd4j.linalg.dataset.api.iterator.DataSetIterator in project deeplearning4j by deeplearning4j.

the class GradientCheckTests method testAutoEncoder.

@Test
public void testAutoEncoder() {
    //As above (testGradientMLP2LayerIrisSimple()) but with L2, L1, and both L2/L1 applied
    //Need to run gradient through updater, so that L2 can be applied
    String[] activFns = { "sigmoid", "tanh" };
    //If true: run some backprop steps first
    boolean[] characteristic = { false, true };
    LossFunction[] lossFunctions = { LossFunction.MCXENT, LossFunction.MSE };
    //i.e., lossFunctions[i] used with outputActivations[i] here
    String[] outputActivations = { "softmax", "tanh" };
    DataNormalization scaler = new NormalizerMinMaxScaler();
    DataSetIterator iter = new IrisDataSetIterator(150, 150);
    scaler.fit(iter);
    iter.setPreProcessor(scaler);
    DataSet ds = iter.next();
    INDArray input = ds.getFeatureMatrix();
    INDArray labels = ds.getLabels();
    NormalizerStandardize norm = new NormalizerStandardize();
    norm.fit(ds);
    norm.transform(ds);
    double[] l2vals = { 0.2, 0.0, 0.2 };
    //i.e., use l2vals[i] with l1vals[i]
    double[] l1vals = { 0.0, 0.3, 0.3 };
    for (String afn : activFns) {
        for (boolean doLearningFirst : characteristic) {
            for (int i = 0; i < lossFunctions.length; i++) {
                for (int k = 0; k < l2vals.length; k++) {
                    LossFunction lf = lossFunctions[i];
                    String outputActivation = outputActivations[i];
                    double l2 = l2vals[k];
                    double l1 = l1vals[k];
                    Nd4j.getRandom().setSeed(12345);
                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).learningRate(1.0).l2(l2).l1(l1).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.SGD).list().layer(0, new AutoEncoder.Builder().nIn(4).nOut(3).activation(afn).build()).layer(1, new OutputLayer.Builder(lf).nIn(3).nOut(3).activation(outputActivation).build()).pretrain(true).backprop(true).build();
                    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                    mln.init();
                    if (doLearningFirst) {
                        //Run a number of iterations of learning
                        mln.setInput(ds.getFeatures());
                        mln.setLabels(ds.getLabels());
                        mln.computeGradientAndScore();
                        double scoreBefore = mln.score();
                        for (int j = 0; j < 10; j++) mln.fit(ds);
                        mln.computeGradientAndScore();
                        double scoreAfter = mln.score();
                        //Can't test in 'characteristic mode of operation' if not learning
                        String msg = "testGradMLP2LayerIrisSimple() - score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l2=" + l2 + ", l1=" + l1 + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")";
                        assertTrue(msg, scoreAfter < scoreBefore);
                    }
                    if (PRINT_RESULTS) {
                        System.out.println("testGradientMLP2LayerIrisSimpleRandom() - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l2=" + l2 + ", l1=" + l1);
                        for (int j = 0; j < mln.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                    }
                    boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
                    String msg = "testGradMLP2LayerIrisSimple() - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l2=" + l2 + ", l1=" + l1;
                    assertTrue(msg, gradOK);
                }
            }
        }
    }
}