Search in sources :

Example 1 with CollectionSequenceRecordReader

use of org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader in project deeplearning4j by deeplearning4j.

the class RecordReaderDataSetiteratorTest method testSeqRRDSIMultipleArrayWritablesOneReader.

@Test
public void testSeqRRDSIMultipleArrayWritablesOneReader() {
    //Input with multiple array writables:
    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 100, 200, 300 })), new IntWritable(0)));
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 })), new NDArrayWritable(Nd4j.create(new double[] { 400, 500, 600 })), new IntWritable(1)));
    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 })), new NDArrayWritable(Nd4j.create(new double[] { 700, 800, 900 })), new IntWritable(2)));
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 })), new NDArrayWritable(Nd4j.create(new double[] { 1000, 1100, 1200 })), new IntWritable(3)));
    SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2));
    SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, 4, 2, false);
    DataSet ds = iter.next();
    //2 examples, 6 values per time step, 2 time steps
    INDArray expFeatures = Nd4j.create(2, 6, 2);
    expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 }, { 100, 400 }, { 200, 500 }, { 300, 600 } }));
    expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 }, { 700, 1000 }, { 800, 1100 }, { 900, 1200 } }));
    INDArray expLabels = Nd4j.create(2, 4, 2);
    expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 0 }, { 0, 1 }, { 0, 0 }, { 0, 0 } }));
    expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 0, 0 }, { 0, 0 }, { 1, 0 }, { 0, 1 } }));
    assertEquals(expFeatures, ds.getFeatureMatrix());
    assertEquals(expLabels, ds.getLabels());
}
Also used : NDArrayWritable(org.datavec.common.data.NDArrayWritable) CSVSequenceRecordReader(org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader) SequenceRecordReader(org.datavec.api.records.reader.SequenceRecordReader) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) INDArray(org.nd4j.linalg.api.ndarray.INDArray) DataSet(org.nd4j.linalg.dataset.DataSet) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) IntWritable(org.datavec.api.writable.IntWritable) NDArrayWritable(org.datavec.common.data.NDArrayWritable) DoubleWritable(org.datavec.api.writable.DoubleWritable) Writable(org.datavec.api.writable.Writable) IntWritable(org.datavec.api.writable.IntWritable) Test(org.junit.Test)

Example 2 with CollectionSequenceRecordReader

use of org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader in project deeplearning4j by deeplearning4j.

the class TestRecordReaders method testClassIndexOutsideOfRangeRRMDSI_MultipleReaders.

@Test
public void testClassIndexOutsideOfRangeRRMDSI_MultipleReaders() {
    Collection<Collection<Collection<Writable>>> c1 = new ArrayList<>();
    Collection<Collection<Writable>> seq1 = new ArrayList<>();
    seq1.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    seq1.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    c1.add(seq1);
    Collection<Collection<Writable>> seq2 = new ArrayList<>();
    seq2.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    seq2.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    c1.add(seq2);
    Collection<Collection<Collection<Writable>>> c2 = new ArrayList<>();
    Collection<Collection<Writable>> seq1a = new ArrayList<>();
    seq1a.add(Arrays.<Writable>asList(new IntWritable(0)));
    seq1a.add(Arrays.<Writable>asList(new IntWritable(1)));
    c2.add(seq1a);
    Collection<Collection<Writable>> seq2a = new ArrayList<>();
    seq2a.add(Arrays.<Writable>asList(new IntWritable(0)));
    seq2a.add(Arrays.<Writable>asList(new IntWritable(2)));
    c2.add(seq2a);
    CollectionSequenceRecordReader csrr = new CollectionSequenceRecordReader(c1);
    CollectionSequenceRecordReader csrrLabels = new CollectionSequenceRecordReader(c2);
    DataSetIterator dsi = new SequenceRecordReaderDataSetIterator(csrr, csrrLabels, 2, 2);
    try {
        DataSet ds = dsi.next();
        fail("Expected exception");
    } catch (DL4JException e) {
        System.out.println("testClassIndexOutsideOfRangeRRMDSI_MultipleReaders(): " + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}
Also used : SequenceRecordReaderDataSetIterator(org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator) DataSet(org.nd4j.linalg.dataset.api.DataSet) ArrayList(java.util.ArrayList) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) IntWritable(org.datavec.api.writable.IntWritable) Writable(org.datavec.api.writable.Writable) DoubleWritable(org.datavec.api.writable.DoubleWritable) DoubleWritable(org.datavec.api.writable.DoubleWritable) DL4JException(org.deeplearning4j.exception.DL4JException) DL4JException(org.deeplearning4j.exception.DL4JException) Collection(java.util.Collection) IntWritable(org.datavec.api.writable.IntWritable) SequenceRecordReaderDataSetIterator(org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator) DataSetIterator(org.nd4j.linalg.dataset.api.iterator.DataSetIterator) RecordReaderDataSetIterator(org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator) Test(org.junit.Test)

Example 3 with CollectionSequenceRecordReader

use of org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader in project deeplearning4j by deeplearning4j.

the class RecordReaderDataSetiteratorTest method testSeqRRDSIArrayWritableOneReaderRegression.

@Test
public void testSeqRRDSIArrayWritableOneReaderRegression() {
    //Regression, where the output is an array writable
    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 100, 200, 300 }))));
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 })), new NDArrayWritable(Nd4j.create(new double[] { 400, 500, 600 }))));
    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 })), new NDArrayWritable(Nd4j.create(new double[] { 700, 800, 900 }))));
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 })), new NDArrayWritable(Nd4j.create(new double[] { 1000, 1100, 1200 }))));
    SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2));
    SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, -1, 1, true);
    DataSet ds = iter.next();
    //2 examples, 3 values per time step, 2 time steps
    INDArray expFeatures = Nd4j.create(2, 3, 2);
    expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 } }));
    expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 } }));
    INDArray expLabels = Nd4j.create(2, 3, 2);
    expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 100, 400 }, { 200, 500 }, { 300, 600 } }));
    expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 700, 1000 }, { 800, 1100 }, { 900, 1200 } }));
    assertEquals(expFeatures, ds.getFeatureMatrix());
    assertEquals(expLabels, ds.getLabels());
}
Also used : NDArrayWritable(org.datavec.common.data.NDArrayWritable) CSVSequenceRecordReader(org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader) SequenceRecordReader(org.datavec.api.records.reader.SequenceRecordReader) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) INDArray(org.nd4j.linalg.api.ndarray.INDArray) DataSet(org.nd4j.linalg.dataset.DataSet) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) IntWritable(org.datavec.api.writable.IntWritable) NDArrayWritable(org.datavec.common.data.NDArrayWritable) DoubleWritable(org.datavec.api.writable.DoubleWritable) Writable(org.datavec.api.writable.Writable) Test(org.junit.Test)

Example 4 with CollectionSequenceRecordReader

use of org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader in project deeplearning4j by deeplearning4j.

the class RecordReaderDataSetiteratorTest method testSeqRRDSIArrayWritableTwoReaders.

@Test
public void testSeqRRDSIArrayWritableTwoReaders() {
    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 })), new IntWritable(100)));
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 })), new IntWritable(200)));
    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 })), new IntWritable(300)));
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 })), new IntWritable(400)));
    SequenceRecordReader rrFeatures = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2));
    List<List<Writable>> sequence1L = new ArrayList<>();
    sequence1L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 100, 200, 300 })), new IntWritable(101)));
    sequence1L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 400, 500, 600 })), new IntWritable(201)));
    List<List<Writable>> sequence2L = new ArrayList<>();
    sequence2L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 700, 800, 900 })), new IntWritable(301)));
    sequence2L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 1000, 1100, 1200 })), new IntWritable(401)));
    SequenceRecordReader rrLabels = new CollectionSequenceRecordReader(Arrays.asList(sequence1L, sequence2L));
    SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rrFeatures, rrLabels, 2, -1, true);
    //2 examples, 4 values per time step, 2 time steps
    INDArray expFeatures = Nd4j.create(2, 4, 2);
    expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 }, { 100, 200 } }));
    expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 }, { 300, 400 } }));
    INDArray expLabels = Nd4j.create(2, 4, 2);
    expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 100, 400 }, { 200, 500 }, { 300, 600 }, { 101, 201 } }));
    expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 700, 1000 }, { 800, 1100 }, { 900, 1200 }, { 301, 401 } }));
    DataSet ds = iter.next();
    assertEquals(expFeatures, ds.getFeatureMatrix());
    assertEquals(expLabels, ds.getLabels());
}
Also used : NDArrayWritable(org.datavec.common.data.NDArrayWritable) CSVSequenceRecordReader(org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader) SequenceRecordReader(org.datavec.api.records.reader.SequenceRecordReader) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) INDArray(org.nd4j.linalg.api.ndarray.INDArray) DataSet(org.nd4j.linalg.dataset.DataSet) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) IntWritable(org.datavec.api.writable.IntWritable) NDArrayWritable(org.datavec.common.data.NDArrayWritable) DoubleWritable(org.datavec.api.writable.DoubleWritable) Writable(org.datavec.api.writable.Writable) IntWritable(org.datavec.api.writable.IntWritable) Test(org.junit.Test)

Example 5 with CollectionSequenceRecordReader

use of org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader in project deeplearning4j by deeplearning4j.

the class RecordReaderDataSetiteratorTest method testSeqRRDSIArrayWritableOneReader.

@Test
public void testSeqRRDSIArrayWritableOneReader() {
    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 })), new IntWritable(0)));
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 })), new IntWritable(1)));
    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 })), new IntWritable(2)));
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 })), new IntWritable(3)));
    SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2));
    SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, 4, 1, false);
    DataSet ds = iter.next();
    //2 examples, 3 values per time step, 2 time steps
    INDArray expFeatures = Nd4j.create(2, 3, 2);
    expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 } }));
    expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 } }));
    INDArray expLabels = Nd4j.create(2, 4, 2);
    expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 0 }, { 0, 1 }, { 0, 0 }, { 0, 0 } }));
    expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 0, 0 }, { 0, 0 }, { 1, 0 }, { 0, 1 } }));
    assertEquals(expFeatures, ds.getFeatureMatrix());
    assertEquals(expLabels, ds.getLabels());
}
Also used : NDArrayWritable(org.datavec.common.data.NDArrayWritable) CSVSequenceRecordReader(org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader) SequenceRecordReader(org.datavec.api.records.reader.SequenceRecordReader) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) INDArray(org.nd4j.linalg.api.ndarray.INDArray) DataSet(org.nd4j.linalg.dataset.DataSet) CollectionSequenceRecordReader(org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader) IntWritable(org.datavec.api.writable.IntWritable) NDArrayWritable(org.datavec.common.data.NDArrayWritable) DoubleWritable(org.datavec.api.writable.DoubleWritable) Writable(org.datavec.api.writable.Writable) IntWritable(org.datavec.api.writable.IntWritable) Test(org.junit.Test)

Aggregations

CollectionSequenceRecordReader (org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader)6 DoubleWritable (org.datavec.api.writable.DoubleWritable)6 IntWritable (org.datavec.api.writable.IntWritable)6 Writable (org.datavec.api.writable.Writable)6 Test (org.junit.Test)6 SequenceRecordReader (org.datavec.api.records.reader.SequenceRecordReader)4 CSVSequenceRecordReader (org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader)4 NDArrayWritable (org.datavec.common.data.NDArrayWritable)4 INDArray (org.nd4j.linalg.api.ndarray.INDArray)4 DataSet (org.nd4j.linalg.dataset.DataSet)4 ArrayList (java.util.ArrayList)2 Collection (java.util.Collection)2 RecordReaderDataSetIterator (org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator)2 SequenceRecordReaderDataSetIterator (org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator)2 DL4JException (org.deeplearning4j.exception.DL4JException)2 DataSet (org.nd4j.linalg.dataset.api.DataSet)2 DataSetIterator (org.nd4j.linalg.dataset.api.iterator.DataSetIterator)2