Examples with Writable org.datavec.api.writable.Writable used on opensource projects

Example 6 with Writable

use of org.datavec.api.writable.Writable in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method getFeaturesLabelsSingleReader.

private INDArray[] getFeaturesLabelsSingleReader(List<List<Writable>> input) {
    Iterator<List<Writable>> iter = input.iterator();
    int i = 0;
    INDArray features = null;
    //= Nd4j.zeros(input.size(), regression ? 1 : numPossibleLabels);
    INDArray labels = null;
    int featureSize = 0;
    while (iter.hasNext()) {
        List<Writable> step = iter.next();
        if (i == 0) {
            //First: determine the features size. Usually equal to the number of Writable objects, except when
            // one or more of the Writables is an INDArray (i.e., NDArrayWritable)
            int j = 0;
            for (Writable w : step) {
                if (j++ != labelIndex) {
                    if (w instanceof NDArrayWritable) {
                        featureSize += ((NDArrayWritable) w).get().length();
                    } else {
                        featureSize += 1;
                    }
                }
            }
            features = Nd4j.zeros(input.size(), featureSize);
            //Second: determine the output (labels) size.
            int labelSize;
            if (regression) {
                if (step.get(labelIndex) instanceof NDArrayWritable) {
                    labelSize = ((NDArrayWritable) step.get(labelIndex)).get().length();
                } else {
                    labelSize = 1;
                }
            } else {
                //Classification: integer -> one-hot
                labelSize = numPossibleLabels;
            }
            labels = Nd4j.zeros(input.size(), labelSize);
        }
        Iterator<Writable> timeStepIter = step.iterator();
        int countIn = 0;
        int countFeatures = 0;
        while (timeStepIter.hasNext()) {
            Writable current = timeStepIter.next();
            if (countIn++ == labelIndex) {
                //label
                if (regression) {
                    if (current instanceof NDArrayWritable) {
                        //Standard case
                        labels.putRow(i, ((NDArrayWritable) current).get());
                    } else {
                        labels.put(i, 0, current.toDouble());
                    }
                } else {
                    int idx = current.toInt();
                    if (idx < 0 || idx >= numPossibleLabels) {
                        throw new DL4JInvalidInputException("Invalid classification data: expect label value (at label index column = " + labelIndex + ") to be in range 0 to " + (numPossibleLabels - 1) + " inclusive (0 to numClasses-1, with numClasses=" + numPossibleLabels + "); got label value of " + current);
                    }
                    //Labels initialized as 0s
                    labels.putScalar(i, current.toInt(), 1.0);
                }
            } else {
                //feature
                if (current instanceof NDArrayWritable) {
                    //NDArrayWritable: multiple values
                    INDArray w = ((NDArrayWritable) current).get();
                    int length = w.length();
                    features.put(new INDArrayIndex[] { NDArrayIndex.point(i), NDArrayIndex.interval(countFeatures, countFeatures + length) }, w);
                    countFeatures += length;
                } else {
                    //Standard case: single value
                    features.put(i, countFeatures++, current.toDouble());
                }
            }
        }
        i++;
    }
    return new INDArray[] { features, labels };
}

Example 7 with Writable

use of org.datavec.api.writable.Writable in project deeplearning4j by deeplearning4j.

the class RecordReaderDataSetiteratorTest method testSeqRRDSIMultipleArrayWritablesOneReader.

@Test
public void testSeqRRDSIMultipleArrayWritablesOneReader() {
    //Input with multiple array writables:
    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 100, 200, 300 })), new IntWritable(0)));
    sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 })), new NDArrayWritable(Nd4j.create(new double[] { 400, 500, 600 })), new IntWritable(1)));
    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 })), new NDArrayWritable(Nd4j.create(new double[] { 700, 800, 900 })), new IntWritable(2)));
    sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 })), new NDArrayWritable(Nd4j.create(new double[] { 1000, 1100, 1200 })), new IntWritable(3)));
    SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2));
    SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, 4, 2, false);
    DataSet ds = iter.next();
    //2 examples, 6 values per time step, 2 time steps
    INDArray expFeatures = Nd4j.create(2, 6, 2);
    expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 }, { 100, 400 }, { 200, 500 }, { 300, 600 } }));
    expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 }, { 700, 1000 }, { 800, 1100 }, { 900, 1200 } }));
    INDArray expLabels = Nd4j.create(2, 4, 2);
    expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 0 }, { 0, 1 }, { 0, 0 }, { 0, 0 } }));
    expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 0, 0 }, { 0, 0 }, { 1, 0 }, { 0, 1 } }));
    assertEquals(expFeatures, ds.getFeatureMatrix());
    assertEquals(expLabels, ds.getLabels());
}

Example 8 with Writable

use of org.datavec.api.writable.Writable in project deeplearning4j by deeplearning4j.

the class TestRecordReaders method testClassIndexOutsideOfRangeRRMDSI_MultipleReaders.

@Test
public void testClassIndexOutsideOfRangeRRMDSI_MultipleReaders() {
    Collection<Collection<Collection<Writable>>> c1 = new ArrayList<>();
    Collection<Collection<Writable>> seq1 = new ArrayList<>();
    seq1.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    seq1.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    c1.add(seq1);
    Collection<Collection<Writable>> seq2 = new ArrayList<>();
    seq2.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    seq2.add(Arrays.<Writable>asList(new DoubleWritable(0.0)));
    c1.add(seq2);
    Collection<Collection<Collection<Writable>>> c2 = new ArrayList<>();
    Collection<Collection<Writable>> seq1a = new ArrayList<>();
    seq1a.add(Arrays.<Writable>asList(new IntWritable(0)));
    seq1a.add(Arrays.<Writable>asList(new IntWritable(1)));
    c2.add(seq1a);
    Collection<Collection<Writable>> seq2a = new ArrayList<>();
    seq2a.add(Arrays.<Writable>asList(new IntWritable(0)));
    seq2a.add(Arrays.<Writable>asList(new IntWritable(2)));
    c2.add(seq2a);
    CollectionSequenceRecordReader csrr = new CollectionSequenceRecordReader(c1);
    CollectionSequenceRecordReader csrrLabels = new CollectionSequenceRecordReader(c2);
    DataSetIterator dsi = new SequenceRecordReaderDataSetIterator(csrr, csrrLabels, 2, 2);
    try {
        DataSet ds = dsi.next();
        fail("Expected exception");
    } catch (DL4JException e) {
        System.out.println("testClassIndexOutsideOfRangeRRMDSI_MultipleReaders(): " + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 9 with Writable

use of org.datavec.api.writable.Writable in project deeplearning4j by deeplearning4j.

the class DataVecSequenceDataSetFunction method call.

@Override
public DataSet call(List<List<Writable>> input) throws Exception {
    Iterator<List<Writable>> iter = input.iterator();
    INDArray features = null;
    INDArray labels = Nd4j.zeros(1, (regression ? 1 : numPossibleLabels), input.size());
    int[] fIdx = new int[3];
    int[] lIdx = new int[3];
    int i = 0;
    while (iter.hasNext()) {
        List<Writable> step = iter.next();
        if (i == 0) {
            features = Nd4j.zeros(1, step.size() - 1, input.size());
        }
        Iterator<Writable> timeStepIter = step.iterator();
        int countIn = 0;
        int countFeatures = 0;
        while (timeStepIter.hasNext()) {
            Writable current = timeStepIter.next();
            if (converter != null)
                current = converter.convert(current);
            if (countIn++ == labelIndex) {
                //label
                if (regression) {
                    lIdx[2] = i;
                    labels.putScalar(lIdx, current.toDouble());
                } else {
                    INDArray line = FeatureUtil.toOutcomeVector(current.toInt(), numPossibleLabels);
                    //1d from [1,nOut,timeSeriesLength] -> tensor i along dimension 1 is at time i
                    labels.tensorAlongDimension(i, 1).assign(line);
                }
            } else {
                //feature
                fIdx[1] = countFeatures++;
                fIdx[2] = i;
                try {
                    features.putScalar(fIdx, current.toDouble());
                } catch (UnsupportedOperationException e) {
                    // This isn't a scalar, so check if we got an array already
                    if (current instanceof NDArrayWritable) {
                        features.get(NDArrayIndex.point(fIdx[0]), NDArrayIndex.all(), NDArrayIndex.point(fIdx[2])).putRow(0, ((NDArrayWritable) current).get());
                    } else {
                        throw e;
                    }
                }
            }
        }
        i++;
    }
    DataSet ds = new DataSet(features, labels);
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return ds;
}

Example 10 with Writable

use of org.datavec.api.writable.Writable in project deeplearning4j by deeplearning4j.

the class CSVRecordToDataSet method convert.

@Override
public DataSet convert(Collection<Collection<Writable>> records, int numLabels) {
    //all but last label
    DataSet ret = new DataSet(Nd4j.create(records.size(), records.iterator().next().size() - 1), Nd4j.create(records.size(), numLabels));
    //  INDArray ret = Nd4j.create(records.size(),records.iterator().next().size() - 1);
    int count = 0;
    for (Collection<Writable> record : records) {
        List<Writable> list;
        if (record instanceof List) {
            list = (List<Writable>) record;
        } else
            list = new ArrayList<>(record);
        DataSet d = new DataSet(Nd4j.create(record.size() - 1), FeatureUtil.toOutcomeVector(list.get(list.size() - 1).toInt(), numLabels));
        ret.addRow(d, count++);
    }
    return ret;
}