Examples with SequenceRecord org.datavec.api.records.SequenceRecord used on opensource projects

Example 1 with SequenceRecord

use of org.datavec.api.records.SequenceRecord in project deeplearning4j by deeplearning4j.

the class RecordReaderMultiDataSetIterator method loadFromMetaData.

/**
     * Load a multiple sequence examples to a DataSet, using the provided RecordMetaData instances.
     *
     * @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
     *             to the SequenceRecordReaderDataSetIterator constructor
     * @return DataSet with the specified examples
     * @throws IOException If an error occurs during loading of the data
     */
public MultiDataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
    //First: load the next values from the RR / SeqRRs
    Map<String, List<List<Writable>>> nextRRVals = new HashMap<>();
    Map<String, List<List<List<Writable>>>> nextSeqRRVals = new HashMap<>();
    List<RecordMetaDataComposableMap> nextMetas = (collectMetaData ? new ArrayList<RecordMetaDataComposableMap>() : null);
    for (Map.Entry<String, RecordReader> entry : recordReaders.entrySet()) {
        RecordReader rr = entry.getValue();
        List<RecordMetaData> thisRRMeta = new ArrayList<>();
        for (RecordMetaData m : list) {
            RecordMetaDataComposableMap m2 = (RecordMetaDataComposableMap) m;
            thisRRMeta.add(m2.getMeta().get(entry.getKey()));
        }
        List<Record> fromMeta = rr.loadFromMetaData(thisRRMeta);
        List<List<Writable>> writables = new ArrayList<>(list.size());
        for (Record r : fromMeta) {
            writables.add(r.getRecord());
        }
        nextRRVals.put(entry.getKey(), writables);
    }
    for (Map.Entry<String, SequenceRecordReader> entry : sequenceRecordReaders.entrySet()) {
        SequenceRecordReader rr = entry.getValue();
        List<RecordMetaData> thisRRMeta = new ArrayList<>();
        for (RecordMetaData m : list) {
            RecordMetaDataComposableMap m2 = (RecordMetaDataComposableMap) m;
            thisRRMeta.add(m2.getMeta().get(entry.getKey()));
        }
        List<SequenceRecord> fromMeta = rr.loadSequenceFromMetaData(thisRRMeta);
        List<List<List<Writable>>> writables = new ArrayList<>(list.size());
        for (SequenceRecord r : fromMeta) {
            writables.add(r.getSequenceRecord());
        }
        nextSeqRRVals.put(entry.getKey(), writables);
    }
    return nextMultiDataSet(nextRRVals, nextSeqRRVals, nextMetas);
}

Example 2 with SequenceRecord

use of org.datavec.api.records.SequenceRecord in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method nextMultipleSequenceReaders.

private DataSet nextMultipleSequenceReaders(int num) {
    List<INDArray> featureList = new ArrayList<>(num);
    List<INDArray> labelList = new ArrayList<>(num);
    List<RecordMetaData> meta = (collectMetaData ? new ArrayList<RecordMetaData>() : null);
    for (int i = 0; i < num && hasNext(); i++) {
        List<List<Writable>> featureSequence;
        List<List<Writable>> labelSequence;
        if (collectMetaData) {
            SequenceRecord f = recordReader.nextSequence();
            SequenceRecord l = labelsReader.nextSequence();
            featureSequence = f.getSequenceRecord();
            labelSequence = l.getSequenceRecord();
            meta.add(new RecordMetaDataComposable(f.getMetaData(), l.getMetaData()));
        } else {
            featureSequence = recordReader.sequenceRecord();
            labelSequence = labelsReader.sequenceRecord();
        }
        assertNonZeroLengthSequence(featureSequence, "features");
        assertNonZeroLengthSequence(labelSequence, "labels");
        INDArray features = getFeatures(featureSequence);
        //2d time series, with shape [timeSeriesLength,vectorSize]
        INDArray labels = getLabels(labelSequence);
        featureList.add(features);
        labelList.add(labels);
    }
    return nextMultipleSequenceReaders(featureList, labelList, meta);
}

Example 3 with SequenceRecord

use of org.datavec.api.records.SequenceRecord in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method nextSingleSequenceReader.

private DataSet nextSingleSequenceReader(int num) {
    List<INDArray> listFeatures = new ArrayList<>(num);
    List<INDArray> listLabels = new ArrayList<>(num);
    List<RecordMetaData> meta = (collectMetaData ? new ArrayList<RecordMetaData>() : null);
    int minLength = 0;
    int maxLength = 0;
    for (int i = 0; i < num && hasNext(); i++) {
        List<List<Writable>> sequence;
        if (collectMetaData) {
            SequenceRecord sequenceRecord = recordReader.nextSequence();
            sequence = sequenceRecord.getSequenceRecord();
            meta.add(sequenceRecord.getMetaData());
        } else {
            sequence = recordReader.sequenceRecord();
        }
        assertNonZeroLengthSequence(sequence, "combined features and labels");
        INDArray[] fl = getFeaturesLabelsSingleReader(sequence);
        if (i == 0) {
            minLength = fl[0].size(0);
            maxLength = minLength;
        } else {
            minLength = Math.min(minLength, fl[0].size(0));
            maxLength = Math.max(maxLength, fl[0].size(0));
        }
        listFeatures.add(fl[0]);
        listLabels.add(fl[1]);
    }
    return getSingleSequenceReader(listFeatures, listLabels, minLength, maxLength, meta);
}

Example 4 with SequenceRecord

use of org.datavec.api.records.SequenceRecord in project deeplearning4j by deeplearning4j.

the class RecordReaderMultiDataSetIterator method next.

@Override
public MultiDataSet next(int num) {
    if (!hasNext())
        throw new NoSuchElementException("No next elements");
    //First: load the next values from the RR / SeqRRs
    Map<String, List<List<Writable>>> nextRRVals = new HashMap<>();
    Map<String, List<List<List<Writable>>>> nextSeqRRVals = new HashMap<>();
    List<RecordMetaDataComposableMap> nextMetas = (collectMetaData ? new ArrayList<RecordMetaDataComposableMap>() : null);
    for (Map.Entry<String, RecordReader> entry : recordReaders.entrySet()) {
        RecordReader rr = entry.getValue();
        List<List<Writable>> writables = new ArrayList<>(num);
        for (int i = 0; i < num && rr.hasNext(); i++) {
            List<Writable> record;
            if (collectMetaData) {
                Record r = rr.nextRecord();
                record = r.getRecord();
                if (nextMetas.size() <= i) {
                    nextMetas.add(new RecordMetaDataComposableMap(new HashMap<String, RecordMetaData>()));
                }
                RecordMetaDataComposableMap map = nextMetas.get(i);
                map.getMeta().put(entry.getKey(), r.getMetaData());
            } else {
                record = rr.next();
            }
            writables.add(record);
        }
        nextRRVals.put(entry.getKey(), writables);
    }
    for (Map.Entry<String, SequenceRecordReader> entry : sequenceRecordReaders.entrySet()) {
        SequenceRecordReader rr = entry.getValue();
        List<List<List<Writable>>> writables = new ArrayList<>(num);
        for (int i = 0; i < num && rr.hasNext(); i++) {
            List<List<Writable>> sequence;
            if (collectMetaData) {
                SequenceRecord r = rr.nextSequence();
                sequence = r.getSequenceRecord();
                if (nextMetas.size() <= i) {
                    nextMetas.add(new RecordMetaDataComposableMap(new HashMap<String, RecordMetaData>()));
                }
                RecordMetaDataComposableMap map = nextMetas.get(i);
                map.getMeta().put(entry.getKey(), r.getMetaData());
            } else {
                sequence = rr.sequenceRecord();
            }
            writables.add(sequence);
        }
        nextSeqRRVals.put(entry.getKey(), writables);
    }
    return nextMultiDataSet(nextRRVals, nextSeqRRVals, nextMetas);
}

Example 5 with SequenceRecord

use of org.datavec.api.records.SequenceRecord in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method loadFromMetaData.

/**
     * Load a multiple sequence examples to a DataSet, using the provided RecordMetaData instances.
     *
     * @param list List of RecordMetaData instances to load from. Should have been produced by the record reader provided
     *             to the SequenceRecordReaderDataSetIterator constructor
     * @return DataSet with the specified examples
     * @throws IOException If an error occurs during loading of the data
     */
public DataSet loadFromMetaData(List<RecordMetaData> list) throws IOException {
    //Two cases: single vs. multiple reader...
    if (singleSequenceReaderMode) {
        List<SequenceRecord> records = recordReader.loadSequenceFromMetaData(list);
        List<INDArray> listFeatures = new ArrayList<>(list.size());
        List<INDArray> listLabels = new ArrayList<>(list.size());
        int minLength = Integer.MAX_VALUE;
        int maxLength = Integer.MIN_VALUE;
        for (SequenceRecord sr : records) {
            INDArray[] fl = getFeaturesLabelsSingleReader(sr.getSequenceRecord());
            listFeatures.add(fl[0]);
            listLabels.add(fl[1]);
            minLength = Math.min(minLength, fl[0].size(0));
            maxLength = Math.max(maxLength, fl[1].size(0));
        }
        return getSingleSequenceReader(listFeatures, listLabels, minLength, maxLength, list);
    } else {
        //Expect to get a RecordReaderMetaComposable here
        List<RecordMetaData> fMeta = new ArrayList<>();
        List<RecordMetaData> lMeta = new ArrayList<>();
        for (RecordMetaData m : list) {
            RecordMetaDataComposable m2 = (RecordMetaDataComposable) m;
            fMeta.add(m2.getMeta()[0]);
            lMeta.add(m2.getMeta()[1]);
        }
        List<SequenceRecord> f = recordReader.loadSequenceFromMetaData(fMeta);
        List<SequenceRecord> l = labelsReader.loadSequenceFromMetaData(lMeta);
        List<INDArray> featureList = new ArrayList<>(fMeta.size());
        List<INDArray> labelList = new ArrayList<>(fMeta.size());
        for (int i = 0; i < fMeta.size(); i++) {
            featureList.add(getFeatures(f.get(i).getSequenceRecord()));
            labelList.add(getLabels(l.get(i).getSequenceRecord()));
        }
        return nextMultipleSequenceReaders(featureList, labelList, list);
    }
}