Examples with INDArray org.nd4j.linalg.api.ndarray.INDArray used on opensource projects

Example 91 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method getLabels.

private INDArray getLabels(List<List<Writable>> labels) {
    //Size of the record?
    //[timeSeriesLength,vectorSize]
    int[] shape = new int[2];
    //time series/sequence length
    shape[0] = labels.size();
    Iterator<List<Writable>> iter = labels.iterator();
    int i = 0;
    INDArray out = null;
    while (iter.hasNext()) {
        List<Writable> step = iter.next();
        if (i == 0) {
            if (regression) {
                for (Writable w : step) {
                    if (w instanceof NDArrayWritable) {
                        shape[1] += ((NDArrayWritable) w).get().length();
                    } else {
                        shape[1]++;
                    }
                }
            } else {
                shape[1] = numPossibleLabels;
            }
            out = Nd4j.create(shape, 'f');
        }
        Iterator<Writable> timeStepIter = step.iterator();
        int f = 0;
        if (regression) {
            //Load all values
            while (timeStepIter.hasNext()) {
                Writable current = timeStepIter.next();
                if (current instanceof NDArrayWritable) {
                    INDArray w = ((NDArrayWritable) current).get();
                    out.put(new INDArrayIndex[] { NDArrayIndex.point(i), NDArrayIndex.interval(f, f + w.length()) }, w);
                    f += w.length();
                } else {
                    out.put(i, f++, current.toDouble());
                }
            }
        } else {
            //Expect a single value (index) -> convert to one-hot vector
            Writable value = timeStepIter.next();
            int idx = value.toInt();
            if (idx < 0 || idx >= numPossibleLabels) {
                throw new DL4JInvalidInputException("Invalid classification data: expect label value to be in range 0 to " + (numPossibleLabels - 1) + " inclusive (0 to numClasses-1, with numClasses=" + numPossibleLabels + "); got label value of " + idx);
            }
            INDArray line = FeatureUtil.toOutcomeVector(idx, numPossibleLabels);
            out.getRow(i).assign(line);
        }
        i++;
    }
    return out;
}

Example 92 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method getFeaturesLabelsSingleReader.

private INDArray[] getFeaturesLabelsSingleReader(List<List<Writable>> input) {
    Iterator<List<Writable>> iter = input.iterator();
    int i = 0;
    INDArray features = null;
    //= Nd4j.zeros(input.size(), regression ? 1 : numPossibleLabels);
    INDArray labels = null;
    int featureSize = 0;
    while (iter.hasNext()) {
        List<Writable> step = iter.next();
        if (i == 0) {
            //First: determine the features size. Usually equal to the number of Writable objects, except when
            // one or more of the Writables is an INDArray (i.e., NDArrayWritable)
            int j = 0;
            for (Writable w : step) {
                if (j++ != labelIndex) {
                    if (w instanceof NDArrayWritable) {
                        featureSize += ((NDArrayWritable) w).get().length();
                    } else {
                        featureSize += 1;
                    }
                }
            }
            features = Nd4j.zeros(input.size(), featureSize);
            //Second: determine the output (labels) size.
            int labelSize;
            if (regression) {
                if (step.get(labelIndex) instanceof NDArrayWritable) {
                    labelSize = ((NDArrayWritable) step.get(labelIndex)).get().length();
                } else {
                    labelSize = 1;
                }
            } else {
                //Classification: integer -> one-hot
                labelSize = numPossibleLabels;
            }
            labels = Nd4j.zeros(input.size(), labelSize);
        }
        Iterator<Writable> timeStepIter = step.iterator();
        int countIn = 0;
        int countFeatures = 0;
        while (timeStepIter.hasNext()) {
            Writable current = timeStepIter.next();
            if (countIn++ == labelIndex) {
                //label
                if (regression) {
                    if (current instanceof NDArrayWritable) {
                        //Standard case
                        labels.putRow(i, ((NDArrayWritable) current).get());
                    } else {
                        labels.put(i, 0, current.toDouble());
                    }
                } else {
                    int idx = current.toInt();
                    if (idx < 0 || idx >= numPossibleLabels) {
                        throw new DL4JInvalidInputException("Invalid classification data: expect label value (at label index column = " + labelIndex + ") to be in range 0 to " + (numPossibleLabels - 1) + " inclusive (0 to numClasses-1, with numClasses=" + numPossibleLabels + "); got label value of " + current);
                    }
                    //Labels initialized as 0s
                    labels.putScalar(i, current.toInt(), 1.0);
                }
            } else {
                //feature
                if (current instanceof NDArrayWritable) {
                    //NDArrayWritable: multiple values
                    INDArray w = ((NDArrayWritable) current).get();
                    int length = w.length();
                    features.put(new INDArrayIndex[] { NDArrayIndex.point(i), NDArrayIndex.interval(countFeatures, countFeatures + length) }, w);
                    countFeatures += length;
                } else {
                    //Standard case: single value
                    features.put(i, countFeatures++, current.toDouble());
                }
            }
        }
        i++;
    }
    return new INDArray[] { features, labels };
}

Example 93 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method nextMultipleSequenceReaders.

private DataSet nextMultipleSequenceReaders(int num) {
    List<INDArray> featureList = new ArrayList<>(num);
    List<INDArray> labelList = new ArrayList<>(num);
    List<RecordMetaData> meta = (collectMetaData ? new ArrayList<RecordMetaData>() : null);
    for (int i = 0; i < num && hasNext(); i++) {
        List<List<Writable>> featureSequence;
        List<List<Writable>> labelSequence;
        if (collectMetaData) {
            SequenceRecord f = recordReader.nextSequence();
            SequenceRecord l = labelsReader.nextSequence();
            featureSequence = f.getSequenceRecord();
            labelSequence = l.getSequenceRecord();
            meta.add(new RecordMetaDataComposable(f.getMetaData(), l.getMetaData()));
        } else {
            featureSequence = recordReader.sequenceRecord();
            labelSequence = labelsReader.sequenceRecord();
        }
        assertNonZeroLengthSequence(featureSequence, "features");
        assertNonZeroLengthSequence(labelSequence, "labels");
        INDArray features = getFeatures(featureSequence);
        //2d time series, with shape [timeSeriesLength,vectorSize]
        INDArray labels = getLabels(labelSequence);
        featureList.add(features);
        labelList.add(labels);
    }
    return nextMultipleSequenceReaders(featureList, labelList, meta);
}

Example 94 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class IrisUtils method loadIris.

public static List<DataSet> loadIris(int from, int to) throws IOException {
    ClassPathResource resource = new ClassPathResource("/iris.dat");
    @SuppressWarnings("unchecked") List<String> lines = IOUtils.readLines(resource.getInputStream());
    List<DataSet> list = new ArrayList<>();
    INDArray ret = Nd4j.ones(Math.abs(to - from), 4);
    double[][] outcomes = new double[lines.size()][3];
    int putCount = 0;
    for (int i = from; i < to; i++) {
        String line = lines.get(i);
        String[] split = line.split(",");
        addRow(ret, putCount++, split);
        String outcome = split[split.length - 1];
        double[] rowOutcome = new double[3];
        rowOutcome[Integer.parseInt(outcome)] = 1;
        outcomes[i] = rowOutcome;
    }
    for (int i = 0; i < ret.rows(); i++) {
        DataSet add = new DataSet(ret.getRow(i), Nd4j.create(outcomes[from + i]));
        list.add(add);
    }
    return list;
}

Example 95 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method nextMultipleSequenceReaders.

private DataSet nextMultipleSequenceReaders(List<INDArray> featureList, List<INDArray> labelList, List<RecordMetaData> meta) {
    //Convert 2d sequences/time series to 3d minibatch data
    INDArray featuresOut;
    INDArray labelsOut;
    INDArray featuresMask = null;
    INDArray labelsMask = null;
    if (alignmentMode == AlignmentMode.EQUAL_LENGTH) {
        int[] featureShape = new int[3];
        //mini batch size
        featureShape[0] = featureList.size();
        //example vector size
        featureShape[1] = featureList.get(0).size(1);
        //time series/sequence length
        featureShape[2] = featureList.get(0).size(0);
        int[] labelShape = new int[3];
        labelShape[0] = labelList.size();
        //label vector size
        labelShape[1] = labelList.get(0).size(1);
        //time series/sequence length
        labelShape[2] = labelList.get(0).size(0);
        featuresOut = Nd4j.create(featureShape, 'f');
        labelsOut = Nd4j.create(labelShape, 'f');
        for (int i = 0; i < featureList.size(); i++) {
            featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).assign(featureList.get(i));
            labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).assign(labelList.get(i));
        }
    } else if (alignmentMode == AlignmentMode.ALIGN_START) {
        int longestTimeSeries = 0;
        for (INDArray features : featureList) {
            longestTimeSeries = Math.max(features.size(0), longestTimeSeries);
        }
        for (INDArray labels : labelList) {
            longestTimeSeries = Math.max(labels.size(0), longestTimeSeries);
        }
        int[] featuresShape = new int[] { //# examples
        featureList.size(), //example vector size
        featureList.get(0).size(1), longestTimeSeries };
        int[] labelsShape = new int[] { //# examples
        labelList.size(), //example vector size
        labelList.get(0).size(1), longestTimeSeries };
        featuresOut = Nd4j.create(featuresShape, 'f');
        labelsOut = Nd4j.create(labelsShape, 'f');
        featuresMask = Nd4j.ones(featureList.size(), longestTimeSeries);
        labelsMask = Nd4j.ones(labelList.size(), longestTimeSeries);
        for (int i = 0; i < featureList.size(); i++) {
            INDArray f = featureList.get(i);
            INDArray l = labelList.get(i);
            //Again, permute is to put [timeSeriesLength,vectorSize] into a [vectorSize,timeSeriesLength] matrix
            featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, f.size(0)), NDArrayIndex.all() }, f);
            labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, l.size(0)), NDArrayIndex.all() }, l);
            for (int j = f.size(0); j < longestTimeSeries; j++) {
                featuresMask.putScalar(i, j, 0.0);
            }
            for (int j = l.size(0); j < longestTimeSeries; j++) {
                labelsMask.putScalar(i, j, 0.0);
            }
        }
    } else if (alignmentMode == AlignmentMode.ALIGN_END) {
        //Align at end
        int longestTimeSeries = 0;
        for (INDArray features : featureList) {
            longestTimeSeries = Math.max(features.size(0), longestTimeSeries);
        }
        for (INDArray labels : labelList) {
            longestTimeSeries = Math.max(labels.size(0), longestTimeSeries);
        }
        int[] featuresShape = new int[] { //# examples
        featureList.size(), //example vector size
        featureList.get(0).size(1), longestTimeSeries };
        int[] labelsShape = new int[] { //# examples
        labelList.size(), //example vector size
        labelList.get(0).size(1), longestTimeSeries };
        featuresOut = Nd4j.create(featuresShape, 'f');
        labelsOut = Nd4j.create(labelsShape, 'f');
        featuresMask = Nd4j.ones(featureList.size(), longestTimeSeries);
        labelsMask = Nd4j.ones(labelList.size(), longestTimeSeries);
        for (int i = 0; i < featureList.size(); i++) {
            INDArray f = featureList.get(i);
            INDArray l = labelList.get(i);
            int fLen = f.size(0);
            int lLen = l.size(0);
            if (fLen >= lLen) {
                //Align labels with end of features (features are longer)
                featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, fLen), NDArrayIndex.all() }, f);
                labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(fLen - lLen, fLen), NDArrayIndex.all() }, l);
                for (int j = fLen; j < longestTimeSeries; j++) {
                    featuresMask.putScalar(i, j, 0.0);
                }
                //labels mask: component before labels
                for (int j = 0; j < fLen - lLen; j++) {
                    labelsMask.putScalar(i, j, 0.0);
                }
                //labels mask: component after labels
                for (int j = fLen; j < longestTimeSeries; j++) {
                    labelsMask.putScalar(i, j, 0.0);
                }
            } else {
                //Align features with end of labels (labels are longer)
                featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(lLen - fLen, lLen), NDArrayIndex.all() }, f);
                labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, lLen), NDArrayIndex.all() }, l);
                //features mask: component before features
                for (int j = 0; j < lLen - fLen; j++) {
                    featuresMask.putScalar(i, j, 0.0);
                }
                //features mask: component after features
                for (int j = lLen; j < longestTimeSeries; j++) {
                    featuresMask.putScalar(i, j, 0.0);
                }
                //labels mask
                for (int j = lLen; j < longestTimeSeries; j++) {
                    labelsMask.putScalar(i, j, 0.0);
                }
            }
        }
    } else {
        throw new UnsupportedOperationException("Unknown alignment mode: " + alignmentMode);
    }
    cursor += featureList.size();
    if (inputColumns == -1)
        inputColumns = featuresOut.size(1);
    if (totalOutcomes == -1)
        totalOutcomes = labelsOut.size(1);
    DataSet ds = new DataSet(featuresOut, labelsOut, featuresMask, labelsMask);
    if (collectMetaData) {
        ds.setExampleMetaData(meta);
    }
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return ds;
}