Examples with WritableConverterException org.datavec.api.io.converters.WritableConverterException used on opensource projects

Example 1 with WritableConverterException

use of org.datavec.api.io.converters.WritableConverterException in project deeplearning4j by deeplearning4j.

the class RecordReaderDataSetIterator method getDataSet.

private DataSet getDataSet(List<Writable> record) {
    List<Writable> currList;
    if (record instanceof List)
        currList = record;
    else
        currList = new ArrayList<>(record);
    //allow people to specify label index as -1 and infer the last possible label
    if (numPossibleLabels >= 1 && labelIndex < 0) {
        labelIndex = record.size() - 1;
    }
    INDArray label = null;
    INDArray featureVector = null;
    int featureCount = 0;
    int labelCount = 0;
    //no labels
    if (currList.size() == 2 && currList.get(1) instanceof NDArrayWritable && currList.get(0) instanceof NDArrayWritable && currList.get(0) == currList.get(1)) {
        NDArrayWritable writable = (NDArrayWritable) currList.get(0);
        return new DataSet(writable.get(), writable.get());
    }
    if (currList.size() == 2 && currList.get(0) instanceof NDArrayWritable) {
        if (!regression) {
            label = FeatureUtil.toOutcomeVector((int) Double.parseDouble(currList.get(1).toString()), numPossibleLabels);
        } else {
            if (currList.get(1) instanceof NDArrayWritable) {
                label = ((NDArrayWritable) currList.get(1)).get();
            } else {
                label = Nd4j.scalar(currList.get(1).toDouble());
            }
        }
        NDArrayWritable ndArrayWritable = (NDArrayWritable) currList.get(0);
        featureVector = ndArrayWritable.get();
        return new DataSet(featureVector, label);
    }
    for (int j = 0; j < currList.size(); j++) {
        Writable current = currList.get(j);
        //ndarray writable is an insane slow down herecd
        if (!(current instanceof NDArrayWritable) && current.toString().isEmpty())
            continue;
        if (regression && j == labelIndex && j == labelIndexTo && current instanceof NDArrayWritable) {
            //Case: NDArrayWritable for the labels
            label = ((NDArrayWritable) current).get();
        } else if (regression && j >= labelIndex && j <= labelIndexTo) {
            //This is the multi-label regression case
            if (label == null)
                label = Nd4j.create(1, (labelIndexTo - labelIndex + 1));
            label.putScalar(labelCount++, current.toDouble());
        } else if (labelIndex >= 0 && j == labelIndex) {
            //single label case (classification, etc)
            if (converter != null)
                try {
                    current = converter.convert(current);
                } catch (WritableConverterException e) {
                    e.printStackTrace();
                }
            if (numPossibleLabels < 1)
                throw new IllegalStateException("Number of possible labels invalid, must be >= 1");
            if (regression) {
                label = Nd4j.scalar(current.toDouble());
            } else {
                int curr = current.toInt();
                if (curr < 0 || curr >= numPossibleLabels) {
                    throw new DL4JInvalidInputException("Invalid classification data: expect label value (at label index column = " + labelIndex + ") to be in range 0 to " + (numPossibleLabels - 1) + " inclusive (0 to numClasses-1, with numClasses=" + numPossibleLabels + "); got label value of " + current);
                }
                label = FeatureUtil.toOutcomeVector(curr, numPossibleLabels);
            }
        } else {
            try {
                double value = current.toDouble();
                if (featureVector == null) {
                    if (regression && labelIndex >= 0) {
                        //Handle the possibly multi-label regression case here:
                        int nLabels = labelIndexTo - labelIndex + 1;
                        featureVector = Nd4j.create(1, currList.size() - nLabels);
                    } else {
                        //Classification case, and also no-labels case
                        featureVector = Nd4j.create(labelIndex >= 0 ? currList.size() - 1 : currList.size());
                    }
                }
                featureVector.putScalar(featureCount++, value);
            } catch (UnsupportedOperationException e) {
                // This isn't a scalar, so check if we got an array already
                if (current instanceof NDArrayWritable) {
                    assert featureVector == null;
                    featureVector = ((NDArrayWritable) current).get();
                } else {
                    throw e;
                }
            }
        }
    }
    return new DataSet(featureVector, labelIndex >= 0 ? label : featureVector);
}

Example 2 with WritableConverterException

use of org.datavec.api.io.converters.WritableConverterException in project deeplearning4j by deeplearning4j.

the class DataVecDataSetFunction method call.

@Override
public DataSet call(List<Writable> currList) throws Exception {
    //allow people to specify label index as -1 and infer the last possible label
    int labelIndex = this.labelIndex;
    if (numPossibleLabels >= 1 && labelIndex < 0) {
        labelIndex = currList.size() - 1;
    }
    INDArray label = null;
    INDArray featureVector = null;
    int featureCount = 0;
    int labelCount = 0;
    //no labels
    if (currList.size() == 2 && currList.get(1) instanceof NDArrayWritable && currList.get(0) instanceof NDArrayWritable && currList.get(0) == currList.get(1)) {
        NDArrayWritable writable = (NDArrayWritable) currList.get(0);
        DataSet ds = new DataSet(writable.get(), writable.get());
        if (preProcessor != null)
            preProcessor.preProcess(ds);
        return ds;
    }
    if (currList.size() == 2 && currList.get(0) instanceof NDArrayWritable) {
        if (!regression)
            label = FeatureUtil.toOutcomeVector((int) Double.parseDouble(currList.get(1).toString()), numPossibleLabels);
        else
            label = Nd4j.scalar(Double.parseDouble(currList.get(1).toString()));
        NDArrayWritable ndArrayWritable = (NDArrayWritable) currList.get(0);
        featureVector = ndArrayWritable.get();
        DataSet ds = new DataSet(featureVector, label);
        if (preProcessor != null)
            preProcessor.preProcess(ds);
        return ds;
    }
    for (int j = 0; j < currList.size(); j++) {
        Writable current = currList.get(j);
        //ndarray writable is an insane slow down here
        if (!(current instanceof NDArrayWritable) && current.toString().isEmpty())
            continue;
        if (labelIndex >= 0 && j >= labelIndex && j <= labelIndexTo) {
            //single label case (classification, single label regression etc)
            if (converter != null) {
                try {
                    current = converter.convert(current);
                } catch (WritableConverterException e) {
                    e.printStackTrace();
                }
            }
            if (regression) {
                //single and multi-label regression
                if (label == null) {
                    label = Nd4j.zeros(labelIndexTo - labelIndex + 1);
                }
                label.putScalar(0, labelCount++, current.toDouble());
            } else {
                if (numPossibleLabels < 1)
                    throw new IllegalStateException("Number of possible labels invalid, must be >= 1 for classification");
                int curr = current.toInt();
                if (curr >= numPossibleLabels)
                    throw new IllegalStateException("Invalid index: got index " + curr + " but numPossibleLabels is " + numPossibleLabels + " (must be 0 <= idx < numPossibleLabels");
                label = FeatureUtil.toOutcomeVector(curr, numPossibleLabels);
            }
        } else {
            try {
                double value = current.toDouble();
                if (featureVector == null) {
                    if (regression && labelIndex >= 0) {
                        //Handle the possibly multi-label regression case here:
                        int nLabels = labelIndexTo - labelIndex + 1;
                        featureVector = Nd4j.create(1, currList.size() - nLabels);
                    } else {
                        //Classification case, and also no-labels case
                        featureVector = Nd4j.create(labelIndex >= 0 ? currList.size() - 1 : currList.size());
                    }
                }
                featureVector.putScalar(featureCount++, value);
            } catch (UnsupportedOperationException e) {
                // This isn't a scalar, so check if we got an array already
                if (current instanceof NDArrayWritable) {
                    assert featureVector == null;
                    featureVector = ((NDArrayWritable) current).get();
                } else {
                    throw e;
                }
            }
        }
    }
    DataSet ds = new DataSet(featureVector, (labelIndex >= 0 ? label : featureVector));
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return ds;
}