Examples with StringSplit org.datavec.api.split.StringSplit used on opensource projects

Example 1 with StringSplit

use of org.datavec.api.split.StringSplit in project deeplearning4j by deeplearning4j.

the class RecordReaderFunction method call.

@Override
public DataSet call(String v1) throws Exception {
    recordReader.initialize(new StringSplit(v1));
    List<DataSet> dataSets = new ArrayList<>();
    List<Writable> record = recordReader.next();
    List<Writable> currList;
    if (record instanceof List)
        currList = (List<Writable>) record;
    else
        currList = new ArrayList<>(record);
    INDArray label = null;
    INDArray featureVector = Nd4j.create(labelIndex >= 0 ? currList.size() - 1 : currList.size());
    int count = 0;
    for (int j = 0; j < currList.size(); j++) {
        if (labelIndex >= 0 && j == labelIndex) {
            if (numPossibleLabels < 1)
                throw new IllegalStateException("Number of possible labels invalid, must be >= 1");
            Writable current = currList.get(j);
            if (converter != null)
                current = converter.convert(current);
            label = FeatureUtil.toOutcomeVector(Double.valueOf(current.toString()).intValue(), numPossibleLabels);
        } else {
            Writable current = currList.get(j);
            featureVector.putScalar(count++, Double.valueOf(current.toString()));
        }
    }
    dataSets.add(new DataSet(featureVector, labelIndex >= 0 ? label : featureVector));
    List<INDArray> inputs = new ArrayList<>();
    List<INDArray> labels = new ArrayList<>();
    for (DataSet data : dataSets) {
        inputs.add(data.getFeatureMatrix());
        labels.add(data.getLabels());
    }
    DataSet ret = new DataSet(Nd4j.vstack(inputs.toArray(new INDArray[0])), Nd4j.vstack(labels.toArray(new INDArray[0])));
    return ret;
}

Example 2 with StringSplit

use of org.datavec.api.split.StringSplit in project deeplearning4j by deeplearning4j.

the class StringToDataSetExportFunction method call.

@Override
public void call(Iterator<String> stringIterator) throws Exception {
    String jvmuid = UIDProvider.getJVMUID();
    uid = Thread.currentThread().getId() + jvmuid.substring(0, Math.min(8, jvmuid.length()));
    List<List<Writable>> list = new ArrayList<>(batchSize);
    while (stringIterator.hasNext()) {
        String next = stringIterator.next();
        recordReader.initialize(new StringSplit(next));
        list.add(recordReader.next());
        processBatchIfRequired(list, !stringIterator.hasNext());
    }
}