Examples with DataSet org.nd4j.linalg.dataset.DataSet used on opensource projects

Example 51 with DataSet

use of org.nd4j.linalg.dataset.DataSet in project deeplearning4j by deeplearning4j.

the class RecordReaderDataSetIterator method inputColumns.

@Override
public int inputColumns() {
    if (last == null) {
        DataSet next = next();
        last = next;
        useCurrent = true;
        return next.numInputs();
    } else
        return last.numInputs();
}

Example 52 with DataSet

use of org.nd4j.linalg.dataset.DataSet in project deeplearning4j by deeplearning4j.

the class IrisUtils method loadIris.

public static List<DataSet> loadIris(int from, int to) throws IOException {
    ClassPathResource resource = new ClassPathResource("/iris.dat");
    @SuppressWarnings("unchecked") List<String> lines = IOUtils.readLines(resource.getInputStream());
    List<DataSet> list = new ArrayList<>();
    INDArray ret = Nd4j.ones(Math.abs(to - from), 4);
    double[][] outcomes = new double[lines.size()][3];
    int putCount = 0;
    for (int i = from; i < to; i++) {
        String line = lines.get(i);
        String[] split = line.split(",");
        addRow(ret, putCount++, split);
        String outcome = split[split.length - 1];
        double[] rowOutcome = new double[3];
        rowOutcome[Integer.parseInt(outcome)] = 1;
        outcomes[i] = rowOutcome;
    }
    for (int i = 0; i < ret.rows(); i++) {
        DataSet add = new DataSet(ret.getRow(i), Nd4j.create(outcomes[from + i]));
        list.add(add);
    }
    return list;
}

Example 53 with DataSet

use of org.nd4j.linalg.dataset.DataSet in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method nextMultipleSequenceReaders.

private DataSet nextMultipleSequenceReaders(List<INDArray> featureList, List<INDArray> labelList, List<RecordMetaData> meta) {
    //Convert 2d sequences/time series to 3d minibatch data
    INDArray featuresOut;
    INDArray labelsOut;
    INDArray featuresMask = null;
    INDArray labelsMask = null;
    if (alignmentMode == AlignmentMode.EQUAL_LENGTH) {
        int[] featureShape = new int[3];
        //mini batch size
        featureShape[0] = featureList.size();
        //example vector size
        featureShape[1] = featureList.get(0).size(1);
        //time series/sequence length
        featureShape[2] = featureList.get(0).size(0);
        int[] labelShape = new int[3];
        labelShape[0] = labelList.size();
        //label vector size
        labelShape[1] = labelList.get(0).size(1);
        //time series/sequence length
        labelShape[2] = labelList.get(0).size(0);
        featuresOut = Nd4j.create(featureShape, 'f');
        labelsOut = Nd4j.create(labelShape, 'f');
        for (int i = 0; i < featureList.size(); i++) {
            featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).assign(featureList.get(i));
            labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).assign(labelList.get(i));
        }
    } else if (alignmentMode == AlignmentMode.ALIGN_START) {
        int longestTimeSeries = 0;
        for (INDArray features : featureList) {
            longestTimeSeries = Math.max(features.size(0), longestTimeSeries);
        }
        for (INDArray labels : labelList) {
            longestTimeSeries = Math.max(labels.size(0), longestTimeSeries);
        }
        int[] featuresShape = new int[] { //# examples
        featureList.size(), //example vector size
        featureList.get(0).size(1), longestTimeSeries };
        int[] labelsShape = new int[] { //# examples
        labelList.size(), //example vector size
        labelList.get(0).size(1), longestTimeSeries };
        featuresOut = Nd4j.create(featuresShape, 'f');
        labelsOut = Nd4j.create(labelsShape, 'f');
        featuresMask = Nd4j.ones(featureList.size(), longestTimeSeries);
        labelsMask = Nd4j.ones(labelList.size(), longestTimeSeries);
        for (int i = 0; i < featureList.size(); i++) {
            INDArray f = featureList.get(i);
            INDArray l = labelList.get(i);
            //Again, permute is to put [timeSeriesLength,vectorSize] into a [vectorSize,timeSeriesLength] matrix
            featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, f.size(0)), NDArrayIndex.all() }, f);
            labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, l.size(0)), NDArrayIndex.all() }, l);
            for (int j = f.size(0); j < longestTimeSeries; j++) {
                featuresMask.putScalar(i, j, 0.0);
            }
            for (int j = l.size(0); j < longestTimeSeries; j++) {
                labelsMask.putScalar(i, j, 0.0);
            }
        }
    } else if (alignmentMode == AlignmentMode.ALIGN_END) {
        //Align at end
        int longestTimeSeries = 0;
        for (INDArray features : featureList) {
            longestTimeSeries = Math.max(features.size(0), longestTimeSeries);
        }
        for (INDArray labels : labelList) {
            longestTimeSeries = Math.max(labels.size(0), longestTimeSeries);
        }
        int[] featuresShape = new int[] { //# examples
        featureList.size(), //example vector size
        featureList.get(0).size(1), longestTimeSeries };
        int[] labelsShape = new int[] { //# examples
        labelList.size(), //example vector size
        labelList.get(0).size(1), longestTimeSeries };
        featuresOut = Nd4j.create(featuresShape, 'f');
        labelsOut = Nd4j.create(labelsShape, 'f');
        featuresMask = Nd4j.ones(featureList.size(), longestTimeSeries);
        labelsMask = Nd4j.ones(labelList.size(), longestTimeSeries);
        for (int i = 0; i < featureList.size(); i++) {
            INDArray f = featureList.get(i);
            INDArray l = labelList.get(i);
            int fLen = f.size(0);
            int lLen = l.size(0);
            if (fLen >= lLen) {
                //Align labels with end of features (features are longer)
                featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, fLen), NDArrayIndex.all() }, f);
                labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(fLen - lLen, fLen), NDArrayIndex.all() }, l);
                for (int j = fLen; j < longestTimeSeries; j++) {
                    featuresMask.putScalar(i, j, 0.0);
                }
                //labels mask: component before labels
                for (int j = 0; j < fLen - lLen; j++) {
                    labelsMask.putScalar(i, j, 0.0);
                }
                //labels mask: component after labels
                for (int j = fLen; j < longestTimeSeries; j++) {
                    labelsMask.putScalar(i, j, 0.0);
                }
            } else {
                //Align features with end of labels (labels are longer)
                featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(lLen - fLen, lLen), NDArrayIndex.all() }, f);
                labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, lLen), NDArrayIndex.all() }, l);
                //features mask: component before features
                for (int j = 0; j < lLen - fLen; j++) {
                    featuresMask.putScalar(i, j, 0.0);
                }
                //features mask: component after features
                for (int j = lLen; j < longestTimeSeries; j++) {
                    featuresMask.putScalar(i, j, 0.0);
                }
                //labels mask
                for (int j = lLen; j < longestTimeSeries; j++) {
                    labelsMask.putScalar(i, j, 0.0);
                }
            }
        }
    } else {
        throw new UnsupportedOperationException("Unknown alignment mode: " + alignmentMode);
    }
    cursor += featureList.size();
    if (inputColumns == -1)
        inputColumns = featuresOut.size(1);
    if (totalOutcomes == -1)
        totalOutcomes = labelsOut.size(1);
    DataSet ds = new DataSet(featuresOut, labelsOut, featuresMask, labelsMask);
    if (collectMetaData) {
        ds.setExampleMetaData(meta);
    }
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return ds;
}

Example 54 with DataSet

use of org.nd4j.linalg.dataset.DataSet in project deeplearning4j by deeplearning4j.

the class SequenceRecordReaderDataSetIterator method getSingleSequenceReader.

private DataSet getSingleSequenceReader(List<INDArray> listFeatures, List<INDArray> listLabels, int minLength, int maxLength, List<RecordMetaData> meta) {
    //Convert to 3d minibatch
    //Note: using f order here, as each  time step is contiguous in the buffer with f order (isn't the case with c order)
    INDArray featuresOut = Nd4j.create(new int[] { listFeatures.size(), listFeatures.get(0).size(1), maxLength }, 'f');
    INDArray labelsOut = Nd4j.create(new int[] { listLabels.size(), listLabels.get(0).size(1), maxLength }, 'f');
    INDArray featuresMask = null;
    INDArray labelsMask = null;
    if (minLength == maxLength) {
        for (int i = 0; i < listFeatures.size(); i++) {
            //Note: this TAD gives us shape [vectorSize,tsLength] whereas we need a [vectorSize,timeSeriesLength] matrix (that listFeatures contains)
            featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).assign(listFeatures.get(i));
            labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).assign(listLabels.get(i));
        }
    } else {
        featuresMask = Nd4j.ones(listFeatures.size(), maxLength);
        labelsMask = Nd4j.ones(listLabels.size(), maxLength);
        for (int i = 0; i < listFeatures.size(); i++) {
            INDArray f = listFeatures.get(i);
            int tsLength = f.size(0);
            featuresOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, tsLength), NDArrayIndex.all() }, f);
            labelsOut.tensorAlongDimension(i, 1, 2).permutei(1, 0).put(new INDArrayIndex[] { NDArrayIndex.interval(0, tsLength), NDArrayIndex.all() }, listLabels.get(i));
            for (int j = tsLength; j < maxLength; j++) {
                featuresMask.put(i, j, 0.0);
                labelsMask.put(i, j, 0.0);
            }
        }
    }
    cursor += listFeatures.size();
    if (inputColumns == -1)
        inputColumns = featuresOut.size(1);
    if (totalOutcomes == -1)
        totalOutcomes = labelsOut.size(1);
    DataSet ds = new DataSet(featuresOut, labelsOut, featuresMask, labelsMask);
    if (collectMetaData) {
        ds.setExampleMetaData(meta);
    }
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return ds;
}

Example 55 with DataSet

use of org.nd4j.linalg.dataset.DataSet in project deeplearning4j by deeplearning4j.

the class ParallelWrapperTest method testParallelWrapperRun.

@Test
public void testParallelWrapperRun() throws Exception {
    int nChannels = 1;
    int outputNum = 10;
    // for GPU you usually want to have higher batchSize
    int batchSize = 128;
    int nEpochs = 10;
    int iterations = 1;
    int seed = 123;
    log.info("Load data....");
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345);
    DataSetIterator mnistTest = new MnistDataSetIterator(batchSize, false, 12345);
    log.info("Build model....");
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations).regularization(true).l2(0.0005).learningRate(//.biasLearningRate(0.02)
    0.01).weightInit(WeightInit.XAVIER).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.NESTEROVS).momentum(0.9).list().layer(0, new ConvolutionLayer.Builder(5, 5).nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(2, new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()).layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()).layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).activation(Activation.SOFTMAX).build()).backprop(true).pretrain(false);
    // The builder needs the dimensions of the image along with the number of channels. these are 28x28 images in one channel
    new ConvolutionLayerSetup(builder, 28, 28, 1);
    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model).prefetchBuffer(24).workers(2).averagingFrequency(3).reportScoreAfterAveraging(true).useLegacyAveraging(true).build();
    log.info("Train model....");
    model.setListeners(new ScoreIterationListener(100));
    long timeX = System.currentTimeMillis();
    for (int i = 0; i < nEpochs; i++) {
        long time1 = System.currentTimeMillis();
        // Please note: we're feeding ParallelWrapper with iterator, not model directly
        //            wrapper.fit(mnistMultiEpochIterator);
        wrapper.fit(mnistTrain);
        long time2 = System.currentTimeMillis();
        log.info("*** Completed epoch {}, time: {} ***", i, (time2 - time1));
    }
    long timeY = System.currentTimeMillis();
    log.info("*** Training complete, time: {} ***", (timeY - timeX));
    log.info("Evaluate model....");
    Evaluation eval = new Evaluation(outputNum);
    while (mnistTest.hasNext()) {
        DataSet ds = mnistTest.next();
        INDArray output = model.output(ds.getFeatureMatrix(), false);
        eval.eval(ds.getLabels(), output);
    }
    log.info(eval.stats());
    mnistTest.reset();
    log.info("****************Example finished********************");
    wrapper.shutdown();
}