Examples with INDArray org.nd4j.linalg.api.ndarray.INDArray used on opensource projects

Example 81 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class ScoreFlatMapFunctionCGDataSetAdapter method call.

@Override
public Iterable<Tuple2<Integer, Double>> call(Iterator<DataSet> dataSetIterator) throws Exception {
    if (!dataSetIterator.hasNext()) {
        return Collections.singletonList(new Tuple2<>(0, 0.0));
    }
    //Does batching where appropriate
    DataSetIterator iter = new IteratorDataSetIterator(dataSetIterator, minibatchSize);
    ComputationGraph network = new ComputationGraph(ComputationGraphConfiguration.fromJson(json));
    network.init();
    //.value() is shared by all executors on single machine -> OK, as params are not changed in score function
    INDArray val = params.value().unsafeDuplication();
    if (val.length() != network.numParams(false))
        throw new IllegalStateException("Network did not have same number of parameters as the broadcast set parameters");
    network.setParams(val);
    List<Tuple2<Integer, Double>> out = new ArrayList<>();
    while (iter.hasNext()) {
        DataSet ds = iter.next();
        double score = network.score(ds, false);
        int numExamples = ds.getFeatureMatrix().size(0);
        out.add(new Tuple2<>(numExamples, score * numExamples));
    }
    if (Nd4j.getExecutioner() instanceof GridExecutioner)
        ((GridExecutioner) Nd4j.getExecutioner()).flushQueueBlocking();
    return out;
}

Example 82 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class ScoreFlatMapFunctionCGMultiDataSetAdapter method call.

@Override
public Iterable<Tuple2<Integer, Double>> call(Iterator<MultiDataSet> dataSetIterator) throws Exception {
    if (!dataSetIterator.hasNext()) {
        return Collections.singletonList(new Tuple2<>(0, 0.0));
    }
    //Does batching where appropriate
    MultiDataSetIterator iter = new IteratorMultiDataSetIterator(dataSetIterator, minibatchSize);
    ComputationGraph network = new ComputationGraph(ComputationGraphConfiguration.fromJson(json));
    network.init();
    //.value() is shared by all executors on single machine -> OK, as params are not changed in score function
    INDArray val = params.value().unsafeDuplication();
    if (val.length() != network.numParams(false))
        throw new IllegalStateException("Network did not have same number of parameters as the broadcast set parameters");
    network.setParams(val);
    List<Tuple2<Integer, Double>> out = new ArrayList<>();
    while (iter.hasNext()) {
        MultiDataSet ds = iter.next();
        double score = network.score(ds, false);
        int numExamples = ds.getFeatures(0).size(0);
        out.add(new Tuple2<>(numExamples, score * numExamples));
    }
    if (Nd4j.getExecutioner() instanceof GridExecutioner)
        ((GridExecutioner) Nd4j.getExecutioner()).flushQueueBlocking();
    return out;
}

Example 83 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class DataVecSequenceDataSetFunction method call.

@Override
public DataSet call(List<List<Writable>> input) throws Exception {
    Iterator<List<Writable>> iter = input.iterator();
    INDArray features = null;
    INDArray labels = Nd4j.zeros(1, (regression ? 1 : numPossibleLabels), input.size());
    int[] fIdx = new int[3];
    int[] lIdx = new int[3];
    int i = 0;
    while (iter.hasNext()) {
        List<Writable> step = iter.next();
        if (i == 0) {
            features = Nd4j.zeros(1, step.size() - 1, input.size());
        }
        Iterator<Writable> timeStepIter = step.iterator();
        int countIn = 0;
        int countFeatures = 0;
        while (timeStepIter.hasNext()) {
            Writable current = timeStepIter.next();
            if (converter != null)
                current = converter.convert(current);
            if (countIn++ == labelIndex) {
                //label
                if (regression) {
                    lIdx[2] = i;
                    labels.putScalar(lIdx, current.toDouble());
                } else {
                    INDArray line = FeatureUtil.toOutcomeVector(current.toInt(), numPossibleLabels);
                    //1d from [1,nOut,timeSeriesLength] -> tensor i along dimension 1 is at time i
                    labels.tensorAlongDimension(i, 1).assign(line);
                }
            } else {
                //feature
                fIdx[1] = countFeatures++;
                fIdx[2] = i;
                try {
                    features.putScalar(fIdx, current.toDouble());
                } catch (UnsupportedOperationException e) {
                    // This isn't a scalar, so check if we got an array already
                    if (current instanceof NDArrayWritable) {
                        features.get(NDArrayIndex.point(fIdx[0]), NDArrayIndex.all(), NDArrayIndex.point(fIdx[2])).putRow(0, ((NDArrayWritable) current).get());
                    } else {
                        throw e;
                    }
                }
            }
        }
        i++;
    }
    DataSet ds = new DataSet(features, labels);
    if (preProcessor != null)
        preProcessor.preProcess(ds);
    return ds;
}

Example 84 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class BarnesHutTsne method computeGaussianPerplexity.

/**
     * Convert data to probability
     * co-occurrences (aka calculating the kernel)
     * @param d the data to convert
     * @param u the perplexity of the model
     * @return the probabilities of co-occurrence
     */
public INDArray computeGaussianPerplexity(final INDArray d, double u) {
    N = d.rows();
    final int k = (int) (3 * u);
    if (u > k)
        throw new IllegalStateException("Illegal k value " + k + "greater than " + u);
    rows = zeros(1, N + 1);
    cols = zeros(1, N * k);
    vals = zeros(1, N * k);
    for (int n = 0; n < N; n++) rows.putScalar(n + 1, rows.getDouble(n) + k);
    final INDArray beta = ones(N, 1);
    final double logU = FastMath.log(u);
    VPTree tree = new VPTree(d, simiarlityFunction, invert);
    log.info("Calculating probabilities of data similarities...");
    for (int i = 0; i < N; i++) {
        if (i % 500 == 0)
            log.info("Handled " + i + " records");
        double betaMin = -Double.MAX_VALUE;
        double betaMax = Double.MAX_VALUE;
        List<DataPoint> results = new ArrayList<>();
        tree.search(new DataPoint(i, d.slice(i)), k + 1, results, new ArrayList<Double>());
        double betas = beta.getDouble(i);
        INDArray cArr = VPTree.buildFromData(results);
        Pair<INDArray, Double> pair = computeGaussianKernel(cArr, beta.getDouble(i), k);
        INDArray currP = pair.getFirst();
        double hDiff = pair.getSecond() - logU;
        int tries = 0;
        boolean found = false;
        //binary search
        while (!found && tries < 200) {
            if (hDiff < tolerance && -hDiff < tolerance)
                found = true;
            else {
                if (hDiff > 0) {
                    betaMin = betas;
                    if (betaMax == Double.MAX_VALUE || betaMax == -Double.MAX_VALUE)
                        betas *= 2;
                    else
                        betas = (betas + betaMax) / 2.0;
                } else {
                    betaMax = betas;
                    if (betaMin == -Double.MAX_VALUE || betaMin == Double.MAX_VALUE)
                        betas /= 2.0;
                    else
                        betas = (betas + betaMin) / 2.0;
                }
                pair = computeGaussianKernel(cArr, betas, k);
                hDiff = pair.getSecond() - logU;
                tries++;
            }
        }
        currP.divi(currP.sum(Integer.MAX_VALUE));
        INDArray indices = Nd4j.create(1, k + 1);
        for (int j = 0; j < indices.length(); j++) {
            if (j >= results.size())
                break;
            indices.putScalar(j, results.get(j).getIndex());
        }
        for (int l = 0; l < k; l++) {
            cols.putScalar(rows.getInt(i) + l, indices.getDouble(l + 1));
            vals.putScalar(rows.getInt(i) + l, currP.getDouble(l));
        }
    }
    return vals;
}

Example 85 with INDArray

use of org.nd4j.linalg.api.ndarray.INDArray in project deeplearning4j by deeplearning4j.

the class BarnesHutTsne method update.

@Override
public void update(INDArray gradient, String paramType) {
    INDArray yGrads = gradient;
    gains = gains.add(.2).muli(sign(yGrads)).neqi(sign(yIncs)).addi(gains.mul(0.8).muli(sign(yGrads)).neqi(sign(yIncs)));
    BooleanIndexing.applyWhere(gains, Conditions.lessThan(minGain), new Value(minGain));
    INDArray gradChange = gains.mul(yGrads);
    if (useAdaGrad) {
        if (adaGrad == null)
            adaGrad = new AdaGrad();
        gradChange = adaGrad.getGradient(gradChange, 0);
    } else
        gradChange.muli(learningRate);
    yIncs.muli(momentum).subi(gradChange);
    Y.addi(yIncs);
}