Examples with FiniteProgress de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress used on opensource projects

Example 81 with FiniteProgress

use of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress in project elki by elki-project.

the class IDOS method computeIDs.

/**
 * Computes all IDs
 *
 * @param ids the DBIDs to process
 * @param knnQ the KNN query
 * @return The computed intrinsic dimensionalities.
 */
protected DoubleDataStore computeIDs(DBIDs ids, KNNQuery<O> knnQ) {
    WritableDoubleDataStore intDims = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
    FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Intrinsic dimensionality", ids.size(), LOG) : null;
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        double id = 0.;
        try {
            id = estimator.estimate(knnQ, iter, k_c + 1);
        } catch (ArithmeticException e) {
            // Too many duplicates, etc.
            id = 0;
        }
        intDims.putDouble(iter, id);
        LOG.incrementProcessed(prog);
    }
    LOG.ensureCompleted(prog);
    return intDims;
}

Example 82 with FiniteProgress

use of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress in project elki by elki-project.

the class FlexibleLOF method computeLOFs.

/**
 * Computes the Local outlier factor (LOF) of the specified objects.
 *
 * @param knnq the precomputed neighborhood of the objects w.r.t. the
 *        reference distance
 * @param ids IDs to process
 * @param lrds Local reachability distances
 * @param lofs Local outlier factor storage
 * @param lofminmax Score minimum/maximum tracker
 */
protected void computeLOFs(KNNQuery<O> knnq, DBIDs ids, DoubleDataStore lrds, WritableDoubleDataStore lofs, DoubleMinMax lofminmax) {
    FiniteProgress progressLOFs = LOG.isVerbose() ? new FiniteProgress("LOF_SCORE for objects", ids.size(), LOG) : null;
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        final double lof;
        final double lrdp = lrds.doubleValue(iter);
        final KNNList neighbors = knnq.getKNNForDBID(iter, krefer);
        if (!Double.isInfinite(lrdp)) {
            double sum = 0.;
            int count = 0;
            for (DBIDIter neighbor = neighbors.iter(); neighbor.valid(); neighbor.advance()) {
                // skip the point itself
                if (DBIDUtil.equal(neighbor, iter)) {
                    continue;
                }
                final double val = lrds.doubleValue(neighbor);
                sum += val;
                count++;
                if (Double.isInfinite(val)) {
                    break;
                }
            }
            lof = sum / (lrdp * count);
        } else {
            lof = 1.0;
        }
        lofs.putDouble(iter, lof);
        // update minimum and maximum
        lofminmax.put(lof);
        LOG.incrementProcessed(progressLOFs);
    }
    LOG.ensureCompleted(progressLOFs);
}

Example 83 with FiniteProgress

use of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress in project elki by elki-project.

the class KDEOS method estimateDensities.

/**
 * Perform the kernel density estimation step.
 *
 * @param rel Relation to query
 * @param knnq kNN query
 * @param ids IDs to process
 * @param densities Density storage
 */
protected void estimateDensities(Relation<O> rel, KNNQuery<O> knnq, final DBIDs ids, WritableDataStore<double[]> densities) {
    final int dim = dimensionality(rel);
    final int knum = kmax + 1 - kmin;
    // Initialize storage:
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        densities.put(iter, new double[knum]);
    }
    // Distribute densities:
    FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Computing densities", ids.size(), LOG) : null;
    double iminbw = (minBandwidth > 0.) ? 1. / (minBandwidth * scale) : Double.POSITIVE_INFINITY;
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        KNNList neighbors = knnq.getKNNForDBID(iter, kmax + 1);
        int k = 1, idx = 0;
        double sum = 0.;
        for (DoubleDBIDListIter kneighbor = neighbors.iter(); k <= kmax && kneighbor.valid(); kneighbor.advance(), k++) {
            sum += kneighbor.doubleValue();
            if (k < kmin) {
                continue;
            }
            final double ibw = Math.min(k / (sum * scale), iminbw);
            final double sca = MathUtil.powi(ibw, dim);
            for (DoubleDBIDListIter neighbor = neighbors.iter(); neighbor.valid(); neighbor.advance()) {
                final double dens;
                if (sca < Double.POSITIVE_INFINITY) {
                    // NaNs with duplicate points!
                    dens = sca * kernel.density(neighbor.doubleValue() * ibw);
                } else {
                    dens = neighbor.doubleValue() == 0. ? 1. : 0.;
                }
                densities.get(neighbor)[idx] += dens;
                if (dens < CUTOFF) {
                    break;
                }
            }
            // Only if k >= kmin
            ++idx;
        }
        LOG.incrementProcessed(prog);
    }
    LOG.ensureCompleted(prog);
}

Example 84 with FiniteProgress

use of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress in project elki by elki-project.

the class KDEOS method computeOutlierScores.

/**
 * Compute the final KDEOS scores.
 *
 * @param knnq kNN query
 * @param ids IDs to process
 * @param densities Density estimates
 * @param kdeos Score outputs
 * @param minmax Minimum and maximum scores
 */
protected void computeOutlierScores(KNNQuery<O> knnq, final DBIDs ids, WritableDataStore<double[]> densities, WritableDoubleDataStore kdeos, DoubleMinMax minmax) {
    final int knum = kmax + 1 - kmin;
    FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Computing KDEOS scores", ids.size(), LOG) : null;
    double[][] scratch = new double[knum][kmax + 5];
    MeanVariance mv = new MeanVariance();
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        double[] dens = densities.get(iter);
        KNNList neighbors = knnq.getKNNForDBID(iter, kmax + 1);
        if (scratch[0].length < neighbors.size()) {
            // Resize scratch. Add some extra margin again.
            scratch = new double[knum][neighbors.size() + 5];
        }
        {
            // Store density matrix of neighbors
            int i = 0;
            for (DoubleDBIDListIter neighbor = neighbors.iter(); neighbor.valid(); neighbor.advance(), i++) {
                double[] ndens = densities.get(neighbor);
                for (int k = 0; k < knum; k++) {
                    scratch[k][i] = ndens[k];
                }
            }
            assert (i == neighbors.size());
        }
        // Compute means and stddevs for each k
        double score = 0.;
        for (int i = 0; i < knum; i++) {
            mv.reset();
            for (int j = 0; j < neighbors.size(); j++) {
                mv.put(scratch[i][j]);
            }
            final double mean = mv.getMean(), stddev = mv.getSampleStddev();
            if (stddev > 0.) {
                score += (mean - dens[i]) / stddev;
            }
        }
        // average
        score /= knum;
        score = NormalDistribution.standardNormalCDF(score);
        minmax.put(score);
        kdeos.put(iter, score);
        LOG.incrementProcessed(prog);
    }
    LOG.ensureCompleted(prog);
}

Example 85 with FiniteProgress

use of de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress in project elki by elki-project.

the class LOF method computeLRDs.

/**
 * Compute local reachability distances.
 *
 * @param knnq KNN query
 * @param ids IDs to process
 * @param lrds Reachability storage
 */
private void computeLRDs(KNNQuery<O> knnq, DBIDs ids, WritableDoubleDataStore lrds) {
    FiniteProgress lrdsProgress = LOG.isVerbose() ? new FiniteProgress("Local Reachability Densities (LRD)", ids.size(), LOG) : null;
    double lrd;
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        lrd = computeLRD(knnq, iter);
        lrds.putDouble(iter, lrd);
        LOG.incrementProcessed(lrdsProgress);
    }
    LOG.ensureCompleted(lrdsProgress);
}