Examples with LOF de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LOF used on opensource projects

Example 1 with LOF

use of de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LOF in project elki by elki-project.

the class ComputeKNNOutlierScores method run.

@Override
public void run() {
    final Database database = inputstep.getDatabase();
    final Relation<O> relation = database.getRelation(distf.getInputTypeRestriction());
    // Ensure we don't go beyond the relation size:
    final int maxk = Math.min(this.maxk, relation.size() - 1);
    // Get a KNN query.
    final int lim = Math.min(maxk + 2, relation.size());
    KNNQuery<O> knnq = QueryUtil.getKNNQuery(relation, distf, lim);
    // Precompute kNN:
    if (!(knnq instanceof PreprocessorKNNQuery)) {
        MaterializeKNNPreprocessor<O> preproc = new MaterializeKNNPreprocessor<>(relation, distf, lim);
        preproc.initialize();
        relation.getHierarchy().add(relation, preproc);
    }
    // Test that we now get a proper index query
    knnq = QueryUtil.getKNNQuery(relation, distf, lim);
    if (!(knnq instanceof PreprocessorKNNQuery)) {
        throw new AbortException("Not using preprocessor knn query -- KNN queries using class: " + knnq.getClass());
    }
    // Warn for some known slow methods and large k:
    if (!isDisabled("LDOF") && maxk > 100) {
        LOG.verbose("Note: LODF needs O(k^2) distance computations. Use -" + Parameterizer.DISABLE_ID.getName() + " LDOF to disable.");
    }
    if (!isDisabled("FastABOD") && maxk > 100) {
        LOG.warning("Note: FastABOD needs quadratic memory. Use -" + Parameterizer.DISABLE_ID.getName() + " FastABOD to disable.");
    }
    if (!isDisabled("DWOF") && maxk > 100) {
        LOG.warning("Note: DWOF needs O(k^2) distance computations. Use -" + Parameterizer.DISABLE_ID.getName() + " DWOF to disable.");
    }
    final DBIDs ids = relation.getDBIDs();
    try (PrintStream fout = new PrintStream(outfile)) {
        // Control: print the DBIDs in case we are seeing an odd iteration
        // 
        fout.append("# Data set size: " + relation.size()).append(" data type: " + relation.getDataTypeInformation()).append(FormatUtil.NEWLINE);
        // Label outlier result (reference)
        writeResult(fout, ids, bylabel.run(database), new IdentityScaling(), "bylabel");
        final int startk = (this.startk > 0) ? this.startk : this.stepk;
        final int startkmin2 = (startk >= 2) ? startk : (startk + stepk);
        final int startkmin3 = (startk >= 3) ? startk : (startkmin2 >= 3) ? startkmin2 : (startkmin2 + stepk);
        // Output function:
        BiConsumer<String, OutlierResult> out = (kstr, result) -> writeResult(fout, ids, result, scaling, kstr);
        // KNN
        runForEachK(// 
        "KNN", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNOutlier<O>(distf, k).run(database, relation), out);
        // KNN Weight
        runForEachK(// 
        "KNNW", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNWeightOutlier<O>(distf, k).run(database, relation), out);
        // Run LOF
        runForEachK(// 
        "LOF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LOF<O>(k, distf).run(database, relation), out);
        // Run Simplified-LOF
        runForEachK(// 
        "SimplifiedLOF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new SimplifiedLOF<O>(k, distf).run(database, relation), out);
        // LoOP
        runForEachK(// 
        "LoOP", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LoOP<O>(k, k, distf, distf, 1.0).run(database, relation), out);
        // LDOF
        runForEachK(// 
        "LDOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new LDOF<O>(distf, k).run(database, relation), out);
        // Run ODIN
        runForEachK(// 
        "ODIN", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new ODIN<O>(distf, k).run(database, relation), out);
        // Run FastABOD
        runForEachK(// 
        "FastABOD", // 
        startkmin3, // 
        stepk, // 
        maxk, k -> // 
        new FastABOD<O>(new PolynomialKernelFunction(2), k).run(database, relation), out);
        // Run KDEOS with intrinsic dimensionality 2.
        runForEachK(// 
        "KDEOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> new // 
        KDEOS<O>(// 
        distf, // 
        k, // 
        k, // 
        GaussianKernelDensityFunction.KERNEL, // 
        0., 0.5 * GaussianKernelDensityFunction.KERNEL.canonicalBandwidth(), // 
        2).run(database, relation), out);
        // Run LDF
        runForEachK(// 
        "LDF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LDF<O>(k, distf, GaussianKernelDensityFunction.KERNEL, 1., .1).run(database, relation), out);
        // Run INFLO
        runForEachK(// 
        "INFLO", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new INFLO<O>(distf, 1.0, k).run(database, relation), out);
        // Run COF
        runForEachK(// 
        "COF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new COF<O>(k, distf).run(database, relation), out);
        // Run simple Intrinsic dimensionality
        runForEachK(// 
        "Intrinsic", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new IntrinsicDimensionalityOutlier<O>(distf, k, AggregatedHillEstimator.STATIC).run(database, relation), out);
        // Run IDOS
        runForEachK(// 
        "IDOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new IDOS<O>(distf, AggregatedHillEstimator.STATIC, k, k).run(database, relation), out);
        // Run simple kernel-density LOF variant
        runForEachK(// 
        "KDLOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new SimpleKernelDensityLOF<O>(k, distf, GaussianKernelDensityFunction.KERNEL).run(database, relation), out);
        // Run DWOF (need pairwise distances, too)
        runForEachK(// 
        "DWOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new DWOF<O>(distf, k, 1.1).run(database, relation), out);
        // Run LIC
        runForEachK(// 
        "LIC", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LocalIsolationCoefficient<O>(distf, k).run(database, relation), out);
        // Run VOV (requires a vector field).
        if (TypeUtil.DOUBLE_VECTOR_FIELD.isAssignableFromType(relation.getDataTypeInformation())) {
            @SuppressWarnings("unchecked") final DistanceFunction<? super DoubleVector> df = (DistanceFunction<? super DoubleVector>) distf;
            @SuppressWarnings("unchecked") final Relation<DoubleVector> rel = (Relation<DoubleVector>) (Relation<?>) relation;
            runForEachK(// 
            "VOV", // 
            startk, // 
            stepk, // 
            maxk, k -> // 
            new VarianceOfVolume<DoubleVector>(k, df).run(database, rel), out);
        }
        // Run KNN DD
        runForEachK(// 
        "KNNDD", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNDD<O>(distf, k).run(database, relation), out);
        // Run KNN SOS
        runForEachK(// 
        "KNNSOS", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNSOS<O>(distf, k).run(relation), out);
        // Run ISOS
        runForEachK(// 
        "ISOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new ISOS<O>(distf, k, AggregatedHillEstimator.STATIC).run(relation), out);
    } catch (FileNotFoundException e) {
        throw new AbortException("Cannot create output file.", e);
    }
}

Example 2 with LOF

use of de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LOF in project elki by elki-project.

the class FeatureBagging method run.

/**
 * Run the algorithm on a data set.
 *
 * @param database Database context
 * @param relation Relation to use
 * @return Outlier detection result
 */
public OutlierResult run(Database database, Relation<NumberVector> relation) {
    final int dbdim = RelationUtil.dimensionality(relation);
    final int mindim = dbdim >> 1;
    final int maxdim = dbdim - 1;
    final Random rand = rnd.getSingleThreadedRandom();
    ArrayList<OutlierResult> results = new ArrayList<>(num);
    {
        FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("LOF iterations", num, LOG) : null;
        for (int i = 0; i < num; i++) {
            long[] dimset = randomSubspace(dbdim, mindim, maxdim, rand);
            SubspaceEuclideanDistanceFunction df = new SubspaceEuclideanDistanceFunction(dimset);
            LOF<NumberVector> lof = new LOF<>(k, df);
            // run LOF and collect the result
            OutlierResult result = lof.run(database, relation);
            results.add(result);
            LOG.incrementProcessed(prog);
        }
        LOG.ensureCompleted(prog);
    }
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
    DoubleMinMax minmax = new DoubleMinMax();
    if (breadth) {
        FiniteProgress cprog = LOG.isVerbose() ? new FiniteProgress("Combining results", relation.size(), LOG) : null;
        @SuppressWarnings("unchecked") Pair<DBIDIter, DoubleRelation>[] IDVectorOntoScoreVector = (Pair<DBIDIter, DoubleRelation>[]) new Pair[results.size()];
        // Mapping score-sorted DBID-Iterators onto their corresponding scores.
        // We need to initialize them now be able to iterate them "in parallel".
        {
            int i = 0;
            for (OutlierResult r : results) {
                IDVectorOntoScoreVector[i] = new Pair<DBIDIter, DoubleRelation>(r.getOrdering().order(relation.getDBIDs()).iter(), r.getScores());
                i++;
            }
        }
        // Iterating over the *lines* of the AS_t(i)-matrix.
        for (int i = 0; i < relation.size(); i++) {
            // Iterating over the elements of a line (breadth-first).
            for (Pair<DBIDIter, DoubleRelation> pair : IDVectorOntoScoreVector) {
                DBIDIter iter = pair.first;
                // for every DBID).
                if (iter.valid()) {
                    double score = pair.second.doubleValue(iter);
                    if (Double.isNaN(scores.doubleValue(iter))) {
                        scores.putDouble(iter, score);
                        minmax.put(score);
                    }
                    iter.advance();
                } else {
                    LOG.warning("Incomplete result: Iterator does not contain |DB| DBIDs");
                }
            }
            // Progress does not take the initial mapping into account.
            LOG.incrementProcessed(cprog);
        }
        LOG.ensureCompleted(cprog);
    } else {
        FiniteProgress cprog = LOG.isVerbose() ? new FiniteProgress("Combining results", relation.size(), LOG) : null;
        for (DBIDIter iter = relation.iterDBIDs(); iter.valid(); iter.advance()) {
            double sum = 0.0;
            for (OutlierResult r : results) {
                final double s = r.getScores().doubleValue(iter);
                if (!Double.isNaN(s)) {
                    sum += s;
                }
            }
            scores.putDouble(iter, sum);
            minmax.put(sum);
            LOG.incrementProcessed(cprog);
        }
        LOG.ensureCompleted(cprog);
    }
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax());
    DoubleRelation scoreres = new MaterializedDoubleRelation("Feature bagging", "fb-outlier", scores, relation.getDBIDs());
    return new OutlierResult(meta, scoreres);
}