Search in sources :

Example 1 with ODIN

use of de.lmu.ifi.dbs.elki.algorithm.outlier.distance.ODIN in project elki by elki-project.

the class ComputeKNNOutlierScores method run.

@Override
public void run() {
    final Database database = inputstep.getDatabase();
    final Relation<O> relation = database.getRelation(distf.getInputTypeRestriction());
    // Ensure we don't go beyond the relation size:
    final int maxk = Math.min(this.maxk, relation.size() - 1);
    // Get a KNN query.
    final int lim = Math.min(maxk + 2, relation.size());
    KNNQuery<O> knnq = QueryUtil.getKNNQuery(relation, distf, lim);
    // Precompute kNN:
    if (!(knnq instanceof PreprocessorKNNQuery)) {
        MaterializeKNNPreprocessor<O> preproc = new MaterializeKNNPreprocessor<>(relation, distf, lim);
        preproc.initialize();
        relation.getHierarchy().add(relation, preproc);
    }
    // Test that we now get a proper index query
    knnq = QueryUtil.getKNNQuery(relation, distf, lim);
    if (!(knnq instanceof PreprocessorKNNQuery)) {
        throw new AbortException("Not using preprocessor knn query -- KNN queries using class: " + knnq.getClass());
    }
    // Warn for some known slow methods and large k:
    if (!isDisabled("LDOF") && maxk > 100) {
        LOG.verbose("Note: LODF needs O(k^2) distance computations. Use -" + Parameterizer.DISABLE_ID.getName() + " LDOF to disable.");
    }
    if (!isDisabled("FastABOD") && maxk > 100) {
        LOG.warning("Note: FastABOD needs quadratic memory. Use -" + Parameterizer.DISABLE_ID.getName() + " FastABOD to disable.");
    }
    if (!isDisabled("DWOF") && maxk > 100) {
        LOG.warning("Note: DWOF needs O(k^2) distance computations. Use -" + Parameterizer.DISABLE_ID.getName() + " DWOF to disable.");
    }
    final DBIDs ids = relation.getDBIDs();
    try (PrintStream fout = new PrintStream(outfile)) {
        // Control: print the DBIDs in case we are seeing an odd iteration
        // 
        fout.append("# Data set size: " + relation.size()).append(" data type: " + relation.getDataTypeInformation()).append(FormatUtil.NEWLINE);
        // Label outlier result (reference)
        writeResult(fout, ids, bylabel.run(database), new IdentityScaling(), "bylabel");
        final int startk = (this.startk > 0) ? this.startk : this.stepk;
        final int startkmin2 = (startk >= 2) ? startk : (startk + stepk);
        final int startkmin3 = (startk >= 3) ? startk : (startkmin2 >= 3) ? startkmin2 : (startkmin2 + stepk);
        // Output function:
        BiConsumer<String, OutlierResult> out = (kstr, result) -> writeResult(fout, ids, result, scaling, kstr);
        // KNN
        runForEachK(// 
        "KNN", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNOutlier<O>(distf, k).run(database, relation), out);
        // KNN Weight
        runForEachK(// 
        "KNNW", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNWeightOutlier<O>(distf, k).run(database, relation), out);
        // Run LOF
        runForEachK(// 
        "LOF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LOF<O>(k, distf).run(database, relation), out);
        // Run Simplified-LOF
        runForEachK(// 
        "SimplifiedLOF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new SimplifiedLOF<O>(k, distf).run(database, relation), out);
        // LoOP
        runForEachK(// 
        "LoOP", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LoOP<O>(k, k, distf, distf, 1.0).run(database, relation), out);
        // LDOF
        runForEachK(// 
        "LDOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new LDOF<O>(distf, k).run(database, relation), out);
        // Run ODIN
        runForEachK(// 
        "ODIN", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new ODIN<O>(distf, k).run(database, relation), out);
        // Run FastABOD
        runForEachK(// 
        "FastABOD", // 
        startkmin3, // 
        stepk, // 
        maxk, k -> // 
        new FastABOD<O>(new PolynomialKernelFunction(2), k).run(database, relation), out);
        // Run KDEOS with intrinsic dimensionality 2.
        runForEachK(// 
        "KDEOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> new // 
        KDEOS<O>(// 
        distf, // 
        k, // 
        k, // 
        GaussianKernelDensityFunction.KERNEL, // 
        0., 0.5 * GaussianKernelDensityFunction.KERNEL.canonicalBandwidth(), // 
        2).run(database, relation), out);
        // Run LDF
        runForEachK(// 
        "LDF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LDF<O>(k, distf, GaussianKernelDensityFunction.KERNEL, 1., .1).run(database, relation), out);
        // Run INFLO
        runForEachK(// 
        "INFLO", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new INFLO<O>(distf, 1.0, k).run(database, relation), out);
        // Run COF
        runForEachK(// 
        "COF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new COF<O>(k, distf).run(database, relation), out);
        // Run simple Intrinsic dimensionality
        runForEachK(// 
        "Intrinsic", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new IntrinsicDimensionalityOutlier<O>(distf, k, AggregatedHillEstimator.STATIC).run(database, relation), out);
        // Run IDOS
        runForEachK(// 
        "IDOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new IDOS<O>(distf, AggregatedHillEstimator.STATIC, k, k).run(database, relation), out);
        // Run simple kernel-density LOF variant
        runForEachK(// 
        "KDLOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new SimpleKernelDensityLOF<O>(k, distf, GaussianKernelDensityFunction.KERNEL).run(database, relation), out);
        // Run DWOF (need pairwise distances, too)
        runForEachK(// 
        "DWOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new DWOF<O>(distf, k, 1.1).run(database, relation), out);
        // Run LIC
        runForEachK(// 
        "LIC", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LocalIsolationCoefficient<O>(distf, k).run(database, relation), out);
        // Run VOV (requires a vector field).
        if (TypeUtil.DOUBLE_VECTOR_FIELD.isAssignableFromType(relation.getDataTypeInformation())) {
            @SuppressWarnings("unchecked") final DistanceFunction<? super DoubleVector> df = (DistanceFunction<? super DoubleVector>) distf;
            @SuppressWarnings("unchecked") final Relation<DoubleVector> rel = (Relation<DoubleVector>) (Relation<?>) relation;
            runForEachK(// 
            "VOV", // 
            startk, // 
            stepk, // 
            maxk, k -> // 
            new VarianceOfVolume<DoubleVector>(k, df).run(database, rel), out);
        }
        // Run KNN DD
        runForEachK(// 
        "KNNDD", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNDD<O>(distf, k).run(database, relation), out);
        // Run KNN SOS
        runForEachK(// 
        "KNNSOS", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNSOS<O>(distf, k).run(relation), out);
        // Run ISOS
        runForEachK(// 
        "ISOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new ISOS<O>(distf, k, AggregatedHillEstimator.STATIC).run(relation), out);
    } catch (FileNotFoundException e) {
        throw new AbortException("Cannot create output file.", e);
    }
}
Also used : PreprocessorKNNQuery(de.lmu.ifi.dbs.elki.database.query.knn.PreprocessorKNNQuery) InputStep(de.lmu.ifi.dbs.elki.workflow.InputStep) NumberVector(de.lmu.ifi.dbs.elki.data.NumberVector) LDOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDOF) DoubleRelation(de.lmu.ifi.dbs.elki.database.relation.DoubleRelation) Reference(de.lmu.ifi.dbs.elki.utilities.documentation.Reference) COF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.COF) ObjectParameter(de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter) DoubleVector(de.lmu.ifi.dbs.elki.data.DoubleVector) Locale(java.util.Locale) KNNWeightOutlier(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNWeightOutlier) FastMath(net.jafama.FastMath) LDF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDF) ByLabelOutlier(de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.ByLabelOutlier) DistanceFunction(de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction) FastABOD(de.lmu.ifi.dbs.elki.algorithm.outlier.anglebased.FastABOD) OptionID(de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID) DistanceBasedAlgorithm(de.lmu.ifi.dbs.elki.algorithm.DistanceBasedAlgorithm) FormatUtil(de.lmu.ifi.dbs.elki.utilities.io.FormatUtil) ScalingFunction(de.lmu.ifi.dbs.elki.utilities.scaling.ScalingFunction) LoOP(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LoOP) TypeUtil(de.lmu.ifi.dbs.elki.data.type.TypeUtil) EuclideanDistanceFunction(de.lmu.ifi.dbs.elki.distance.distancefunction.minkowski.EuclideanDistanceFunction) Database(de.lmu.ifi.dbs.elki.database.Database) AggregatedHillEstimator(de.lmu.ifi.dbs.elki.math.statistics.intrinsicdimensionality.AggregatedHillEstimator) PolynomialKernelFunction(de.lmu.ifi.dbs.elki.distance.similarityfunction.kernel.PolynomialKernelFunction) KNNDD(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNDD) FileNotFoundException(java.io.FileNotFoundException) ISOS(de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.ISOS) DBIDIter(de.lmu.ifi.dbs.elki.database.ids.DBIDIter) OutlierResult(de.lmu.ifi.dbs.elki.result.outlier.OutlierResult) IntParameter(de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter) IdentityScaling(de.lmu.ifi.dbs.elki.utilities.scaling.IdentityScaling) Pattern(java.util.regex.Pattern) LOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LOF) DBIDs(de.lmu.ifi.dbs.elki.database.ids.DBIDs) Parameterization(de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization) KNNOutlier(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNOutlier) KDEOS(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.KDEOS) QueryUtil(de.lmu.ifi.dbs.elki.database.QueryUtil) BiConsumer(java.util.function.BiConsumer) GaussianKernelDensityFunction(de.lmu.ifi.dbs.elki.math.statistics.kernelfunctions.GaussianKernelDensityFunction) CommonConstraints(de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.CommonConstraints) AbortException(de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException) INFLO(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.INFLO) IntFunction(java.util.function.IntFunction) PrintStream(java.io.PrintStream) Relation(de.lmu.ifi.dbs.elki.database.relation.Relation) MaterializeKNNPreprocessor(de.lmu.ifi.dbs.elki.index.preprocessed.knn.MaterializeKNNPreprocessor) AbstractApplication(de.lmu.ifi.dbs.elki.application.AbstractApplication) IntrinsicDimensionalityOutlier(de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.IntrinsicDimensionalityOutlier) File(java.io.File) Duration(de.lmu.ifi.dbs.elki.logging.statistics.Duration) ODIN(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.ODIN) SimplifiedLOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.SimplifiedLOF) DWOF(de.lmu.ifi.dbs.elki.algorithm.outlier.DWOF) LocalIsolationCoefficient(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.LocalIsolationCoefficient) VarianceOfVolume(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.VarianceOfVolume) SimpleKernelDensityLOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.SimpleKernelDensityLOF) IDOS(de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.IDOS) KNNSOS(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNSOS) Logging(de.lmu.ifi.dbs.elki.logging.Logging) OutlierScalingFunction(de.lmu.ifi.dbs.elki.utilities.scaling.outlier.OutlierScalingFunction) PatternParameter(de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.PatternParameter) KNNQuery(de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery) IdentityScaling(de.lmu.ifi.dbs.elki.utilities.scaling.IdentityScaling) PolynomialKernelFunction(de.lmu.ifi.dbs.elki.distance.similarityfunction.kernel.PolynomialKernelFunction) OutlierResult(de.lmu.ifi.dbs.elki.result.outlier.OutlierResult) FileNotFoundException(java.io.FileNotFoundException) DWOF(de.lmu.ifi.dbs.elki.algorithm.outlier.DWOF) SimpleKernelDensityLOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.SimpleKernelDensityLOF) MaterializeKNNPreprocessor(de.lmu.ifi.dbs.elki.index.preprocessed.knn.MaterializeKNNPreprocessor) KNNDD(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNDD) INFLO(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.INFLO) LDF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDF) SimplifiedLOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.SimplifiedLOF) DoubleRelation(de.lmu.ifi.dbs.elki.database.relation.DoubleRelation) Relation(de.lmu.ifi.dbs.elki.database.relation.Relation) IntrinsicDimensionalityOutlier(de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.IntrinsicDimensionalityOutlier) Database(de.lmu.ifi.dbs.elki.database.Database) COF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.COF) IDOS(de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.IDOS) ODIN(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.ODIN) LoOP(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LoOP) PrintStream(java.io.PrintStream) LOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LOF) SimplifiedLOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.SimplifiedLOF) SimpleKernelDensityLOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.SimpleKernelDensityLOF) DBIDs(de.lmu.ifi.dbs.elki.database.ids.DBIDs) KNNSOS(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNSOS) KNNWeightOutlier(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNWeightOutlier) VarianceOfVolume(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.VarianceOfVolume) LDOF(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDOF) ISOS(de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.ISOS) INFLO(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.INFLO) KNNOutlier(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNOutlier) DistanceFunction(de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction) EuclideanDistanceFunction(de.lmu.ifi.dbs.elki.distance.distancefunction.minkowski.EuclideanDistanceFunction) KDEOS(de.lmu.ifi.dbs.elki.algorithm.outlier.lof.KDEOS) PreprocessorKNNQuery(de.lmu.ifi.dbs.elki.database.query.knn.PreprocessorKNNQuery) LocalIsolationCoefficient(de.lmu.ifi.dbs.elki.algorithm.outlier.distance.LocalIsolationCoefficient) DoubleVector(de.lmu.ifi.dbs.elki.data.DoubleVector) AbortException(de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException) FastABOD(de.lmu.ifi.dbs.elki.algorithm.outlier.anglebased.FastABOD)

Aggregations

DistanceBasedAlgorithm (de.lmu.ifi.dbs.elki.algorithm.DistanceBasedAlgorithm)1 DWOF (de.lmu.ifi.dbs.elki.algorithm.outlier.DWOF)1 FastABOD (de.lmu.ifi.dbs.elki.algorithm.outlier.anglebased.FastABOD)1 KNNDD (de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNDD)1 KNNOutlier (de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNOutlier)1 KNNSOS (de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNSOS)1 KNNWeightOutlier (de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNWeightOutlier)1 LocalIsolationCoefficient (de.lmu.ifi.dbs.elki.algorithm.outlier.distance.LocalIsolationCoefficient)1 ODIN (de.lmu.ifi.dbs.elki.algorithm.outlier.distance.ODIN)1 IDOS (de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.IDOS)1 ISOS (de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.ISOS)1 IntrinsicDimensionalityOutlier (de.lmu.ifi.dbs.elki.algorithm.outlier.intrinsic.IntrinsicDimensionalityOutlier)1 COF (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.COF)1 INFLO (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.INFLO)1 KDEOS (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.KDEOS)1 LDF (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDF)1 LDOF (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDOF)1 LOF (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LOF)1 LoOP (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LoOP)1 SimpleKernelDensityLOF (de.lmu.ifi.dbs.elki.algorithm.outlier.lof.SimpleKernelDensityLOF)1