Examples with DBIDsTest de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DBIDsTest used on opensource projects

Example 1 with DBIDsTest

use of de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DBIDsTest in project elki by elki-project.

the class ROCEvaluationTest method testROCCurve.

/**
 * Test ROC curve generation, including curve simplification
 */
@Test
public void testROCCurve() {
    HashSetModifiableDBIDs positive = DBIDUtil.newHashSet();
    positive.add(DBIDUtil.importInteger(1));
    positive.add(DBIDUtil.importInteger(2));
    positive.add(DBIDUtil.importInteger(3));
    positive.add(DBIDUtil.importInteger(4));
    positive.add(DBIDUtil.importInteger(5));
    final ModifiableDoubleDBIDList distances = DBIDUtil.newDistanceDBIDList();
    // Starting point: ................................ 0.0,0. ++
    // + 0.0,.2 -- redundant
    distances.add(0.0, DBIDUtil.importInteger(1));
    // + 0.0,.4 ++
    distances.add(1.0, DBIDUtil.importInteger(2));
    // - .25,.4 ++
    distances.add(2.0, DBIDUtil.importInteger(6));
    // -
    distances.add(3.0, DBIDUtil.importInteger(7));
    // + .50,.6 -- redundant
    distances.add(3.0, DBIDUtil.importInteger(3));
    // -
    distances.add(4.0, DBIDUtil.importInteger(8));
    // + .75,.8 ++
    distances.add(4.0, DBIDUtil.importInteger(4));
    // - 1.0,.8 ++
    distances.add(5.0, DBIDUtil.importInteger(9));
    // + 1.0,1. ++
    distances.add(6.0, DBIDUtil.importInteger(5));
    XYCurve roccurve = ROCEvaluation.materializeROC(new DBIDsTest(positive), new DistanceResultAdapter(distances.iter()));
    // System.err.println(roccurve);
    assertEquals("ROC curve too complex", 6, roccurve.size());
    double auc = XYCurve.areaUnderCurve(roccurve);
    assertEquals("ROC AUC (curve) not correct.", 0.6, auc, 1e-14);
    double auc2 = new ROCEvaluation().evaluate(positive, distances);
    assertEquals("ROC AUC (direct) not correct.", 0.6, auc2, 1e-14);
}

Example 2 with DBIDsTest

use of de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DBIDsTest in project elki by elki-project.

the class OutlierRankingEvaluation method evaluateOrderingResult.

private EvaluationResult evaluateOrderingResult(int size, SetDBIDs positiveids, DBIDs order) {
    if (order.size() != size) {
        throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?");
    }
    EvaluationResult res = new EvaluationResult("Evaluation of ranking", "ranking-evaluation");
    DBIDsTest test = new DBIDsTest(positiveids);
    double rate = positiveids.size() / (double) size;
    MeasurementGroup g = res.newGroup("Evaluation measures:");
    double rocauc = ROCEvaluation.STATIC.evaluate(test, new SimpleAdapter(order.iter()));
    g.addMeasure("ROC AUC", rocauc, 0., 1., .5, false);
    double avep = AveragePrecisionEvaluation.STATIC.evaluate(test, new SimpleAdapter(order.iter()));
    g.addMeasure("Average Precision", avep, 0., 1., rate, false);
    double rprec = PrecisionAtKEvaluation.RPRECISION.evaluate(test, new SimpleAdapter(order.iter()));
    g.addMeasure("R-Precision", rprec, 0., 1., rate, false);
    double maxf1 = MaximumF1Evaluation.STATIC.evaluate(test, new SimpleAdapter(order.iter()));
    g.addMeasure("Maximum F1", maxf1, 0., 1., rate, false);
    g = res.newGroup("Adjusted for chance:");
    double adjauc = 2 * rocauc - 1;
    g.addMeasure("Adjusted AUC", adjauc, 0., 1., 0., false);
    double adjavep = (avep - rate) / (1 - rate);
    g.addMeasure("Adjusted AveP", adjavep, 0., 1., 0., false);
    double adjrprec = (rprec - rate) / (1 - rate);
    g.addMeasure("Adjusted R-Prec", adjrprec, 0., 1., 0., false);
    double adjmaxf1 = (maxf1 - rate) / (1 - rate);
    g.addMeasure("Adjusted Max F1", adjmaxf1, 0., 1., 0., false);
    if (LOG.isStatistics()) {
        LOG.statistics(new DoubleStatistic(key + ".rocauc", rocauc));
        LOG.statistics(new DoubleStatistic(key + ".rocauc.adjusted", adjauc));
        LOG.statistics(new DoubleStatistic(key + ".precision.average", avep));
        LOG.statistics(new DoubleStatistic(key + ".precision.average.adjusted", adjavep));
        LOG.statistics(new DoubleStatistic(key + ".precision.r", rprec));
        LOG.statistics(new DoubleStatistic(key + ".precision.r.adjusted", adjrprec));
        LOG.statistics(new DoubleStatistic(key + ".f1.maximum", maxf1));
        LOG.statistics(new DoubleStatistic(key + ".f1.maximum.adjusted", adjmaxf1));
    }
    return res;
}

Example 3 with DBIDsTest

use of de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DBIDsTest in project elki by elki-project.

the class OutlierROCCurve method computeROCResult.

private ROCResult computeROCResult(int size, SetDBIDs positiveids, OutlierResult or) {
    XYCurve roccurve = ROCEvaluation.materializeROC(new DBIDsTest(positiveids), new OutlierScoreAdapter(or));
    double rocauc = XYCurve.areaUnderCurve(roccurve);
    return new ROCResult(roccurve, rocauc);
}

Example 4 with DBIDsTest

use of de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DBIDsTest in project elki by elki-project.

the class OutlierRankingEvaluation method evaluateOutlierResult.

private EvaluationResult evaluateOutlierResult(int size, SetDBIDs positiveids, OutlierResult or) {
    EvaluationResult res = EvaluationResult.findOrCreate(or.getHierarchy(), or, "Evaluation of ranking", "ranking-evaluation");
    DBIDsTest test = new DBIDsTest(positiveids);
    final int pos = positiveids.size();
    final double rate = pos / (double) size;
    MeasurementGroup g = res.findOrCreateGroup("Evaluation measures");
    double rocauc = ROCEvaluation.STATIC.evaluate(test, new OutlierScoreAdapter(or));
    if (!g.hasMeasure("ROC AUC")) {
        g.addMeasure("ROC AUC", rocauc, 0., 1., .5, false);
    }
    double avep = AveragePrecisionEvaluation.STATIC.evaluate(test, new OutlierScoreAdapter(or));
    g.addMeasure("Average Precision", avep, 0., 1., rate, false);
    double rprec = PrecisionAtKEvaluation.RPRECISION.evaluate(test, new OutlierScoreAdapter(or));
    g.addMeasure("R-Precision", rprec, 0., 1., rate, false);
    double maxf1 = MaximumF1Evaluation.STATIC.evaluate(test, new OutlierScoreAdapter(or));
    g.addMeasure("Maximum F1", maxf1, 0., 1., rate, false);
    double maxdcg = DCGEvaluation.maximum(pos);
    double dcg = DCGEvaluation.STATIC.evaluate(test, new OutlierScoreAdapter(or));
    g.addMeasure("DCG", dcg, 0., maxdcg, DCGEvaluation.STATIC.expected(pos, size), false);
    double ndcg = NDCGEvaluation.STATIC.evaluate(test, new OutlierScoreAdapter(or));
    g.addMeasure("NDCG", ndcg, 0., 1., NDCGEvaluation.STATIC.expected(pos, size), false);
    g = res.findOrCreateGroup("Adjusted for chance");
    double adjauc = 2 * rocauc - 1;
    g.addMeasure("Adjusted AUC", adjauc, 0., 1., 0., false);
    double adjavep = (avep - rate) / (1 - rate);
    g.addMeasure("Adjusted AveP", adjavep, 0., 1., 0., false);
    double adjrprec = (rprec - rate) / (1 - rate);
    g.addMeasure("Adjusted R-Prec", adjrprec, 0., 1., 0., false);
    double adjmaxf1 = (maxf1 - rate) / (1 - rate);
    g.addMeasure("Adjusted Max F1", adjmaxf1, 0., 1., 0., false);
    double endcg = NDCGEvaluation.STATIC.expected(pos, size);
    double adjndcg = (ndcg - endcg) / (1. - endcg);
    g.addMeasure("Adjusted DCG", adjndcg, 0., 1., 0., false);
    if (LOG.isStatistics()) {
        LOG.statistics(new DoubleStatistic(key + ".rocauc", rocauc));
        LOG.statistics(new DoubleStatistic(key + ".rocauc.adjusted", adjauc));
        LOG.statistics(new DoubleStatistic(key + ".precision.average", avep));
        LOG.statistics(new DoubleStatistic(key + ".precision.average.adjusted", adjavep));
        LOG.statistics(new DoubleStatistic(key + ".precision.r", rprec));
        LOG.statistics(new DoubleStatistic(key + ".precision.r.adjusted", adjrprec));
        LOG.statistics(new DoubleStatistic(key + ".f1.maximum", maxf1));
        LOG.statistics(new DoubleStatistic(key + ".f1.maximum.adjusted", adjmaxf1));
        LOG.statistics(new DoubleStatistic(key + ".dcg", dcg));
        LOG.statistics(new DoubleStatistic(key + ".dcg.normalized", ndcg));
        LOG.statistics(new DoubleStatistic(key + ".dcg.adjusted", adjndcg));
    }
    return res;
}

Example 5 with DBIDsTest

use of de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DBIDsTest in project elki by elki-project.

the class OutlierROCCurve method computeROCResult.

private ROCResult computeROCResult(int size, SetDBIDs positiveids, DBIDs order) {
    if (order.size() != size) {
        throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?");
    }
    XYCurve roccurve = ROCEvaluation.materializeROC(new DBIDsTest(positiveids), new SimpleAdapter(order.iter()));
    double rocauc = XYCurve.areaUnderCurve(roccurve);
    return new ROCResult(roccurve, rocauc);
}