Examples with TrecJudge org.apache.lucene.benchmark.quality.trec.TrecJudge used on opensource projects

Example 1 with TrecJudge

use of org.apache.lucene.benchmark.quality.trec.TrecJudge in project lucene-solr by apache.

the class TestQualityRun method testTrecQuality.

public void testTrecQuality() throws Exception {
    // first create the partial reuters index
    createReutersIndex();
    int maxResults = 1000;
    // orig docID is in the linedoc format title 
    String docNameField = "doctitle";
    PrintWriter logger = VERBOSE ? new PrintWriter(new OutputStreamWriter(System.out, Charset.defaultCharset()), true) : null;
    // prepare topics
    InputStream topics = getClass().getResourceAsStream("trecTopics.txt");
    TrecTopicsReader qReader = new TrecTopicsReader();
    QualityQuery[] qqs = qReader.readQueries(new BufferedReader(new InputStreamReader(topics, StandardCharsets.UTF_8)));
    // prepare judge
    InputStream qrels = getClass().getResourceAsStream("trecQRels.txt");
    Judge judge = new TrecJudge(new BufferedReader(new InputStreamReader(qrels, StandardCharsets.UTF_8)));
    // validate topics & judgments match each other
    judge.validateData(qqs, logger);
    Directory dir = newFSDirectory(getWorkDir().resolve("index"));
    IndexReader reader = DirectoryReader.open(dir);
    IndexSearcher searcher = new IndexSearcher(reader);
    QualityQueryParser qqParser = new SimpleQQParser("title", "body");
    QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);
    SubmissionReport submitLog = VERBOSE ? new SubmissionReport(logger, "TestRun") : null;
    qrun.setMaxResults(maxResults);
    QualityStats[] stats = qrun.execute(judge, submitLog, logger);
    // m>=3: these queries remain perfect
    for (int i = 0; i < stats.length; i++) {
        QualityStats s = stats[i];
        switch(i % 8) {
            case 0:
                assertTrue("avg-p should be hurt: " + s.getAvp(), 1.0 > s.getAvp());
                assertTrue("recall should be hurt: " + s.getRecall(), 1.0 > s.getRecall());
                for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
                    assertEquals("p_at_" + j + " should be perfect: " + s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2);
                }
                break;
            case 1:
                assertTrue("avg-p should be hurt", 1.0 > s.getAvp());
                assertEquals("recall should be perfect: " + s.getRecall(), 1.0, s.getRecall(), 1E-2);
                for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
                    assertTrue("p_at_" + j + " should be hurt: " + s.getPrecisionAt(j), 1.0 > s.getPrecisionAt(j));
                }
                break;
            case 2:
                assertTrue("avg-p should be hurt: " + s.getAvp(), 1.0 > s.getAvp());
                assertTrue("recall should be hurt: " + s.getRecall(), 1.0 > s.getRecall());
                for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
                    assertTrue("p_at_" + j + " should be hurt: " + s.getPrecisionAt(j), 1.0 > s.getPrecisionAt(j));
                }
                break;
            default:
                {
                    assertEquals("avg-p should be perfect: " + s.getAvp(), 1.0, s.getAvp(), 1E-2);
                    assertEquals("recall should be perfect: " + s.getRecall(), 1.0, s.getRecall(), 1E-2);
                    for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
                        assertEquals("p_at_" + j + " should be perfect: " + s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2);
                    }
                }
        }
    }
    QualityStats avg = QualityStats.average(stats);
    if (logger != null) {
        avg.log("Average statistis:", 1, logger, "  ");
    }
    assertTrue("mean avg-p should be hurt: " + avg.getAvp(), 1.0 > avg.getAvp());
    assertTrue("avg recall should be hurt: " + avg.getRecall(), 1.0 > avg.getRecall());
    for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
        assertTrue("avg p_at_" + j + " should be hurt: " + avg.getPrecisionAt(j), 1.0 > avg.getPrecisionAt(j));
    }
    reader.close();
    dir.close();
}