Search in sources :

Example 6 with MutualInformation

use of org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation in project elasticsearch by elastic.

the class SignificanceHeuristicTests method testBuilderAndParser.

// test that
// 1. The output of the builders can actually be parsed
// 2. The parser does not swallow parameters after a significance heuristic was defined
public void testBuilderAndParser() throws Exception {
    SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
    ParseFieldRegistry<SignificanceHeuristicParser> heuristicParserMapper = searchModule.getSignificanceHeuristicParserRegistry();
    // test jlh with string
    assertTrue(parseFromString(heuristicParserMapper, "\"jlh\":{}") instanceof JLHScore);
    // test gnd with string
    assertTrue(parseFromString(heuristicParserMapper, "\"gnd\":{}") instanceof GND);
    // test mutual information with string
    boolean includeNegatives = randomBoolean();
    boolean backgroundIsSuperset = randomBoolean();
    String mutual = "\"mutual_information\":{\"include_negatives\": " + includeNegatives + ", \"background_is_superset\":" + backgroundIsSuperset + "}";
    assertEquals(new MutualInformation(includeNegatives, backgroundIsSuperset), parseFromString(heuristicParserMapper, mutual));
    String chiSquare = "\"chi_square\":{\"include_negatives\": " + includeNegatives + ", \"background_is_superset\":" + backgroundIsSuperset + "}";
    assertEquals(new ChiSquare(includeNegatives, backgroundIsSuperset), parseFromString(heuristicParserMapper, chiSquare));
    // test with builders
    assertThat(parseFromBuilder(heuristicParserMapper, new JLHScore()), instanceOf(JLHScore.class));
    assertThat(parseFromBuilder(heuristicParserMapper, new GND(backgroundIsSuperset)), instanceOf(GND.class));
    assertEquals(new MutualInformation(includeNegatives, backgroundIsSuperset), parseFromBuilder(heuristicParserMapper, new MutualInformation(includeNegatives, backgroundIsSuperset)));
    assertEquals(new ChiSquare(includeNegatives, backgroundIsSuperset), parseFromBuilder(heuristicParserMapper, new ChiSquare(includeNegatives, backgroundIsSuperset)));
    // test exceptions
    String faultyHeuristicdefinition = "\"mutual_information\":{\"include_negatives\": false, \"some_unknown_field\": false}";
    String expectedError = "unknown field [some_unknown_field]";
    checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
    faultyHeuristicdefinition = "\"chi_square\":{\"unknown_field\": true}";
    expectedError = "unknown field [unknown_field]";
    checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
    faultyHeuristicdefinition = "\"jlh\":{\"unknown_field\": true}";
    expectedError = "expected an empty object, but found ";
    checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
    faultyHeuristicdefinition = "\"gnd\":{\"unknown_field\": true}";
    expectedError = "unknown field [unknown_field]";
    checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
}
Also used : JLHScore(org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore) ChiSquare(org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare) SearchModule(org.elasticsearch.search.SearchModule) MutualInformation(org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation) Matchers.containsString(org.hamcrest.Matchers.containsString) SignificanceHeuristicParser(org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser) GND(org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND)

Example 7 with MutualInformation

use of org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation in project elasticsearch by elastic.

the class SignificanceHeuristicTests method testScoreMutual.

public void testScoreMutual() throws Exception {
    SignificanceHeuristic heuristic = new MutualInformation(true, true);
    assertThat(heuristic.getScore(1, 1, 1, 3), greaterThan(0.0));
    assertThat(heuristic.getScore(1, 1, 2, 3), lessThan(heuristic.getScore(1, 1, 1, 3)));
    assertThat(heuristic.getScore(2, 2, 2, 4), equalTo(1.0));
    assertThat(heuristic.getScore(0, 2, 2, 4), equalTo(1.0));
    assertThat(heuristic.getScore(2, 2, 4, 4), equalTo(0.0));
    assertThat(heuristic.getScore(1, 2, 2, 4), equalTo(0.0));
    assertThat(heuristic.getScore(3, 6, 9, 18), equalTo(0.0));
    double score = 0.0;
    try {
        long a = randomLong();
        long b = randomLong();
        long c = randomLong();
        long d = randomLong();
        score = heuristic.getScore(a, b, c, d);
    } catch (IllegalArgumentException e) {
    }
    assertThat(score, lessThanOrEqualTo(1.0));
    assertThat(score, greaterThanOrEqualTo(0.0));
    heuristic = new MutualInformation(false, true);
    assertThat(heuristic.getScore(0, 1, 2, 3), equalTo(Double.NEGATIVE_INFINITY));
    heuristic = new MutualInformation(true, false);
    score = heuristic.getScore(2, 3, 1, 4);
    assertThat(score, greaterThanOrEqualTo(0.0));
    assertThat(score, lessThanOrEqualTo(1.0));
    score = heuristic.getScore(1, 4, 2, 3);
    assertThat(score, greaterThanOrEqualTo(0.0));
    assertThat(score, lessThanOrEqualTo(1.0));
    score = heuristic.getScore(1, 3, 4, 4);
    assertThat(score, greaterThanOrEqualTo(0.0));
    assertThat(score, lessThanOrEqualTo(1.0));
}
Also used : SignificanceHeuristic(org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic) MutualInformation(org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation)

Example 8 with MutualInformation

use of org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation in project elasticsearch by elastic.

the class SignificanceHeuristicTests method testAssertions.

public void testAssertions() throws Exception {
    testBackgroundAssertions(new MutualInformation(true, true), new MutualInformation(true, false));
    testBackgroundAssertions(new ChiSquare(true, true), new ChiSquare(true, false));
    testBackgroundAssertions(new GND(true), new GND(false));
    testAssertions(new PercentageScore());
    testAssertions(new JLHScore());
}
Also used : JLHScore(org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore) ChiSquare(org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare) MutualInformation(org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation) GND(org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND) PercentageScore(org.elasticsearch.search.aggregations.bucket.significant.heuristics.PercentageScore)

Example 9 with MutualInformation

use of org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation in project elasticsearch by elastic.

the class SignificantTermsSignificanceScoreIT method testBackgroundVsSeparateSet.

public void testBackgroundVsSeparateSet() throws Exception {
    String type = randomBoolean() ? "text" : "long";
    String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
    SharedSignificantTermsTestMethods.index01Docs(type, settings, this);
    testBackgroundVsSeparateSet(new MutualInformation(true, true), new MutualInformation(true, false));
    testBackgroundVsSeparateSet(new ChiSquare(true, true), new ChiSquare(true, false));
    testBackgroundVsSeparateSet(new GND(true), new GND(false));
}
Also used : ChiSquare(org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare) MutualInformation(org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation) GND(org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND)

Aggregations

MutualInformation (org.elasticsearch.search.aggregations.bucket.significant.heuristics.MutualInformation)9 ChiSquare (org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare)7 GND (org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND)6 JLHScore (org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore)5 PercentageScore (org.elasticsearch.search.aggregations.bucket.significant.heuristics.PercentageScore)3 SignificanceHeuristic (org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic)3 ArrayList (java.util.ArrayList)1 TreeSet (java.util.TreeSet)1 BytesRef (org.apache.lucene.util.BytesRef)1 RegExp (org.apache.lucene.util.automaton.RegExp)1 SearchResponse (org.elasticsearch.action.search.SearchResponse)1 TermQueryBuilder (org.elasticsearch.index.query.TermQueryBuilder)1 Script (org.elasticsearch.script.Script)1 SearchModule (org.elasticsearch.search.SearchModule)1 SignificantTerms (org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms)1 SignificantTermsAggregationBuilder (org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder)1 ScriptHeuristic (org.elasticsearch.search.aggregations.bucket.significant.heuristics.ScriptHeuristic)1 SignificanceHeuristicParser (org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser)1 IncludeExclude (org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude)1 ElasticsearchAssertions.assertSearchResponse (org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse)1