Examples with LindenSearchRequest com.xiaomi.linden.thrift.common.LindenSearchRequest used on opensource projects

Example 41 with LindenSearchRequest

use of com.xiaomi.linden.thrift.common.LindenSearchRequest in project linden by XiaoMi.

the class TestFlexibleQuery method testDynamicField.

@Test
public void testDynamicField() throws IOException {
    String bql = "SELECT * FROM LINDEN BY query is 'text:(hello world)'";
    LindenSearchRequest request = bqlCompiler.compile(bql).getSearchRequest();
    LindenResult result0 = lindenCore.search(request);
    Assert.assertEquals(4, result0.getHitsSize());
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world\" in (text.STRING) USING MODEL simplest BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    LindenResult result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals(result0.getHits(), result1.getHits());
    // Field boost is 2
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world\" in (text.string^2) USING MODEL fieldBoost1 BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals(result0.getHits().get(0).getScore() * 2, result1.getHits().get(0).getScore(), 0.001);
    Assert.assertEquals(result0.getHits().get(1).getScore() * 2, result1.getHits().get(1).getScore(), 0.001);
    Assert.assertEquals(result0.getHits().get(2).getScore() * 2, result1.getHits().get(2).getScore(), 0.001);
    Assert.assertEquals(result0.getHits().get(3).getScore() * 2, result1.getHits().get(3).getScore(), 0.001);
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello lucene\" in (text.STRING, title.STRING^2) USING MODEL fieldBoost2 BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals("0", result1.getHits().get(0).id);
    Assert.assertEquals("2", result1.getHits().get(1).id);
    Assert.assertEquals("1", result1.getHits().get(2).id);
    Assert.assertEquals("3", result1.getHits().get(3).id);
}

Example 42 with LindenSearchRequest

use of com.xiaomi.linden.thrift.common.LindenSearchRequest in project linden by XiaoMi.

the class TestFlexibleQuery method testMatchRatio.

@Test
public void testMatchRatio() throws IOException {
    String bql = "SELECT * FROM LINDEN BY flexible_query is \"hello lucene\" full_match in (title) USING MODEL simplest BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    LindenSearchRequest request = bqlCompiler.compile(bql).getSearchRequest();
    LindenResult result = lindenCore.search(request);
    Assert.assertEquals(1, result.getHitsSize());
    Assert.assertEquals("0", result.getHits().get(0).id);
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world lucene\" match 1 in (title) USING MODEL simplest BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result = lindenCore.search(request);
    Assert.assertEquals(0, result.getHitsSize());
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world lucene\" match 0.5 in (title) USING MODEL simplest BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    System.out.println(bql);
    request = bqlCompiler.compile(bql).getSearchRequest();
    result = lindenCore.search(request);
    Assert.assertEquals(2, result.getHitsSize());
    Assert.assertEquals("0", result.getHits().get(0).id);
    Assert.assertEquals("1", result.getHits().get(1).id);
}

Example 43 with LindenSearchRequest

use of com.xiaomi.linden.thrift.common.LindenSearchRequest in project linden by XiaoMi.

the class TestFlexibleQuery method testBasic.

@Test
public void testBasic() throws IOException {
    String bql = "SELECT * FROM LINDEN BY query is 'text:(hello world)'";
    LindenSearchRequest request = bqlCompiler.compile(bql).getSearchRequest();
    LindenResult result0 = lindenCore.search(request);
    Assert.assertEquals(4, result0.getHitsSize());
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world\" in (text) USING MODEL simplest BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    LindenResult result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals(result0.getHits(), result1.getHits());
    // Field boost is 2
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world\" in (text^2) USING MODEL fieldBoost1 BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals(result0.getHits().get(0).getScore() * 2, result1.getHits().get(0).getScore(), 0.001);
    Assert.assertEquals(result0.getHits().get(1).getScore() * 2, result1.getHits().get(1).getScore(), 0.001);
    Assert.assertEquals(result0.getHits().get(2).getScore() * 2, result1.getHits().get(2).getScore(), 0.001);
    Assert.assertEquals(result0.getHits().get(3).getScore() * 2, result1.getHits().get(3).getScore(), 0.001);
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello lucene\" in (text, title^2) USING MODEL fieldBoost2 BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals("0", result1.getHits().get(0).id);
    Assert.assertEquals("2", result1.getHits().get(1).id);
    Assert.assertEquals("1", result1.getHits().get(2).id);
    Assert.assertEquals("3", result1.getHits().get(3).id);
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello lucene\" in (text, title) USING MODEL noBoost BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals("0", result1.getHits().get(0).id);
    Assert.assertEquals("1", result1.getHits().get(1).id);
    Assert.assertEquals("2", result1.getHits().get(2).id);
    Assert.assertEquals("3", result1.getHits().get(3).id);
    // Term "text:world" boost is 3
    bql = "SELECT * FROM LINDEN BY query is 'text:(hello world^3)'";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result0 = lindenCore.search(request);
    Assert.assertEquals(4, result0.getHitsSize());
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world^3\" in (text) USING MODEL termBoost BEGIN\n" + "   float sum = 0f;\n" + "    for (int i = 0; i < getFieldLength(); ++i) {\n" + "        for (int j = 0; j < getTermLength(); ++j) {\n" + "            if (isMatched(i, j)) {\n" + "                sum += getScore(i, j);\n" + "            }\n" + "        } \n" + "    } \n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals(result0.getHits(), result1.getHits());
    bql = "SELECT * FROM LINDEN BY query is 'text:(hello world lucene)'";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result0 = lindenCore.search(request);
    Assert.assertEquals(4, result0.getHitsSize());
    Assert.assertEquals("1", result0.getHits().get(0).id);
    Assert.assertEquals("3", result0.getHits().get(1).id);
    Assert.assertEquals("0", result0.getHits().get(2).id);
    Assert.assertEquals("2", result0.getHits().get(3).id);
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world lucene\" in (text) USING MODEL continuousMatchBoost BEGIN\n" + "   float sum = 0f;\n" + "   int continuousMatches = 0;\n" + "     for (int i = 0; i < getFieldLength(); ++i) {\n" + "      int lastMatechedTermIdx = Integer.MIN_VALUE;\n" + "      int[] lastPositions = null;\n" + "      int[] curPositions;\n" + "      for (int j = 0; j < getTermLength(); ++j) {\n" + "        if (isMatched(i, j)) {\n" + "          curPositions = positions(i, j);\n" + "          if (lastMatechedTermIdx + 1 == j) {\n" + "            for (int ii = 0; ii < lastPositions.length; ++ii)\n" + "              for (int jj = 0; jj < curPositions.length; ++jj) {\n" + "                if (lastPositions[ii] + 1 == curPositions[jj]) {\n" + "                  ++continuousMatches;\n" + "                }\n" + "              }\n" + "          }\n" + "          lastMatechedTermIdx = j;\n" + "          lastPositions = curPositions;\n" + "          sum += getScore(i, j);\n" + "        }\n" + "      }\n" + "    }\n" + "    sum  += continuousMatches * 0.5;\n" + "    return sum;\n" + "END\n" + "Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals("3", result1.getHits().get(0).id);
    Assert.assertEquals("1", result1.getHits().get(1).id);
    Assert.assertEquals("0", result1.getHits().get(2).id);
    Assert.assertEquals("2", result1.getHits().get(3).id);
    Assert.assertEquals(1.0, result1.getHits().get(0).score - result0.getHits().get(1).score, 0.001);
    Assert.assertEquals(0.5, result1.getHits().get(1).score - result0.getHits().get(0).score, 0.001);
    Assert.assertEquals(0.5, result1.getHits().get(2).score - result0.getHits().get(2).score, 0.001);
    Assert.assertEquals(0, result1.getHits().get(3).score - result0.getHits().get(3).score, 0.001);
    bql = "SELECT * FROM LINDEN BY flexible_query is \"hello world lucene\" in (text) USING MODEL continuousMatchBoostExplained BEGIN\n" + "   setRootExpl(\"Explanation of continuousMatchBoostExplained model:\");\n" + "   float sum = 0f;\n" + "   for (int i = 0; i < getFieldLength(); ++i) {\n" + "      int continuousMatches = 0;\n" + "      float fieldScore = 0f;\n" + "      int lastMatechedTermIdx = Integer.MIN_VALUE;\n" + "      int[] lastPositions = null;\n" + "      int[] curPositions;\n" + "      for (int j = 0; j < getTermLength(); ++j) {\n" + "        if (isMatched(i, j)) {\n" + "          curPositions = positions(i, j);\n" + "          if (lastMatechedTermIdx + 1 == j) {\n" + "            for (int ii = 0; ii < lastPositions.length; ++ii)\n" + "              for (int jj = 0; jj < curPositions.length; ++jj) {\n" + "                if (lastPositions[ii] + 1 == curPositions[jj]) {\n" + "                  ++continuousMatches;\n" + "                }\n" + "              }\n" + "          }\n" + "          lastMatechedTermIdx = j;\n" + "          lastPositions = curPositions;\n" + "          float termScore = getScore(i, j);\n" + "          fieldScore += termScore;\n" + "          addTermExpl(i, j, termScore, getExpl(\"%s is matched in %s field, positions are %s\", text(i, j), field(i, j), curPositions));\n" + "        }\n" + "      }\n" + "      fieldScore += continuousMatches * 0.5;\n" + "      addFieldExpl(i, fieldScore, getExpl(\"%d continuous matches in %s field\", continuousMatches, field(i, 0)));\n" + "      sum += fieldScore;\n" + "    }\n" + "    return sum;\n" + "END\n" + "EXPLAIN Limit 0, 10\n";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result1 = lindenCore.search(request);
    Assert.assertEquals(4, result1.getHitsSize());
    Assert.assertEquals("3", result1.getHits().get(0).id);
    Assert.assertEquals("1", result1.getHits().get(1).id);
    Assert.assertEquals("0", result1.getHits().get(2).id);
    Assert.assertEquals("2", result1.getHits().get(3).id);
    Assert.assertEquals(1.0, result1.getHits().get(0).score - result0.getHits().get(1).score, 0.001);
    Assert.assertEquals(0.5, result1.getHits().get(1).score - result0.getHits().get(0).score, 0.001);
    Assert.assertEquals(0.5, result1.getHits().get(2).score - result0.getHits().get(2).score, 0.001);
    Assert.assertEquals(0, result1.getHits().get(3).score - result0.getHits().get(3).score, 0.001);
    for (int i = 0; i < 4; ++i) {
        System.out.println("\nExplanation of Doc " + result1.getHits().get(i).id);
        OutputExplanation(result1.getHits().get(i).getExplanation(), "*");
    }
}

Example 44 with LindenSearchRequest

use of com.xiaomi.linden.thrift.common.LindenSearchRequest in project linden by XiaoMi.

the class TestGlobalIDF method testGlobalIdfV2.

@Test
public void testGlobalIdfV2() throws IOException {
    String bql1 = "select * from linden by flexible_query is 'lucene' global_idf of (title, field1) in (field1)\n" + "using model test\n" + "begin\n" + "    return 1;\n" + "end\n" + "source explain;";
    LindenSearchRequest request1 = bqlCompiler.compile(bql1).getSearchRequest();
    LindenResult result1 = lindenCore.search(request1);
    String detail1 = result1.getHits().get(0).getExplanation().getDetails().get(0).getDetails().get(0).getDescription();
    Assert.assertEquals(detail1.split(" \\* ")[0], "0.82");
    String bql2 = "select * from linden by flexible_query is 'lucene' global_idf of (field1, field2) in (field1)\n" + "using model test\n" + "begin\n" + "    return 1;\n" + "end\n" + "source explain;";
    LindenSearchRequest request2 = bqlCompiler.compile(bql2).getSearchRequest();
    LindenResult result2 = lindenCore.search(request2);
    String detail2 = result2.getHits().get(0).getExplanation().getDetails().get(0).getDetails().get(0).getDescription();
    Assert.assertEquals(detail2.split(" \\* ")[0], "1.22");
    String bql3 = "select * from linden by flexible_query is 'lucene' global_idf in (field1)\n" + "using model test\n" + "begin\n" + "    return 1;\n" + "end\n" + "source explain;";
    LindenSearchRequest request3 = bqlCompiler.compile(bql3).getSearchRequest();
    LindenResult result3 = lindenCore.search(request3);
    String detail3 = result3.getHits().get(0).getExplanation().getDetails().get(0).getDetails().get(0).getDescription();
    Assert.assertEquals(detail3.split(" \\* ")[0], "1.51");
}

Example 45 with LindenSearchRequest

use of com.xiaomi.linden.thrift.common.LindenSearchRequest in project linden by XiaoMi.

the class TestLindenJiebaAnalyzerIndexMode method testIndexMode.

@Test
public void testIndexMode() throws IOException {
    String bql = "select * from linden by query is \"title:刘华清\"";
    LindenSearchRequest request = bqlCompiler.compile(bql).getSearchRequest();
    LindenResult result = lindenCore.search(request);
    Assert.assertEquals(2, result.getTotalHits());
    // phrase test
    bql = "select * from linden by query is 'title:\"刘华清\"'";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result = lindenCore.search(request);
    Assert.assertEquals(2, result.getTotalHits());
    bql = "select * from linden by query is 'title:\"海军上将刘华清\"'";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result = lindenCore.search(request);
    Assert.assertEquals(0, result.getTotalHits());
    // snippet test
    bql = "select * from linden by query is 'title:(海军上将刘华清中国龙)' snippet title";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result = lindenCore.search(request);
    Assert.assertEquals(2, result.getTotalHits());
    Assert.assertEquals("<b>海军上将</b><b>刘华清</b>", result.getHits().get(0).getSnippets().get("title").getSnippet());
    Assert.assertEquals("<b>海军上将</b><b>刘华清</b>!!! <b>中国</b> 人民 李玉洁 尚铁龙 胡晓光", result.getHits().get(1).getSnippets().get("title").getSnippet());
    bql = "select * from linden by query is 'title:(海军中国铁龙)' snippet title";
    request = bqlCompiler.compile(bql).getSearchRequest();
    result = lindenCore.search(request);
    Assert.assertEquals(2, result.getTotalHits());
    Assert.assertEquals("<b>海军</b>上将刘华清!!! <b>中国</b> 人民 李玉洁 尚铁龙 胡晓光", result.getHits().get(0).getSnippets().get("title").getSnippet());
}