Examples with InputArrayIterator org.apache.lucene.search.suggest.InputArrayIterator used on opensource projects

Example 71 with InputArrayIterator

use of org.apache.lucene.search.suggest.InputArrayIterator in project lucene-solr by apache.

the class AnalyzingInfixSuggesterTest method testCommitAfterBuild.

public void testCommitAfterBuild() throws Exception {
    performOperationWithAllOptionCombinations(suggester -> {
        suggester.build(new InputArrayIterator(sharedInputs));
        suggester.commit();
    });
}

Example 72 with InputArrayIterator

use of org.apache.lucene.search.suggest.InputArrayIterator in project lucene-solr by apache.

the class AnalyzingInfixSuggesterTest method testBinaryContext.

public void testBinaryContext() throws Exception {
    byte[] context1 = new byte[4];
    byte[] context2 = new byte[5];
    byte[] context3 = new byte[1];
    context3[0] = (byte) 0xff;
    Input[] keys = new Input[] { new Input("lend me your ear", 8, new BytesRef("foobar"), asSet(context1, context2)), new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz"), asSet(context1, context3)) };
    Path tempDir = createTempDir("analyzingInfixContext");
    for (int iter = 0; iter < 2; iter++) {
        AnalyzingInfixSuggester suggester;
        Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
        if (iter == 0) {
            suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
            suggester.build(new InputArrayIterator(keys));
        } else {
            // Test again, after close/reopen:
            suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
        }
        // Both have context1:
        List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), asSet(context1), 10, true, true);
        assertEquals(2, results.size());
        LookupResult result = results.get(0);
        assertEquals("a penny saved is a penny earned", result.key);
        assertEquals("a penny saved is a penny <b>ear</b>ned", result.highlightKey);
        assertEquals(10, result.value);
        assertEquals(new BytesRef("foobaz"), result.payload);
        assertNotNull(result.contexts);
        assertEquals(2, result.contexts.size());
        assertTrue(result.contexts.contains(new BytesRef(context1)));
        assertTrue(result.contexts.contains(new BytesRef(context3)));
        result = results.get(1);
        assertEquals("lend me your ear", result.key);
        assertEquals("lend me your <b>ear</b>", result.highlightKey);
        assertEquals(8, result.value);
        assertEquals(new BytesRef("foobar"), result.payload);
        assertNotNull(result.contexts);
        assertEquals(2, result.contexts.size());
        assertTrue(result.contexts.contains(new BytesRef(context1)));
        assertTrue(result.contexts.contains(new BytesRef(context2)));
        suggester.close();
        a.close();
    }
}

Example 73 with InputArrayIterator

use of org.apache.lucene.search.suggest.InputArrayIterator in project lucene-solr by apache.

the class AnalyzingSuggesterTest method testExactFirst.

public void testExactFirst() throws Exception {
    Analyzer a = getUnusualAnalyzer();
    int options = AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP;
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, options, 256, -1, true);
    suggester.build(new InputArrayIterator(new Input[] { new Input("x y", 1), new Input("x y z", 3), new Input("x", 2), new Input("z z z", 20) }));
    for (int topN = 1; topN < 6; topN++) {
        List<LookupResult> results = suggester.lookup("x y", false, topN);
        //System.out.println("topN=" + topN + " " + results);
        assertEquals(Math.min(topN, 4), results.size());
        assertEquals("x y", results.get(0).key);
        assertEquals(1, results.get(0).value);
        if (topN > 1) {
            assertEquals("z z z", results.get(1).key);
            assertEquals(20, results.get(1).value);
            if (topN > 2) {
                assertEquals("x y z", results.get(2).key);
                assertEquals(3, results.get(2).value);
                if (topN > 3) {
                    assertEquals("x", results.get(3).key);
                    assertEquals(2, results.get(3).value);
                }
            }
        }
    }
    IOUtils.close(a, tempDir);
}

Example 74 with InputArrayIterator

use of org.apache.lucene.search.suggest.InputArrayIterator in project lucene-solr by apache.

the class AnalyzingSuggesterTest method testNonExactFirst.

public void testNonExactFirst() throws Exception {
    Analyzer a = getUnusualAnalyzer();
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);
    suggester.build(new InputArrayIterator(new Input[] { new Input("x y", 1), new Input("x y z", 3), new Input("x", 2), new Input("z z z", 20) }));
    for (int topN = 1; topN < 6; topN++) {
        List<LookupResult> results = suggester.lookup("p", false, topN);
        assertEquals(Math.min(topN, 4), results.size());
        assertEquals("z z z", results.get(0).key);
        assertEquals(20, results.get(0).value);
        if (topN > 1) {
            assertEquals("x y z", results.get(1).key);
            assertEquals(3, results.get(1).value);
            if (topN > 2) {
                assertEquals("x", results.get(2).key);
                assertEquals(2, results.get(2).value);
                if (topN > 3) {
                    assertEquals("x y", results.get(3).key);
                    assertEquals(1, results.get(3).value);
                }
            }
        }
    }
    IOUtils.close(a, tempDir);
}

Example 75 with InputArrayIterator

use of org.apache.lucene.search.suggest.InputArrayIterator in project lucene-solr by apache.

the class AnalyzingSuggesterTest method testRandomRealisticKeys.

public void testRandomRealisticKeys() throws IOException {
    LineFileDocs lineFile = new LineFileDocs(random());
    Map<String, Long> mapping = new HashMap<>();
    List<Input> keys = new ArrayList<>();
    // this might bring up duplicates
    int howMany = atLeast(100);
    for (int i = 0; i < howMany; i++) {
        Document nextDoc = lineFile.nextDoc();
        String title = nextDoc.getField("title").stringValue();
        int randomWeight = random().nextInt(100);
        keys.add(new Input(title, randomWeight));
        if (!mapping.containsKey(title) || mapping.get(title) < randomWeight) {
            mapping.put(title, Long.valueOf(randomWeight));
        }
    }
    Analyzer indexAnalyzer = new MockAnalyzer(random());
    Analyzer queryAnalyzer = new MockAnalyzer(random());
    Directory tempDir = getDirectory();
    AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(tempDir, "suggest", indexAnalyzer, queryAnalyzer, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, random().nextBoolean());
    boolean doPayloads = random().nextBoolean();
    if (doPayloads) {
        List<Input> keysAndPayloads = new ArrayList<>();
        for (Input termFreq : keys) {
            keysAndPayloads.add(new Input(termFreq.term, termFreq.v, new BytesRef(Long.toString(termFreq.v))));
        }
        analyzingSuggester.build(new InputArrayIterator(keysAndPayloads));
    } else {
        analyzingSuggester.build(new InputArrayIterator(keys));
    }
    for (Input termFreq : keys) {
        List<LookupResult> lookup = analyzingSuggester.lookup(termFreq.term.utf8ToString(), false, keys.size());
        for (LookupResult lookupResult : lookup) {
            assertEquals(mapping.get(lookupResult.key), Long.valueOf(lookupResult.value));
            if (doPayloads) {
                assertEquals(lookupResult.payload.utf8ToString(), Long.toString(lookupResult.value));
            } else {
                assertNull(lookupResult.payload);
            }
        }
    }
    IOUtils.close(lineFile, indexAnalyzer, queryAnalyzer, tempDir);
}