Examples with StandardAnalyzer org.apache.lucene.analysis.standard.StandardAnalyzer used on opensource projects

Example 51 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project lucene-solr by apache.

the class BlendedInfixSuggesterTest method testBlendedSort.

/**
   * Test the weight transformation depending on the position
   * of the matching term.
   */
public void testBlendedSort() throws IOException {
    BytesRef payload = new BytesRef("star");
    Input[] keys = new Input[] { new Input("star wars: episode v - the empire strikes back", 8, payload) };
    Path tempDir = createTempDir("BlendedInfixSuggesterTest");
    Analyzer a = new StandardAnalyzer(CharArraySet.EMPTY_SET);
    BlendedInfixSuggester suggester = new BlendedInfixSuggester(newFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_LINEAR, BlendedInfixSuggester.DEFAULT_NUM_FACTOR, false);
    suggester.build(new InputArrayIterator(keys));
    // we query for star wars and check that the weight
    // is smaller when we search for tokens that are far from the beginning
    long w0 = getInResults(suggester, "star ", payload, 1);
    long w1 = getInResults(suggester, "war", payload, 1);
    long w2 = getInResults(suggester, "empire ba", payload, 1);
    long w3 = getInResults(suggester, "back", payload, 1);
    long w4 = getInResults(suggester, "bacc", payload, 1);
    assertTrue(w0 > w1);
    assertTrue(w1 > w2);
    assertTrue(w2 > w3);
    assertTrue(w4 < 0);
    suggester.close();
}

Example 52 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project lucene-solr by apache.

the class BlendedInfixSuggesterTest method testNullPrefixToken.

/**
   * Handle trailing spaces that result in no prefix token LUCENE-6093
   */
public void testNullPrefixToken() throws IOException {
    BytesRef payload = new BytesRef("lake");
    Input[] keys = new Input[] { new Input("top of the lake", 8, payload) };
    Path tempDir = createTempDir("BlendedInfixSuggesterTest");
    Analyzer a = new StandardAnalyzer(CharArraySet.EMPTY_SET);
    BlendedInfixSuggester suggester = new BlendedInfixSuggester(newFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_LINEAR, BlendedInfixSuggester.DEFAULT_NUM_FACTOR, false);
    suggester.build(new InputArrayIterator(keys));
    getInResults(suggester, "of ", payload, 1);
    getInResults(suggester, "the ", payload, 1);
    getInResults(suggester, "lake ", payload, 1);
    suggester.close();
}

Example 53 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project lucene-solr by apache.

the class BlendedInfixSuggesterTest method rying.

public void rying() throws IOException {
    BytesRef lake = new BytesRef("lake");
    BytesRef star = new BytesRef("star");
    BytesRef ret = new BytesRef("ret");
    Input[] keys = new Input[] { new Input("top of the lake", 15, lake), new Input("star wars: episode v - the empire strikes back", 12, star), new Input("the returned", 10, ret) };
    Path tempDir = createTempDir("BlendedInfixSuggesterTest");
    Analyzer a = new StandardAnalyzer(CharArraySet.EMPTY_SET);
    // if factor is small, we don't get the expected element
    BlendedInfixSuggester suggester = new BlendedInfixSuggester(newFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, BlendedInfixSuggester.DEFAULT_NUM_FACTOR, false);
    suggester.build(new InputArrayIterator(keys));
    List<Lookup.LookupResult> responses = suggester.lookup("the", 4, true, false);
    for (Lookup.LookupResult response : responses) {
        System.out.println(response);
    }
    suggester.close();
}

Example 54 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project lucene-solr by apache.

the class BlendedInfixSuggesterTest method testSuggesterCountForAllLookups.

public void testSuggesterCountForAllLookups() throws IOException {
    Input[] keys = new Input[] { new Input("lend me your ears", 1), new Input("as you sow so shall you reap", 1) };
    Path tempDir = createTempDir("BlendedInfixSuggesterTest");
    Analyzer a = new StandardAnalyzer(CharArraySet.EMPTY_SET);
    // BlenderType.LINEAR is used by default (remove position*10%)
    BlendedInfixSuggester suggester = new BlendedInfixSuggester(newFSDirectory(tempDir), a);
    suggester.build(new InputArrayIterator(keys));
    String term = "you";
    List<Lookup.LookupResult> responses = suggester.lookup(term, false, 1);
    assertEquals(1, responses.size());
    responses = suggester.lookup(term, false, 2);
    assertEquals(2, responses.size());
    responses = suggester.lookup(term, 1, false, false);
    assertEquals(1, responses.size());
    responses = suggester.lookup(term, 2, false, false);
    assertEquals(2, responses.size());
    responses = suggester.lookup(term, (Map) null, 1, false, false);
    assertEquals(1, responses.size());
    responses = suggester.lookup(term, (Map) null, 2, false, false);
    assertEquals(2, responses.size());
    responses = suggester.lookup(term, (Set) null, 1, false, false);
    assertEquals(1, responses.size());
    responses = suggester.lookup(term, (Set) null, 2, false, false);
    assertEquals(2, responses.size());
    responses = suggester.lookup(term, null, false, 1);
    assertEquals(1, responses.size());
    responses = suggester.lookup(term, null, false, 2);
    assertEquals(2, responses.size());
    responses = suggester.lookup(term, (BooleanQuery) null, 1, false, false);
    assertEquals(1, responses.size());
    responses = suggester.lookup(term, (BooleanQuery) null, 2, false, false);
    assertEquals(2, responses.size());
    suggester.close();
}

Example 55 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project lucene-solr by apache.

the class SearchFiles method main.

/** Simple command-line based search demo. */
public static void main(String[] args) throws Exception {
    String usage = "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-query string] [-raw] [-paging hitsPerPage]\n\nSee http://lucene.apache.org/core/4_1_0/demo/ for details.";
    if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0]))) {
        System.out.println(usage);
        System.exit(0);
    }
    String index = "index";
    String field = "contents";
    String queries = null;
    int repeat = 0;
    boolean raw = false;
    String queryString = null;
    int hitsPerPage = 10;
    for (int i = 0; i < args.length; i++) {
        if ("-index".equals(args[i])) {
            index = args[i + 1];
            i++;
        } else if ("-field".equals(args[i])) {
            field = args[i + 1];
            i++;
        } else if ("-queries".equals(args[i])) {
            queries = args[i + 1];
            i++;
        } else if ("-query".equals(args[i])) {
            queryString = args[i + 1];
            i++;
        } else if ("-repeat".equals(args[i])) {
            repeat = Integer.parseInt(args[i + 1]);
            i++;
        } else if ("-raw".equals(args[i])) {
            raw = true;
        } else if ("-paging".equals(args[i])) {
            hitsPerPage = Integer.parseInt(args[i + 1]);
            if (hitsPerPage <= 0) {
                System.err.println("There must be at least 1 hit per page.");
                System.exit(1);
            }
            i++;
        }
    }
    IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(index)));
    IndexSearcher searcher = new IndexSearcher(reader);
    Analyzer analyzer = new StandardAnalyzer();
    BufferedReader in = null;
    if (queries != null) {
        in = Files.newBufferedReader(Paths.get(queries), StandardCharsets.UTF_8);
    } else {
        in = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
    }
    QueryParser parser = new QueryParser(field, analyzer);
    while (true) {
        if (queries == null && queryString == null) {
            // prompt the user
            System.out.println("Enter query: ");
        }
        String line = queryString != null ? queryString : in.readLine();
        if (line == null || line.length() == -1) {
            break;
        }
        line = line.trim();
        if (line.length() == 0) {
            break;
        }
        Query query = parser.parse(line);
        System.out.println("Searching for: " + query.toString(field));
        if (repeat > 0) {
            // repeat & time as benchmark
            Date start = new Date();
            for (int i = 0; i < repeat; i++) {
                searcher.search(query, 100);
            }
            Date end = new Date();
            System.out.println("Time: " + (end.getTime() - start.getTime()) + "ms");
        }
        doPagingSearch(in, searcher, query, hitsPerPage, raw, queries == null && queryString == null);
        if (queryString != null) {
            break;
        }
    }
    reader.close();
}