Examples with KeywordAnalyzer org.apache.lucene.analysis.core.KeywordAnalyzer used on opensource projects

Example 16 with KeywordAnalyzer

use of org.apache.lucene.analysis.core.KeywordAnalyzer in project geode by apache.

the class LuceneQueriesIntegrationTest method shouldNotTokenizeWordsWithKeywordAnalyzer.

@Test()
public void shouldNotTokenizeWordsWithKeywordAnalyzer() throws Exception {
    Map<String, Analyzer> fields = new HashMap<String, Analyzer>();
    fields.put("field1", new StandardAnalyzer());
    fields.put("field2", new KeywordAnalyzer());
    luceneService.createIndexFactory().setFields(fields).create(INDEX_NAME, REGION_NAME);
    Region region = cache.createRegionFactory(RegionShortcut.PARTITION).create(REGION_NAME);
    final LuceneIndex index = luceneService.getIndex(INDEX_NAME, REGION_NAME);
    // Put two values with some of the same tokens
    String value1 = "one three";
    String value2 = "one two three";
    String value3 = "one@three";
    region.put("A", new TestObject(value1, value1));
    region.put("B", new TestObject(value2, value2));
    region.put("C", new TestObject(value3, value3));
    // The value will be tokenized into following documents using the analyzers:
    // <field1:one three> <field2:one three>
    // <field1:one two three> <field2:one two three>
    // <field1:one@three> <field2:one@three>
    luceneService.waitUntilFlushed(INDEX_NAME, REGION_NAME, 60000, TimeUnit.MILLISECONDS);
    // standard analyzer with double quote
    // this query string will be parsed as "one three"
    // but standard analyzer will parse value "one@three" to be "one three"
    // query will be--fields1:"one three"
    // so C will be hit by query
    verifyQuery("field1:\"one three\"", DEFAULT_FIELD, "A", "C");
    // standard analyzer will not tokenize by '_'
    // this query string will be parsed as "one_three"
    // query will be--field1:one_three
    verifyQuery("field1:one_three", DEFAULT_FIELD);
    // standard analyzer will tokenize by '@'
    // this query string will be parsed as "one" "three"
    // query will be--field1:one field1:three
    verifyQuery("field1:one@three", DEFAULT_FIELD, "A", "B", "C");
    HashMap expectedResults = new HashMap();
    expectedResults.put("A", new TestObject(value1, value1));
    expectedResults.put("B", new TestObject(value2, value2));
    expectedResults.put("C", new TestObject(value3, value3));
    verifyQuery("field1:one@three", DEFAULT_FIELD, expectedResults);
    // keyword analyzer, this query will only match the entry that exactly matches
    // this query string will be parsed as "one three"
    // but keyword analyzer will parse one@three to be "one three"
    // query will be--field2:one three
    verifyQuery("field2:\"one three\"", DEFAULT_FIELD, "A");
    // keyword analyzer without double quote. It should be the same as
    // with double quote
    // query will be--field2:one@three
    verifyQuery("field2:one@three", DEFAULT_FIELD, "C");
}

Example 17 with KeywordAnalyzer

use of org.apache.lucene.analysis.core.KeywordAnalyzer in project geode by apache.

the class LuceneListIndexFunctionJUnitTest method getMockLuceneIndex.

private LuceneIndexImpl getMockLuceneIndex(final String indexName) {
    String[] searchableFields = { "field1", "field2" };
    Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
    fieldAnalyzers.put("field1", new StandardAnalyzer());
    fieldAnalyzers.put("field2", new KeywordAnalyzer());
    LuceneIndexImpl index = mock(LuceneIndexImpl.class);
    when(index.getName()).thenReturn(indexName);
    when(index.getRegionPath()).thenReturn("/region");
    when(index.getFieldNames()).thenReturn(searchableFields);
    when(index.getFieldAnalyzers()).thenReturn(fieldAnalyzers);
    return index;
}

Example 18 with KeywordAnalyzer

use of org.apache.lucene.analysis.core.KeywordAnalyzer in project geode by apache.

the class LuceneIndexCommandsJUnitTest method testListIndexWithStats.

@Test
public void testListIndexWithStats() {
    final InternalCache mockCache = mock(InternalCache.class, "InternalCache");
    final String serverName = "mockServer";
    final AbstractExecution mockFunctionExecutor = mock(AbstractExecution.class, "Function Executor");
    final ResultCollector mockResultCollector = mock(ResultCollector.class, "ResultCollector");
    final LuceneIndexStats mockIndexStats1 = getMockIndexStats(1, 10, 5, 1);
    final LuceneIndexStats mockIndexStats2 = getMockIndexStats(2, 20, 10, 2);
    final LuceneIndexStats mockIndexStats3 = getMockIndexStats(3, 30, 15, 3);
    String[] searchableFields = { "field1", "field2", "field3" };
    Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
    fieldAnalyzers.put("field1", new StandardAnalyzer());
    fieldAnalyzers.put("field2", new KeywordAnalyzer());
    fieldAnalyzers.put("field3", null);
    final LuceneIndexDetails indexDetails1 = createIndexDetails("memberFive", "/Employees", searchableFields, fieldAnalyzers, mockIndexStats1, true, serverName);
    final LuceneIndexDetails indexDetails2 = createIndexDetails("memberSix", "/Employees", searchableFields, fieldAnalyzers, mockIndexStats2, true, serverName);
    final LuceneIndexDetails indexDetails3 = createIndexDetails("memberTen", "/Employees", searchableFields, fieldAnalyzers, mockIndexStats3, true, serverName);
    final List<Set<LuceneIndexDetails>> results = new ArrayList<>();
    results.add(CollectionUtils.asSet(indexDetails2, indexDetails1, indexDetails3));
    when(mockFunctionExecutor.execute(isA(LuceneListIndexFunction.class))).thenReturn(mockResultCollector);
    when(mockResultCollector.getResult()).thenReturn(results);
    final LuceneIndexCommands commands = createIndexCommands(mockCache, mockFunctionExecutor);
    CommandResult result = (CommandResult) commands.listIndex(true);
    TabularResultData data = (TabularResultData) result.getResultData();
    assertEquals(Arrays.asList("memberFive", "memberSix", "memberTen"), data.retrieveAllValues("Index Name"));
    assertEquals(Arrays.asList("/Employees", "/Employees", "/Employees"), data.retrieveAllValues("Region Path"));
    assertEquals(Arrays.asList("[field1, field2, field3]", "[field1, field2, field3]", "[field1, field2, field3]"), data.retrieveAllValues("Indexed Fields"));
    assertEquals(Arrays.asList("{field1=StandardAnalyzer, field2=KeywordAnalyzer}", "{field1=StandardAnalyzer, field2=KeywordAnalyzer}", "{field1=StandardAnalyzer, field2=KeywordAnalyzer}"), data.retrieveAllValues("Field Analyzer"));
    assertEquals(Arrays.asList("1", "2", "3"), data.retrieveAllValues("Query Executions"));
    assertEquals(Arrays.asList("10", "20", "30"), data.retrieveAllValues("Commits"));
    assertEquals(Arrays.asList("5", "10", "15"), data.retrieveAllValues("Updates"));
    assertEquals(Arrays.asList("1", "2", "3"), data.retrieveAllValues("Documents"));
}

Example 19 with KeywordAnalyzer

use of org.apache.lucene.analysis.core.KeywordAnalyzer in project geode by apache.

the class LuceneIndexCommandsDUnitTest method createIndexWithoutRegion.

private void createIndexWithoutRegion(final VM vm1) {
    vm1.invoke(() -> {
        LuceneService luceneService = LuceneServiceProvider.get(getCache());
        Map<String, Analyzer> fieldAnalyzers = new HashMap();
        fieldAnalyzers.put("field1", new StandardAnalyzer());
        fieldAnalyzers.put("field2", new KeywordAnalyzer());
        fieldAnalyzers.put("field3", null);
        luceneService.createIndexFactory().setFields(fieldAnalyzers).create(INDEX_NAME, REGION_NAME);
    });
}

Example 20 with KeywordAnalyzer

use of org.apache.lucene.analysis.core.KeywordAnalyzer in project geode by apache.

the class LuceneIndexCreationProfileJUnitTest method getNullField2AnalyzerLuceneIndexCreationProfile.

private LuceneIndexCreationProfile getNullField2AnalyzerLuceneIndexCreationProfile() {
    Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
    fieldAnalyzers.put("field1", new KeywordAnalyzer());
    fieldAnalyzers.put("field2", null);
    return new LuceneIndexCreationProfile(INDEX_NAME, REGION_NAME, new String[] { "field1", "field2" }, getPerFieldAnalyzerWrapper(fieldAnalyzers), fieldAnalyzers);
}