Examples with CustomAnalyzer org.opensearch.index.analysis.CustomAnalyzer used on opensource projects

Example 1 with CustomAnalyzer

use of org.opensearch.index.analysis.CustomAnalyzer in project OpenSearch by opensearch-project.

the class AnalysisModuleTests method testSimpleConfiguration.

private void testSimpleConfiguration(Settings settings) throws IOException {
    IndexAnalyzers indexAnalyzers = getIndexAnalyzers(settings);
    Analyzer analyzer = indexAnalyzers.get("custom1").analyzer();
    assertThat(analyzer, instanceOf(CustomAnalyzer.class));
    CustomAnalyzer custom1 = (CustomAnalyzer) analyzer;
    assertThat(custom1.tokenizerFactory(), instanceOf(StandardTokenizerFactory.class));
    assertThat(custom1.tokenFilters().length, equalTo(2));
    StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0];
    assertThat(stop1.stopWords().size(), equalTo(1));
    // verify position increment gap
    analyzer = indexAnalyzers.get("custom6").analyzer();
    assertThat(analyzer, instanceOf(CustomAnalyzer.class));
    CustomAnalyzer custom6 = (CustomAnalyzer) analyzer;
    assertThat(custom6.getPositionIncrementGap("any_string"), equalTo(256));
    // check custom class name (my)
    analyzer = indexAnalyzers.get("custom4").analyzer();
    assertThat(analyzer, instanceOf(CustomAnalyzer.class));
    CustomAnalyzer custom4 = (CustomAnalyzer) analyzer;
    assertThat(custom4.tokenFilters()[0], instanceOf(MyFilterTokenFilterFactory.class));
}

Example 2 with CustomAnalyzer

use of org.opensearch.index.analysis.CustomAnalyzer in project OpenSearch by opensearch-project.

the class AnnotatedTextFieldMapperTests method createIndexAnalyzers.

@Override
protected IndexAnalyzers createIndexAnalyzers(IndexSettings indexSettings) {
    NamedAnalyzer dflt = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer(), TextFieldMapper.Defaults.POSITION_INCREMENT_GAP);
    NamedAnalyzer standard = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer());
    NamedAnalyzer keyword = new NamedAnalyzer("keyword", AnalyzerScope.INDEX, new KeywordAnalyzer());
    NamedAnalyzer whitespace = new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer());
    NamedAnalyzer stop = new NamedAnalyzer("my_stop_analyzer", AnalyzerScope.INDEX, new CustomAnalyzer(new StandardTokenizerFactory(indexSettings, null, "standard", indexSettings.getSettings()), new CharFilterFactory[0], new TokenFilterFactory[] { new TokenFilterFactory() {

        @Override
        public String name() {
            return "stop";
        }

        @Override
        public TokenStream create(TokenStream tokenStream) {
            return new StopFilter(tokenStream, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
        }
    } }));
    Map<String, NamedAnalyzer> analyzers = new HashMap<>();
    analyzers.put("default", dflt);
    analyzers.put("standard", standard);
    analyzers.put("keyword", keyword);
    analyzers.put("whitespace", whitespace);
    analyzers.put("my_stop_analyzer", stop);
    return new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
}

Example 3 with CustomAnalyzer

use of org.opensearch.index.analysis.CustomAnalyzer in project OpenSearch by opensearch-project.

the class TextFieldMapperTests method createIndexAnalyzers.

@Override
protected IndexAnalyzers createIndexAnalyzers(IndexSettings indexSettings) {
    NamedAnalyzer dflt = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer(), TextFieldMapper.Defaults.POSITION_INCREMENT_GAP);
    NamedAnalyzer standard = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer());
    NamedAnalyzer keyword = new NamedAnalyzer("keyword", AnalyzerScope.INDEX, new KeywordAnalyzer());
    NamedAnalyzer whitespace = new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer());
    NamedAnalyzer stop = new NamedAnalyzer("my_stop_analyzer", AnalyzerScope.INDEX, new CustomAnalyzer(new StandardTokenizerFactory(indexSettings, null, "standard", indexSettings.getSettings()), new CharFilterFactory[0], new TokenFilterFactory[] { new TokenFilterFactory() {

        @Override
        public String name() {
            return "stop";
        }

        @Override
        public TokenStream create(TokenStream tokenStream) {
            return new StopFilter(tokenStream, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
        }
    } }));
    return new IndexAnalyzers(org.opensearch.common.collect.Map.of("default", dflt, "standard", standard, "keyword", keyword, "whitespace", whitespace, "my_stop_analyzer", stop), org.opensearch.common.collect.Map.of(), org.opensearch.common.collect.Map.of());
}