Examples with ShingleTokenFilterFactory org.opensearch.index.analysis.ShingleTokenFilterFactory used on opensource projects

Example 1 with ShingleTokenFilterFactory

use of org.opensearch.index.analysis.ShingleTokenFilterFactory in project OpenSearch by opensearch-project.

the class AnalysisModule method setupTokenFilters.

private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(List<AnalysisPlugin> plugins, HunspellService hunspellService) {
    NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = new NamedRegistry<>("token_filter");
    tokenFilters.register("stop", StopTokenFilterFactory::new);
    // Add "standard" for old indices (bwc)
    tokenFilters.register("standard", new AnalysisProvider<TokenFilterFactory>() {

        @Override
        public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
            if (indexSettings.getIndexVersionCreated().before(LegacyESVersion.V_7_0_0)) {
                deprecationLogger.deprecate("standard_deprecation", "The [standard] token filter name is deprecated and will be removed in a future version.");
            } else {
                throw new IllegalArgumentException("The [standard] token filter has been removed.");
            }
            return new AbstractTokenFilterFactory(indexSettings, name, settings) {

                @Override
                public TokenStream create(TokenStream tokenStream) {
                    return tokenStream;
                }
            };
        }

        @Override
        public boolean requiresAnalysisSettings() {
            return false;
        }
    });
    tokenFilters.register("shingle", ShingleTokenFilterFactory::new);
    tokenFilters.register("hunspell", requiresAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService)));
    tokenFilters.extractAndRegister(plugins, AnalysisPlugin::getTokenFilters);
    return tokenFilters;
}