Examples with PreConfiguredTokenizer org.elasticsearch.index.analysis.PreConfiguredTokenizer used on opensource projects

Example 1 with PreConfiguredTokenizer

use of org.elasticsearch.index.analysis.PreConfiguredTokenizer in project crate by crate.

the class AnalysisModule method setupPreConfiguredTokenizers.

static Map<String, PreConfiguredTokenizer> setupPreConfiguredTokenizers(List<AnalysisPlugin> plugins) {
    NamedRegistry<PreConfiguredTokenizer> preConfiguredTokenizers = new NamedRegistry<>("pre-configured tokenizer");
    // Temporary shim to register old style pre-configured tokenizers
    for (PreBuiltTokenizers tokenizer : PreBuiltTokenizers.values()) {
        String name = tokenizer.name().toLowerCase(Locale.ROOT);
        PreConfiguredTokenizer preConfigured;
        switch(tokenizer.getCachingStrategy()) {
            case ONE:
                preConfigured = PreConfiguredTokenizer.singleton(name, () -> tokenizer.create(Version.CURRENT), null);
                break;
            default:
                throw new UnsupportedOperationException("Caching strategy unsupported by temporary shim [" + tokenizer + "]");
        }
        preConfiguredTokenizers.register(name, preConfigured);
    }
    for (AnalysisPlugin plugin : plugins) {
        for (PreConfiguredTokenizer tokenizer : plugin.getPreConfiguredTokenizers()) {
            preConfiguredTokenizers.register(tokenizer.getName(), tokenizer);
        }
    }
    return unmodifiableMap(preConfiguredTokenizers.getRegistry());
}