Examples with CharFilterFactory org.elasticsearch.index.analysis.CharFilterFactory used on opensource projects

Example 1 with CharFilterFactory

use of org.elasticsearch.index.analysis.CharFilterFactory in project elasticsearch by elastic.

the class TransportAnalyzeAction method createStackedTokenStream.

private static TokenStream createStackedTokenStream(String source, CharFilterFactory[] charFilterFactories, TokenizerFactory tokenizerFactory, TokenFilterFactory[] tokenFilterFactories, int current) {
    Reader reader = new FastStringReader(source);
    for (CharFilterFactory charFilterFactory : charFilterFactories) {
        reader = charFilterFactory.create(reader);
    }
    Tokenizer tokenizer = tokenizerFactory.create();
    tokenizer.setReader(reader);
    TokenStream tokenStream = tokenizer;
    for (int i = 0; i < current; i++) {
        tokenStream = tokenFilterFactories[i].create(tokenStream);
    }
    return tokenStream;
}

Example 2 with CharFilterFactory

use of org.elasticsearch.index.analysis.CharFilterFactory in project elasticsearch by elastic.

the class TransportAnalyzeAction method getCharFilterFactories.

private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry, Environment environment, CharFilterFactory[] charFilterFactories) throws IOException {
    if (request.charFilters() != null && request.charFilters().size() > 0) {
        charFilterFactories = new CharFilterFactory[request.charFilters().size()];
        for (int i = 0; i < request.charFilters().size(); i++) {
            final AnalyzeRequest.NameOrDefinition charFilter = request.charFilters().get(i);
            // parse anonymous settings
            if (charFilter.definition != null) {
                Settings settings = getAnonymousSettings(charFilter.definition);
                String charFilterTypeName = settings.get("type");
                if (charFilterTypeName == null) {
                    throw new IllegalArgumentException("Missing [type] setting for anonymous char filter: " + charFilter.definition);
                }
                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilterTypeName);
                if (charFilterFactoryFactory == null) {
                    throw new IllegalArgumentException("failed to find global char filter under [" + charFilterTypeName + "]");
                }
                // Need to set anonymous "name" of char_filter
                charFilterFactories[i] = charFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_charfilter_[" + i + "]", settings);
            } else {
                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory;
                if (indexSettings == null) {
                    charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name);
                    if (charFilterFactoryFactory == null) {
                        throw new IllegalArgumentException("failed to find global char filter under [" + charFilter.name + "]");
                    }
                    charFilterFactories[i] = charFilterFactoryFactory.get(environment, charFilter.name);
                } else {
                    charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name, indexSettings);
                    if (charFilterFactoryFactory == null) {
                        throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]");
                    }
                    charFilterFactories[i] = charFilterFactoryFactory.get(indexSettings, environment, charFilter.name, AnalysisRegistry.getSettingsFromIndexSettings(indexSettings, AnalysisRegistry.INDEX_ANALYSIS_CHAR_FILTER + "." + charFilter.name));
                }
            }
            if (charFilterFactories[i] == null) {
                throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]");
            }
        }
    }
    return charFilterFactories;
}

Example 3 with CharFilterFactory

use of org.elasticsearch.index.analysis.CharFilterFactory in project elasticsearch by elastic.

the class TransportAnalyzeAction method detailAnalyze.

private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analyzer analyzer, String field) {
    DetailAnalyzeResponse detailResponse;
    final Set<String> includeAttributes = new HashSet<>();
    if (request.attributes() != null) {
        for (String attribute : request.attributes()) {
            includeAttributes.add(attribute.toLowerCase(Locale.ROOT));
        }
    }
    CustomAnalyzer customAnalyzer = null;
    if (analyzer instanceof CustomAnalyzer) {
        customAnalyzer = (CustomAnalyzer) analyzer;
    } else if (analyzer instanceof NamedAnalyzer && ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) {
        customAnalyzer = (CustomAnalyzer) ((NamedAnalyzer) analyzer).analyzer();
    }
    if (customAnalyzer != null) {
        // customAnalyzer = divide charfilter, tokenizer tokenfilters
        CharFilterFactory[] charFilterFactories = customAnalyzer.charFilters();
        TokenizerFactory tokenizerFactory = customAnalyzer.tokenizerFactory();
        TokenFilterFactory[] tokenFilterFactories = customAnalyzer.tokenFilters();
        String[][] charFiltersTexts = new String[charFilterFactories != null ? charFilterFactories.length : 0][request.text().length];
        TokenListCreator[] tokenFiltersTokenListCreator = new TokenListCreator[tokenFilterFactories != null ? tokenFilterFactories.length : 0];
        TokenListCreator tokenizerTokenListCreator = new TokenListCreator();
        for (int textIndex = 0; textIndex < request.text().length; textIndex++) {
            String charFilteredSource = request.text()[textIndex];
            Reader reader = new FastStringReader(charFilteredSource);
            if (charFilterFactories != null) {
                for (int charFilterIndex = 0; charFilterIndex < charFilterFactories.length; charFilterIndex++) {
                    reader = charFilterFactories[charFilterIndex].create(reader);
                    Reader readerForWriteOut = new FastStringReader(charFilteredSource);
                    readerForWriteOut = charFilterFactories[charFilterIndex].create(readerForWriteOut);
                    charFilteredSource = writeCharStream(readerForWriteOut);
                    charFiltersTexts[charFilterIndex][textIndex] = charFilteredSource;
                }
            }
            // analyzing only tokenizer
            Tokenizer tokenizer = tokenizerFactory.create();
            tokenizer.setReader(reader);
            tokenizerTokenListCreator.analyze(tokenizer, customAnalyzer, field, includeAttributes);
            // analyzing each tokenfilter
            if (tokenFilterFactories != null) {
                for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFilterFactories.length; tokenFilterIndex++) {
                    if (tokenFiltersTokenListCreator[tokenFilterIndex] == null) {
                        tokenFiltersTokenListCreator[tokenFilterIndex] = new TokenListCreator();
                    }
                    TokenStream stream = createStackedTokenStream(request.text()[textIndex], charFilterFactories, tokenizerFactory, tokenFilterFactories, tokenFilterIndex + 1);
                    tokenFiltersTokenListCreator[tokenFilterIndex].analyze(stream, customAnalyzer, field, includeAttributes);
                }
            }
        }
        DetailAnalyzeResponse.CharFilteredText[] charFilteredLists = new DetailAnalyzeResponse.CharFilteredText[charFiltersTexts.length];
        if (charFilterFactories != null) {
            for (int charFilterIndex = 0; charFilterIndex < charFiltersTexts.length; charFilterIndex++) {
                charFilteredLists[charFilterIndex] = new DetailAnalyzeResponse.CharFilteredText(charFilterFactories[charFilterIndex].name(), charFiltersTexts[charFilterIndex]);
            }
        }
        DetailAnalyzeResponse.AnalyzeTokenList[] tokenFilterLists = new DetailAnalyzeResponse.AnalyzeTokenList[tokenFiltersTokenListCreator.length];
        if (tokenFilterFactories != null) {
            for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFiltersTokenListCreator.length; tokenFilterIndex++) {
                tokenFilterLists[tokenFilterIndex] = new DetailAnalyzeResponse.AnalyzeTokenList(tokenFilterFactories[tokenFilterIndex].name(), tokenFiltersTokenListCreator[tokenFilterIndex].getArrayTokens());
            }
        }
        detailResponse = new DetailAnalyzeResponse(charFilteredLists, new DetailAnalyzeResponse.AnalyzeTokenList(tokenizerFactory.name(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists);
    } else {
        String name;
        if (analyzer instanceof NamedAnalyzer) {
            name = ((NamedAnalyzer) analyzer).name();
        } else {
            name = analyzer.getClass().getName();
        }
        TokenListCreator tokenListCreator = new TokenListCreator();
        for (String text : request.text()) {
            tokenListCreator.analyze(analyzer.tokenStream(field, text), analyzer, field, includeAttributes);
        }
        detailResponse = new DetailAnalyzeResponse(new DetailAnalyzeResponse.AnalyzeTokenList(name, tokenListCreator.getArrayTokens()));
    }
    return detailResponse;
}