Examples with TokenizerChain org.apache.solr.analysis.TokenizerChain used on opensource projects

Example 1 with TokenizerChain

use of org.apache.solr.analysis.TokenizerChain in project lucene-solr by apache.

the class AnalysisRequestHandlerBase method analyzeValue.

/**
   * Analyzes the given value using the given Analyzer.
   *
   * @param value   Value to analyze
   * @param context The {@link AnalysisContext analysis context}.
   *
   * @return NamedList containing the tokens produced by analyzing the given value
   */
protected NamedList<? extends Object> analyzeValue(String value, AnalysisContext context) {
    Analyzer analyzer = context.getAnalyzer();
    if (!TokenizerChain.class.isInstance(analyzer)) {
        try (TokenStream tokenStream = analyzer.tokenStream(context.getFieldName(), value)) {
            NamedList<List<NamedList>> namedList = new NamedList<>();
            namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(analyzeTokenStream(tokenStream), context));
            return namedList;
        } catch (IOException e) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
        }
    }
    TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
    CharFilterFactory[] cfiltfacs = tokenizerChain.getCharFilterFactories();
    TokenizerFactory tfac = tokenizerChain.getTokenizerFactory();
    TokenFilterFactory[] filtfacs = tokenizerChain.getTokenFilterFactories();
    NamedList<Object> namedList = new NamedList<>();
    if (0 < cfiltfacs.length) {
        String source = value;
        for (CharFilterFactory cfiltfac : cfiltfacs) {
            Reader reader = new StringReader(source);
            reader = cfiltfac.create(reader);
            source = writeCharStream(namedList, reader);
        }
    }
    TokenStream tokenStream = tfac.create();
    ((Tokenizer) tokenStream).setReader(tokenizerChain.initReader(null, new StringReader(value)));
    List<AttributeSource> tokens = analyzeTokenStream(tokenStream);
    namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
    ListBasedTokenStream listBasedTokenStream = new ListBasedTokenStream(tokenStream, tokens);
    for (TokenFilterFactory tokenFilterFactory : filtfacs) {
        for (final AttributeSource tok : tokens) {
            tok.getAttribute(TokenTrackingAttribute.class).freezeStage();
        }
        // overwrite the vars "tokenStream", "tokens", and "listBasedTokenStream"
        tokenStream = tokenFilterFactory.create(listBasedTokenStream);
        tokens = analyzeTokenStream(tokenStream);
        namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
        listBasedTokenStream = new ListBasedTokenStream(listBasedTokenStream, tokens);
    }
    return namedList;
}

Example 2 with TokenizerChain

use of org.apache.solr.analysis.TokenizerChain in project lucene-solr by apache.

the class LukeRequestHandler method getAnalyzerInfo.

private static SimpleOrderedMap<Object> getAnalyzerInfo(Analyzer analyzer) {
    SimpleOrderedMap<Object> aninfo = new SimpleOrderedMap<>();
    aninfo.add("className", analyzer.getClass().getName());
    if (analyzer instanceof TokenizerChain) {
        TokenizerChain tchain = (TokenizerChain) analyzer;
        CharFilterFactory[] cfiltfacs = tchain.getCharFilterFactories();
        if (0 < cfiltfacs.length) {
            SimpleOrderedMap<Map<String, Object>> cfilters = new SimpleOrderedMap<>();
            for (CharFilterFactory cfiltfac : cfiltfacs) {
                Map<String, Object> tok = new HashMap<>();
                String className = cfiltfac.getClass().getName();
                tok.put("className", className);
                tok.put("args", cfiltfac.getOriginalArgs());
                cfilters.add(className.substring(className.lastIndexOf('.') + 1), tok);
            }
            aninfo.add("charFilters", cfilters);
        }
        SimpleOrderedMap<Object> tokenizer = new SimpleOrderedMap<>();
        TokenizerFactory tfac = tchain.getTokenizerFactory();
        tokenizer.add("className", tfac.getClass().getName());
        tokenizer.add("args", tfac.getOriginalArgs());
        aninfo.add("tokenizer", tokenizer);
        TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
        if (0 < filtfacs.length) {
            SimpleOrderedMap<Map<String, Object>> filters = new SimpleOrderedMap<>();
            for (TokenFilterFactory filtfac : filtfacs) {
                Map<String, Object> tok = new HashMap<>();
                String className = filtfac.getClass().getName();
                tok.put("className", className);
                tok.put("args", filtfac.getOriginalArgs());
                filters.add(className.substring(className.lastIndexOf('.') + 1), tok);
            }
            aninfo.add("filters", filters);
        }
    }
    return aninfo;
}

Example 3 with TokenizerChain

use of org.apache.solr.analysis.TokenizerChain in project lucene-solr by apache.

the class ManagedIndexSchema method informResourceLoaderAwareObjectsForFieldType.

/**
   * Informs analyzers used by a fieldType.
   */
protected void informResourceLoaderAwareObjectsForFieldType(FieldType fieldType) {
    // tokenizer chain if they are ResourceLoaderAware
    if (!fieldType.supportsAnalyzers())
        return;
    Analyzer indexAnalyzer = fieldType.getIndexAnalyzer();
    if (indexAnalyzer != null && indexAnalyzer instanceof TokenizerChain)
        informResourceLoaderAwareObjectsInChain((TokenizerChain) indexAnalyzer);
    Analyzer queryAnalyzer = fieldType.getQueryAnalyzer();
    // actually safe to call inform multiple times on an object anyway
    if (queryAnalyzer != null && queryAnalyzer != indexAnalyzer && queryAnalyzer instanceof TokenizerChain)
        informResourceLoaderAwareObjectsInChain((TokenizerChain) queryAnalyzer);
    // if fieldType is a TextField, it might have a multi-term analyzer
    if (fieldType instanceof TextField) {
        TextField textFieldType = (TextField) fieldType;
        Analyzer multiTermAnalyzer = textFieldType.getMultiTermAnalyzer();
        if (multiTermAnalyzer != null && multiTermAnalyzer != indexAnalyzer && multiTermAnalyzer != queryAnalyzer && multiTermAnalyzer instanceof TokenizerChain)
            informResourceLoaderAwareObjectsInChain((TokenizerChain) multiTermAnalyzer);
    }
}

Example 4 with TokenizerChain

use of org.apache.solr.analysis.TokenizerChain in project lucene-solr by apache.

the class FieldType method getAnalyzerProperties.

/** 
   * Returns a description of the given analyzer, by either reporting the Analyzer class
   * name (and optionally luceneMatchVersion) if it's not a TokenizerChain, or if it is,
   * querying each analysis factory for its name and args.
   */
protected static SimpleOrderedMap<Object> getAnalyzerProperties(Analyzer analyzer) {
    SimpleOrderedMap<Object> analyzerProps = new SimpleOrderedMap<>();
    if (analyzer instanceof TokenizerChain) {
        Map<String, String> factoryArgs;
        TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
        CharFilterFactory[] charFilterFactories = tokenizerChain.getCharFilterFactories();
        if (0 < charFilterFactories.length) {
            List<SimpleOrderedMap<Object>> charFilterProps = new ArrayList<>();
            for (CharFilterFactory charFilterFactory : charFilterFactories) {
                SimpleOrderedMap<Object> props = new SimpleOrderedMap<>();
                props.add(CLASS_NAME, charFilterFactory.getClassArg());
                factoryArgs = charFilterFactory.getOriginalArgs();
                if (null != factoryArgs) {
                    for (String key : factoryArgs.keySet()) {
                        if (!CLASS_NAME.equals(key)) {
                            if (LUCENE_MATCH_VERSION_PARAM.equals(key)) {
                                if (charFilterFactory.isExplicitLuceneMatchVersion()) {
                                    props.add(key, factoryArgs.get(key));
                                }
                            } else {
                                props.add(key, factoryArgs.get(key));
                            }
                        }
                    }
                }
                charFilterProps.add(props);
            }
            analyzerProps.add(CHAR_FILTERS, charFilterProps);
        }
        SimpleOrderedMap<Object> tokenizerProps = new SimpleOrderedMap<>();
        TokenizerFactory tokenizerFactory = tokenizerChain.getTokenizerFactory();
        tokenizerProps.add(CLASS_NAME, tokenizerFactory.getClassArg());
        factoryArgs = tokenizerFactory.getOriginalArgs();
        if (null != factoryArgs) {
            for (String key : factoryArgs.keySet()) {
                if (!CLASS_NAME.equals(key)) {
                    if (LUCENE_MATCH_VERSION_PARAM.equals(key)) {
                        if (tokenizerFactory.isExplicitLuceneMatchVersion()) {
                            tokenizerProps.add(key, factoryArgs.get(key));
                        }
                    } else {
                        tokenizerProps.add(key, factoryArgs.get(key));
                    }
                }
            }
        }
        analyzerProps.add(TOKENIZER, tokenizerProps);
        TokenFilterFactory[] filterFactories = tokenizerChain.getTokenFilterFactories();
        if (0 < filterFactories.length) {
            List<SimpleOrderedMap<Object>> filterProps = new ArrayList<>();
            for (TokenFilterFactory filterFactory : filterFactories) {
                SimpleOrderedMap<Object> props = new SimpleOrderedMap<>();
                props.add(CLASS_NAME, filterFactory.getClassArg());
                factoryArgs = filterFactory.getOriginalArgs();
                if (null != factoryArgs) {
                    for (String key : factoryArgs.keySet()) {
                        if (!CLASS_NAME.equals(key)) {
                            if (LUCENE_MATCH_VERSION_PARAM.equals(key)) {
                                if (filterFactory.isExplicitLuceneMatchVersion()) {
                                    props.add(key, factoryArgs.get(key));
                                }
                            } else {
                                props.add(key, factoryArgs.get(key));
                            }
                        }
                    }
                }
                filterProps.add(props);
            }
            analyzerProps.add(FILTERS, filterProps);
        }
    } else {
        // analyzer is not instanceof TokenizerChain
        analyzerProps.add(CLASS_NAME, analyzer.getClass().getName());
        if (analyzer.getVersion() != Version.LATEST) {
            analyzerProps.add(LUCENE_MATCH_VERSION_PARAM, analyzer.getVersion().toString());
        }
    }
    return analyzerProps;
}

Example 5 with TokenizerChain

use of org.apache.solr.analysis.TokenizerChain in project lucene-solr by apache.

the class FieldTypePluginLoader method readAnalyzer.

//
// <analyzer><tokenizer class="...."/><tokenizer class="...." arg="....">
//
//
private Analyzer readAnalyzer(Node node) throws XPathExpressionException {
    final SolrResourceLoader loader = schema.getResourceLoader();
    if (node == null)
        return null;
    NamedNodeMap attrs = node.getAttributes();
    String analyzerName = DOMUtil.getAttr(attrs, "class");
    // check for all of these up front, so we can error if used in 
    // conjunction with an explicit analyzer class.
    NodeList charFilterNodes = (NodeList) xpath.evaluate("./charFilter", node, XPathConstants.NODESET);
    NodeList tokenizerNodes = (NodeList) xpath.evaluate("./tokenizer", node, XPathConstants.NODESET);
    NodeList tokenFilterNodes = (NodeList) xpath.evaluate("./filter", node, XPathConstants.NODESET);
    if (analyzerName != null) {
        // own custom nodes (ie: <description> or something like that)
        if (0 != charFilterNodes.getLength() || 0 != tokenizerNodes.getLength() || 0 != tokenFilterNodes.getLength()) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Configuration Error: Analyzer class='" + analyzerName + "' can not be combined with nested analysis factories");
        }
        try {
            // No need to be core-aware as Analyzers are not in the core-aware list
            final Class<? extends Analyzer> clazz = loader.findClass(analyzerName, Analyzer.class);
            Analyzer analyzer = clazz.newInstance();
            final String matchVersionStr = DOMUtil.getAttr(attrs, LUCENE_MATCH_VERSION_PARAM);
            final Version luceneMatchVersion = (matchVersionStr == null) ? schema.getDefaultLuceneMatchVersion() : Config.parseLuceneVersionString(matchVersionStr);
            if (luceneMatchVersion == null) {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Configuration Error: Analyzer '" + clazz.getName() + "' needs a 'luceneMatchVersion' parameter");
            }
            analyzer.setVersion(luceneMatchVersion);
            return analyzer;
        } catch (Exception e) {
            log.error("Cannot load analyzer: " + analyzerName, e);
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Cannot load analyzer: " + analyzerName, e);
        }
    }
    // Load the CharFilters
    final ArrayList<CharFilterFactory> charFilters = new ArrayList<>();
    AbstractPluginLoader<CharFilterFactory> charFilterLoader = new AbstractPluginLoader<CharFilterFactory>("[schema.xml] analyzer/charFilter", CharFilterFactory.class, false, false) {

        @Override
        protected CharFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception {
            final Map<String, String> params = DOMUtil.toMap(node.getAttributes());
            String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM);
            params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, CharFilterFactory.class.getSimpleName()).toString());
            CharFilterFactory factory = loader.newInstance(className, CharFilterFactory.class, getDefaultPackages(), new Class[] { Map.class }, new Object[] { params });
            factory.setExplicitLuceneMatchVersion(null != configuredVersion);
            return factory;
        }

        @Override
        protected void init(CharFilterFactory plugin, Node node) throws Exception {
            if (plugin != null) {
                charFilters.add(plugin);
            }
        }

        @Override
        protected CharFilterFactory register(String name, CharFilterFactory plugin) {
            // used for map registration
            return null;
        }
    };
    charFilterLoader.load(loader, charFilterNodes);
    // Load the Tokenizer
    // Although an analyzer only allows a single Tokenizer, we load a list to make sure
    // the configuration is ok
    final ArrayList<TokenizerFactory> tokenizers = new ArrayList<>(1);
    AbstractPluginLoader<TokenizerFactory> tokenizerLoader = new AbstractPluginLoader<TokenizerFactory>("[schema.xml] analyzer/tokenizer", TokenizerFactory.class, false, false) {

        @Override
        protected TokenizerFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception {
            final Map<String, String> params = DOMUtil.toMap(node.getAttributes());
            String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM);
            params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenizerFactory.class.getSimpleName()).toString());
            TokenizerFactory factory = loader.newInstance(className, TokenizerFactory.class, getDefaultPackages(), new Class[] { Map.class }, new Object[] { params });
            factory.setExplicitLuceneMatchVersion(null != configuredVersion);
            return factory;
        }

        @Override
        protected void init(TokenizerFactory plugin, Node node) throws Exception {
            if (!tokenizers.isEmpty()) {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The schema defines multiple tokenizers for: " + node);
            }
            tokenizers.add(plugin);
        }

        @Override
        protected TokenizerFactory register(String name, TokenizerFactory plugin) {
            // used for map registration
            return null;
        }
    };
    tokenizerLoader.load(loader, tokenizerNodes);
    // Make sure something was loaded
    if (tokenizers.isEmpty()) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "analyzer without class or tokenizer");
    }
    // Load the Filters
    final ArrayList<TokenFilterFactory> filters = new ArrayList<>();
    AbstractPluginLoader<TokenFilterFactory> filterLoader = new AbstractPluginLoader<TokenFilterFactory>("[schema.xml] analyzer/filter", TokenFilterFactory.class, false, false) {

        @Override
        protected TokenFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception {
            final Map<String, String> params = DOMUtil.toMap(node.getAttributes());
            String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM);
            params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenFilterFactory.class.getSimpleName()).toString());
            TokenFilterFactory factory = loader.newInstance(className, TokenFilterFactory.class, getDefaultPackages(), new Class[] { Map.class }, new Object[] { params });
            factory.setExplicitLuceneMatchVersion(null != configuredVersion);
            return factory;
        }

        @Override
        protected void init(TokenFilterFactory plugin, Node node) throws Exception {
            if (plugin != null) {
                filters.add(plugin);
            }
        }

        @Override
        protected TokenFilterFactory register(String name, TokenFilterFactory plugin) throws Exception {
            // used for map registration
            return null;
        }
    };
    filterLoader.load(loader, tokenFilterNodes);
    return new TokenizerChain(charFilters.toArray(new CharFilterFactory[charFilters.size()]), tokenizers.get(0), filters.toArray(new TokenFilterFactory[filters.size()]));
}