Examples with IndexSchema org.apache.solr.schema.IndexSchema used on opensource projects

Example 66 with IndexSchema

use of org.apache.solr.schema.IndexSchema in project lucene-solr by apache.

the class FieldOptions method process.

@Override
public void process(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    if (!params.getBool(COMPONENT_NAME, false)) {
        return;
    }
    NamedList<Object> termVectors = new NamedList<>();
    rb.rsp.add(TERM_VECTORS, termVectors);
    IndexSchema schema = rb.req.getSchema();
    SchemaField keyField = schema.getUniqueKeyField();
    String uniqFieldName = null;
    if (keyField != null) {
        uniqFieldName = keyField.getName();
    }
    FieldOptions allFields = new FieldOptions();
    //figure out what options we have, and try to get the appropriate vector
    allFields.termFreq = params.getBool(TermVectorParams.TF, false);
    allFields.positions = params.getBool(TermVectorParams.POSITIONS, false);
    allFields.offsets = params.getBool(TermVectorParams.OFFSETS, false);
    allFields.payloads = params.getBool(TermVectorParams.PAYLOADS, false);
    allFields.docFreq = params.getBool(TermVectorParams.DF, false);
    allFields.tfIdf = params.getBool(TermVectorParams.TF_IDF, false);
    //short cut to all values.
    if (params.getBool(TermVectorParams.ALL, false)) {
        allFields.termFreq = true;
        allFields.positions = true;
        allFields.offsets = true;
        allFields.payloads = true;
        allFields.docFreq = true;
        allFields.tfIdf = true;
    }
    //Build up our per field mapping
    Map<String, FieldOptions> fieldOptions = new HashMap<>();
    NamedList<List<String>> warnings = new NamedList<>();
    List<String> noTV = new ArrayList<>();
    List<String> noPos = new ArrayList<>();
    List<String> noOff = new ArrayList<>();
    List<String> noPay = new ArrayList<>();
    Set<String> fields = getFields(rb);
    if (null != fields) {
        //we have specific fields to retrieve, or no fields
        for (String field : fields) {
            // workaround SOLR-3523
            if (null == field || "score".equals(field))
                continue;
            // we don't want to issue warnings about the uniqueKey field
            // since it can cause lots of confusion in distributed requests
            // where the uniqueKey field is injected into the fl for merging
            final boolean fieldIsUniqueKey = field.equals(uniqFieldName);
            SchemaField sf = schema.getFieldOrNull(field);
            if (sf != null) {
                if (sf.storeTermVector()) {
                    FieldOptions option = fieldOptions.get(field);
                    if (option == null) {
                        option = new FieldOptions();
                        option.fieldName = field;
                        fieldOptions.put(field, option);
                    }
                    //get the per field mappings
                    option.termFreq = params.getFieldBool(field, TermVectorParams.TF, allFields.termFreq);
                    option.docFreq = params.getFieldBool(field, TermVectorParams.DF, allFields.docFreq);
                    option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf);
                    //Validate these are even an option
                    option.positions = params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions);
                    if (option.positions && !sf.storeTermPositions() && !fieldIsUniqueKey) {
                        noPos.add(field);
                    }
                    option.offsets = params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets);
                    if (option.offsets && !sf.storeTermOffsets() && !fieldIsUniqueKey) {
                        noOff.add(field);
                    }
                    option.payloads = params.getFieldBool(field, TermVectorParams.PAYLOADS, allFields.payloads);
                    if (option.payloads && !sf.storeTermPayloads() && !fieldIsUniqueKey) {
                        noPay.add(field);
                    }
                } else {
                    //field doesn't have term vectors
                    if (!fieldIsUniqueKey)
                        noTV.add(field);
                }
            } else {
                //field doesn't exist
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "undefined field: " + field);
            }
        }
    }
    // to be changed to account for that.
    if (!noTV.isEmpty()) {
        warnings.add("noTermVectors", noTV);
    }
    if (!noPos.isEmpty()) {
        warnings.add("noPositions", noPos);
    }
    if (!noOff.isEmpty()) {
        warnings.add("noOffsets", noOff);
    }
    if (!noPay.isEmpty()) {
        warnings.add("noPayloads", noPay);
    }
    if (warnings.size() > 0) {
        termVectors.add(TV_KEY_WARNINGS, warnings);
    }
    DocListAndSet listAndSet = rb.getResults();
    List<Integer> docIds = getInts(params.getParams(TermVectorParams.DOC_IDS));
    Iterator<Integer> iter;
    if (docIds != null && !docIds.isEmpty()) {
        iter = docIds.iterator();
    } else {
        DocList list = listAndSet.docList;
        iter = list.iterator();
    }
    SolrIndexSearcher searcher = rb.req.getSearcher();
    IndexReader reader = searcher.getIndexReader();
    //the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors
    //Only load the id field to get the uniqueKey of that
    //field
    final String finalUniqFieldName = uniqFieldName;
    final List<String> uniqValues = new ArrayList<>();
    // TODO: is this required to be single-valued? if so, we should STOP
    // once we find it...
    final StoredFieldVisitor getUniqValue = new StoredFieldVisitor() {

        @Override
        public void stringField(FieldInfo fieldInfo, byte[] bytes) {
            uniqValues.add(new String(bytes, StandardCharsets.UTF_8));
        }

        @Override
        public void intField(FieldInfo fieldInfo, int value) {
            uniqValues.add(Integer.toString(value));
        }

        @Override
        public void longField(FieldInfo fieldInfo, long value) {
            uniqValues.add(Long.toString(value));
        }

        @Override
        public Status needsField(FieldInfo fieldInfo) {
            return (fieldInfo.name.equals(finalUniqFieldName)) ? Status.YES : Status.NO;
        }
    };
    while (iter.hasNext()) {
        Integer docId = iter.next();
        NamedList<Object> docNL = new NamedList<>();
        if (keyField != null) {
            reader.document(docId, getUniqValue);
            String uniqVal = null;
            if (uniqValues.size() != 0) {
                uniqVal = uniqValues.get(0);
                uniqValues.clear();
                docNL.add("uniqueKey", uniqVal);
                termVectors.add(uniqVal, docNL);
            }
        } else {
            // support for schemas w/o a unique key,
            termVectors.add("doc-" + docId, docNL);
        }
        if (null != fields) {
            for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
                final String field = entry.getKey();
                final Terms vector = reader.getTermVector(docId, field);
                if (vector != null) {
                    TermsEnum termsEnum = vector.iterator();
                    mapOneVector(docNL, entry.getValue(), reader, docId, termsEnum, field);
                }
            }
        } else {
            // extract all fields
            final Fields vectors = reader.getTermVectors(docId);
            for (String field : vectors) {
                Terms terms = vectors.terms(field);
                if (terms != null) {
                    TermsEnum termsEnum = terms.iterator();
                    mapOneVector(docNL, allFields, reader, docId, termsEnum, field);
                }
            }
        }
    }
}

Example 67 with IndexSchema

use of org.apache.solr.schema.IndexSchema in project lucene-solr by apache.

the class SpellCheckComponent method inform.

@Override
public void inform(SolrCore core) {
    if (initParams != null) {
        LOG.info("Initializing spell checkers");
        boolean hasDefault = false;
        for (int i = 0; i < initParams.size(); i++) {
            if (initParams.getName(i).equals("spellchecker")) {
                Object cfg = initParams.getVal(i);
                if (cfg instanceof NamedList) {
                    addSpellChecker(core, hasDefault, (NamedList) cfg);
                } else if (cfg instanceof Map) {
                    addSpellChecker(core, hasDefault, new NamedList((Map) cfg));
                } else if (cfg instanceof List) {
                    for (Object o : (List) cfg) {
                        if (o instanceof Map) {
                            addSpellChecker(core, hasDefault, new NamedList((Map) o));
                        }
                    }
                }
            }
        }
        Map<String, QueryConverter> queryConverters = new HashMap<>();
        core.initPlugins(queryConverters, QueryConverter.class);
        //ensure that there is at least one query converter defined
        if (queryConverters.size() == 0) {
            LOG.trace("No queryConverter defined, using default converter");
            queryConverters.put("queryConverter", new SpellingQueryConverter());
        }
        //there should only be one
        if (queryConverters.size() == 1) {
            queryConverter = queryConverters.values().iterator().next();
            IndexSchema schema = core.getLatestSchema();
            String fieldTypeName = (String) initParams.get("queryAnalyzerFieldType");
            FieldType fieldType = schema.getFieldTypes().get(fieldTypeName);
            Analyzer analyzer = fieldType == null ? new WhitespaceAnalyzer() : fieldType.getQueryAnalyzer();
            //TODO: There's got to be a better way!  Where's Spring when you need it?
            queryConverter.setAnalyzer(analyzer);
        }
    }
}

Example 68 with IndexSchema

use of org.apache.solr.schema.IndexSchema in project lucene-solr by apache.

the class StatsField method computeLocalValueSourceStats.

private StatsValues computeLocalValueSourceStats(DocSet base) throws IOException {
    IndexSchema schema = searcher.getSchema();
    final StatsValues allstats = StatsValuesFactory.createStatsValues(this);
    List<FieldFacetStats> facetStats = new ArrayList<>();
    for (String facetField : facets) {
        SchemaField fsf = schema.getField(facetField);
        if (fsf.multiValued()) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Stats can only facet on single-valued fields, not: " + facetField);
        }
        facetStats.add(new FieldFacetStats(searcher, fsf, this));
    }
    final Iterator<LeafReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
    LeafReaderContext ctx = null;
    for (DocIterator docsIt = base.iterator(); docsIt.hasNext(); ) {
        final int doc = docsIt.nextDoc();
        if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
            // advance
            do {
                ctx = ctxIt.next();
            } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
            assert doc >= ctx.docBase;
            // propagate the context among accumulators.
            allstats.setNextReader(ctx);
            for (FieldFacetStats f : facetStats) {
                f.setNextReader(ctx);
            }
        }
        // accumulate
        allstats.accumulate(doc - ctx.docBase);
        for (FieldFacetStats f : facetStats) {
            f.facet(doc - ctx.docBase);
        }
    }
    for (FieldFacetStats f : facetStats) {
        allstats.addFacet(f.name, f.facetStatsValues);
    }
    return allstats;
}

Example 69 with IndexSchema

use of org.apache.solr.schema.IndexSchema in project lucene-solr by apache.

the class SolrPluginUtils method doStandardResultsDebug.

public static void doStandardResultsDebug(SolrQueryRequest req, Query query, DocList results, boolean dbgResults, NamedList dbg) throws IOException {
    if (dbgResults) {
        SolrIndexSearcher searcher = req.getSearcher();
        IndexSchema schema = searcher.getSchema();
        boolean explainStruct = req.getParams().getBool(CommonParams.EXPLAIN_STRUCT, false);
        if (results != null) {
            NamedList<Explanation> explain = getExplanations(query, results, searcher, schema);
            dbg.add("explain", explainStruct ? explanationsToNamedLists(explain) : explanationsToStrings(explain));
        }
        String otherQueryS = req.getParams().get(CommonParams.EXPLAIN_OTHER);
        if (otherQueryS != null && otherQueryS.length() > 0) {
            DocList otherResults = doSimpleQuery(otherQueryS, req, 0, 10);
            dbg.add("otherQuery", otherQueryS);
            NamedList<Explanation> explainO = getExplanations(query, otherResults, searcher, schema);
            dbg.add("explainOther", explainStruct ? explanationsToNamedLists(explainO) : explanationsToStrings(explainO));
        }
    }
}

Example 70 with IndexSchema

use of org.apache.solr.schema.IndexSchema in project lucene-solr by apache.

the class LukeRequestHandler method getIndexedFieldsInfo.

private static SimpleOrderedMap<Object> getIndexedFieldsInfo(SolrQueryRequest req) throws Exception {
    SolrIndexSearcher searcher = req.getSearcher();
    SolrParams params = req.getParams();
    Set<String> fields = null;
    String fl = params.get(CommonParams.FL);
    if (fl != null) {
        fields = new TreeSet<>(Arrays.asList(fl.split("[,\\s]+")));
    }
    LeafReader reader = searcher.getSlowAtomicReader();
    IndexSchema schema = searcher.getSchema();
    // Don't be tempted to put this in the loop below, the whole point here is to alphabetize the fields!
    Set<String> fieldNames = new TreeSet<>();
    for (FieldInfo fieldInfo : reader.getFieldInfos()) {
        fieldNames.add(fieldInfo.name);
    }
    // Walk the term enum and keep a priority queue for each map in our set
    SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<>();
    for (String fieldName : fieldNames) {
        if (fields != null && !fields.contains(fieldName) && !fields.contains("*")) {
            //we're not interested in this field Still an issue here
            continue;
        }
        SimpleOrderedMap<Object> fieldMap = new SimpleOrderedMap<>();
        SchemaField sfield = schema.getFieldOrNull(fieldName);
        FieldType ftype = (sfield == null) ? null : sfield.getType();
        fieldMap.add("type", (ftype == null) ? null : ftype.getTypeName());
        fieldMap.add("schema", getFieldFlags(sfield));
        if (sfield != null && schema.isDynamicField(sfield.getName()) && schema.getDynamicPattern(sfield.getName()) != null) {
            fieldMap.add("dynamicBase", schema.getDynamicPattern(sfield.getName()));
        }
        Terms terms = reader.fields().terms(fieldName);
        if (terms == null) {
            // Not indexed, so we need to report what we can (it made it through the fl param if specified)
            finfo.add(fieldName, fieldMap);
            continue;
        }
        if (sfield != null && sfield.indexed()) {
            if (params.getBool(INCLUDE_INDEX_FIELD_FLAGS, true)) {
                Document doc = getFirstLiveDoc(terms, reader);
                if (doc != null) {
                    // Found a document with this field
                    try {
                        IndexableField fld = doc.getField(fieldName);
                        if (fld != null) {
                            fieldMap.add("index", getFieldFlags(fld));
                        } else {
                            // it is a non-stored field...
                            fieldMap.add("index", "(unstored field)");
                        }
                    } catch (Exception ex) {
                        log.warn("error reading field: " + fieldName);
                    }
                }
            }
            fieldMap.add("docs", terms.getDocCount());
        }
        if (fields != null && (fields.contains(fieldName) || fields.contains("*"))) {
            getDetailedFieldInfo(req, fieldName, fieldMap);
        }
        // Add the field
        finfo.add(fieldName, fieldMap);
    }
    return finfo;
}