Search in sources :

Example 11 with SortedDocValues

use of org.apache.lucene.index.SortedDocValues in project lucene-solr by apache.

the class ReverseOrdFieldSource method getValues.

@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
    final int off = readerContext.docBase;
    final LeafReader r;
    Object o = context.get("searcher");
    if (o instanceof SolrIndexSearcher) {
        SolrIndexSearcher is = (SolrIndexSearcher) o;
        SchemaField sf = is.getSchema().getFieldOrNull(field);
        if (sf != null && sf.hasDocValues() == false && sf.multiValued() == false && sf.getType().getNumberType() != null) {
            // it's a single-valued numeric field: we must currently create insanity :(
            List<LeafReaderContext> leaves = is.getIndexReader().leaves();
            LeafReader[] insaneLeaves = new LeafReader[leaves.size()];
            int upto = 0;
            for (LeafReaderContext raw : leaves) {
                insaneLeaves[upto++] = Insanity.wrapInsanity(raw.reader(), field);
            }
            r = SlowCompositeReaderWrapper.wrap(new MultiReader(insaneLeaves));
        } else {
            // reuse ordinalmap
            r = ((SolrIndexSearcher) o).getSlowAtomicReader();
        }
    } else {
        IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
        r = SlowCompositeReaderWrapper.wrap(topReader);
    }
    // if it's e.g. tokenized/multivalued, emulate old behavior of single-valued fc
    final SortedDocValues sindex = SortedSetSelector.wrap(DocValues.getSortedSet(r, field), SortedSetSelector.Type.MIN);
    final int end = sindex.getValueCount();
    return new IntDocValues(this) {

        @Override
        public int intVal(int doc) throws IOException {
            if (doc + off > sindex.docID()) {
                sindex.advance(doc + off);
            }
            if (doc + off == sindex.docID()) {
                return (end - sindex.ordValue() - 1);
            } else {
                return end;
            }
        }
    };
}
Also used : SchemaField(org.apache.solr.schema.SchemaField) LeafReader(org.apache.lucene.index.LeafReader) MultiReader(org.apache.lucene.index.MultiReader) IndexReader(org.apache.lucene.index.IndexReader) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) SolrIndexSearcher(org.apache.solr.search.SolrIndexSearcher) IntDocValues(org.apache.lucene.queries.function.docvalues.IntDocValues) SortedDocValues(org.apache.lucene.index.SortedDocValues)

Example 12 with SortedDocValues

use of org.apache.lucene.index.SortedDocValues in project lucene-solr by apache.

the class FacetFieldProcessorByHashDV method collectDocs.

private void collectDocs() throws IOException {
    if (calc instanceof TermOrdCalc) {
        // Strings
        // TODO support SortedSetDocValues
        SortedDocValues globalDocValues = FieldUtil.getSortedDocValues(fcontext.qcontext, sf, null);
        ((TermOrdCalc) calc).lookupOrdFunction = ord -> {
            try {
                return globalDocValues.lookupOrd(ord);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        DocSetUtil.collectSortedDocSet(fcontext.base, fcontext.searcher.getIndexReader(), new SimpleCollector() {

            // this segment/leaf. NN
            SortedDocValues docValues = globalDocValues;

            // this segment to global ordinal. NN
            LongValues toGlobal = LongValues.IDENTITY;

            @Override
            public boolean needsScores() {
                return false;
            }

            @Override
            protected void doSetNextReader(LeafReaderContext ctx) throws IOException {
                setNextReaderFirstPhase(ctx);
                if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
                    MultiDocValues.MultiSortedDocValues multiDocValues = (MultiDocValues.MultiSortedDocValues) globalDocValues;
                    docValues = multiDocValues.values[ctx.ord];
                    toGlobal = multiDocValues.mapping.getGlobalOrds(ctx.ord);
                }
            }

            @Override
            public void collect(int segDoc) throws IOException {
                if (segDoc > docValues.docID()) {
                    docValues.advance(segDoc);
                }
                if (segDoc == docValues.docID()) {
                    long val = toGlobal.get(docValues.ordValue());
                    collectValFirstPhase(segDoc, val);
                }
            }
        });
    } else {
        // Numeric:
        // TODO support SortedNumericDocValues
        DocSetUtil.collectSortedDocSet(fcontext.base, fcontext.searcher.getIndexReader(), new SimpleCollector() {

            //NN
            NumericDocValues values = null;

            @Override
            public boolean needsScores() {
                return false;
            }

            @Override
            protected void doSetNextReader(LeafReaderContext ctx) throws IOException {
                setNextReaderFirstPhase(ctx);
                values = DocValues.getNumeric(ctx.reader(), sf.getName());
            }

            @Override
            public void collect(int segDoc) throws IOException {
                if (segDoc > values.docID()) {
                    values.advance(segDoc);
                }
                if (segDoc == values.docID()) {
                    collectValFirstPhase(segDoc, values.longValue());
                }
            }
        });
    }
}
Also used : NumericDocValues(org.apache.lucene.index.NumericDocValues) IOException(java.io.IOException) MultiDocValues(org.apache.lucene.index.MultiDocValues) SortedDocValues(org.apache.lucene.index.SortedDocValues) SimpleCollector(org.apache.lucene.search.SimpleCollector) LongValues(org.apache.lucene.util.LongValues) LeafReaderContext(org.apache.lucene.index.LeafReaderContext)

Example 13 with SortedDocValues

use of org.apache.lucene.index.SortedDocValues in project lucene-solr by apache.

the class UninvertingReader method getSortedDocValues.

@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
    SortedDocValues values = in.getSortedDocValues(field);
    if (values != null) {
        return values;
    }
    Type v = getType(field);
    if (v == Type.SORTED) {
        return FieldCache.DEFAULT.getTermsIndex(in, field);
    } else {
        return null;
    }
}
Also used : DocValuesType(org.apache.lucene.index.DocValuesType) SortedDocValues(org.apache.lucene.index.SortedDocValues)

Example 14 with SortedDocValues

use of org.apache.lucene.index.SortedDocValues in project lucene-solr by apache.

the class ExportWriter method getSortDoc.

private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
    SortValue[] sortValues = new SortValue[sortFields.length];
    IndexSchema schema = searcher.getSchema();
    for (int i = 0; i < sortFields.length; ++i) {
        SortField sf = sortFields[i];
        String field = sf.getField();
        boolean reverse = sf.getReverse();
        SchemaField schemaField = schema.getField(field);
        FieldType ft = schemaField.getType();
        if (!schemaField.hasDocValues()) {
            throw new IOException(field + " must have DocValues to use this feature.");
        }
        if (ft instanceof TrieIntField) {
            if (reverse) {
                sortValues[i] = new IntValue(field, new IntDesc());
            } else {
                sortValues[i] = new IntValue(field, new IntAsc());
            }
        } else if (ft instanceof TrieFloatField) {
            if (reverse) {
                sortValues[i] = new FloatValue(field, new FloatDesc());
            } else {
                sortValues[i] = new FloatValue(field, new FloatAsc());
            }
        } else if (ft instanceof TrieDoubleField) {
            if (reverse) {
                sortValues[i] = new DoubleValue(field, new DoubleDesc());
            } else {
                sortValues[i] = new DoubleValue(field, new DoubleAsc());
            }
        } else if (ft instanceof TrieLongField) {
            if (reverse) {
                sortValues[i] = new LongValue(field, new LongDesc());
            } else {
                sortValues[i] = new LongValue(field, new LongAsc());
            }
        } else if (ft instanceof StrField) {
            LeafReader reader = searcher.getSlowAtomicReader();
            SortedDocValues vals = reader.getSortedDocValues(field);
            if (reverse) {
                sortValues[i] = new StringValue(vals, field, new IntDesc());
            } else {
                sortValues[i] = new StringValue(vals, field, new IntAsc());
            }
        } else if (ft instanceof TrieDateField) {
            if (reverse) {
                sortValues[i] = new LongValue(field, new LongDesc());
            } else {
                sortValues[i] = new LongValue(field, new LongAsc());
            }
        } else if (ft instanceof BoolField) {
            // This is a bit of a hack, but since the boolean field stores ByteRefs, just like Strings
            // _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
            // we can just use the existing StringValue here.
            LeafReader reader = searcher.getSlowAtomicReader();
            SortedDocValues vals = reader.getSortedDocValues(field);
            if (reverse) {
                sortValues[i] = new StringValue(vals, field, new IntDesc());
            } else {
                sortValues[i] = new StringValue(vals, field, new IntAsc());
            }
        } else {
            throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
        }
    }
    if (sortValues.length == 1) {
        return new SingleValueSortDoc(sortValues[0]);
    } else if (sortValues.length == 2) {
        return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
    } else if (sortValues.length == 3) {
        return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
    } else if (sortValues.length == 4) {
        return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
    } else {
        throw new IOException("A max of 4 sorts can be specified");
    }
}
Also used : StrField(org.apache.solr.schema.StrField) TrieIntField(org.apache.solr.schema.TrieIntField) SortField(org.apache.lucene.search.SortField) TrieDateField(org.apache.solr.schema.TrieDateField) TrieLongField(org.apache.solr.schema.TrieLongField) TrieFloatField(org.apache.solr.schema.TrieFloatField) TrieDoubleField(org.apache.solr.schema.TrieDoubleField) BoolField(org.apache.solr.schema.BoolField) LeafReader(org.apache.lucene.index.LeafReader) IOException(java.io.IOException) SortedDocValues(org.apache.lucene.index.SortedDocValues) FieldType(org.apache.solr.schema.FieldType) SchemaField(org.apache.solr.schema.SchemaField) IndexSchema(org.apache.solr.schema.IndexSchema)

Example 15 with SortedDocValues

use of org.apache.lucene.index.SortedDocValues in project lucene-solr by apache.

the class SolrDocumentFetcher method decorateDocValueFields.

/**
   * This will fetch and add the docValues fields to a given SolrDocument/SolrInputDocument
   *
   * @param doc
   *          A SolrDocument or SolrInputDocument instance where docValues will be added
   * @param docid
   *          The lucene docid of the document to be populated
   * @param fields
   *          The list of docValues fields to be decorated
   */
public void decorateDocValueFields(@SuppressWarnings("rawtypes") SolrDocumentBase doc, int docid, Set<String> fields) throws IOException {
    final List<LeafReaderContext> leafContexts = searcher.getLeafContexts();
    final int subIndex = ReaderUtil.subIndex(docid, leafContexts);
    final int localId = docid - leafContexts.get(subIndex).docBase;
    final LeafReader leafReader = leafContexts.get(subIndex).reader();
    for (String fieldName : fields) {
        final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldName);
        if (schemaField == null || !schemaField.hasDocValues() || doc.containsKey(fieldName)) {
            log.warn("Couldn't decorate docValues for field: [{}], schemaField: [{}]", fieldName, schemaField);
            continue;
        }
        FieldInfo fi = searcher.getFieldInfos().fieldInfo(fieldName);
        if (fi == null) {
            // Searcher doesn't have info about this field, hence ignore it.
            continue;
        }
        final DocValuesType dvType = fi.getDocValuesType();
        switch(dvType) {
            case NUMERIC:
                final NumericDocValues ndv = leafReader.getNumericDocValues(fieldName);
                if (ndv == null) {
                    continue;
                }
                Long val;
                if (ndv.advanceExact(localId)) {
                    val = ndv.longValue();
                } else {
                    continue;
                }
                Object newVal = val;
                if (schemaField.getType().isPointField()) {
                    // TODO: Maybe merge PointField with TrieFields here
                    NumberType type = schemaField.getType().getNumberType();
                    switch(type) {
                        case INTEGER:
                            newVal = val.intValue();
                            break;
                        case LONG:
                            newVal = val.longValue();
                            break;
                        case FLOAT:
                            newVal = Float.intBitsToFloat(val.intValue());
                            break;
                        case DOUBLE:
                            newVal = Double.longBitsToDouble(val);
                            break;
                        case DATE:
                            newVal = new Date(val);
                            break;
                        default:
                            throw new AssertionError("Unexpected PointType: " + type);
                    }
                } else {
                    if (schemaField.getType() instanceof TrieIntField) {
                        newVal = val.intValue();
                    } else if (schemaField.getType() instanceof TrieFloatField) {
                        newVal = Float.intBitsToFloat(val.intValue());
                    } else if (schemaField.getType() instanceof TrieDoubleField) {
                        newVal = Double.longBitsToDouble(val);
                    } else if (schemaField.getType() instanceof TrieDateField) {
                        newVal = new Date(val);
                    } else if (schemaField.getType() instanceof EnumField) {
                        newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue());
                    }
                }
                doc.addField(fieldName, newVal);
                break;
            case BINARY:
                BinaryDocValues bdv = leafReader.getBinaryDocValues(fieldName);
                if (bdv == null) {
                    continue;
                }
                BytesRef value;
                if (bdv.advanceExact(localId)) {
                    value = BytesRef.deepCopyOf(bdv.binaryValue());
                } else {
                    continue;
                }
                doc.addField(fieldName, value);
                break;
            case SORTED:
                SortedDocValues sdv = leafReader.getSortedDocValues(fieldName);
                if (sdv == null) {
                    continue;
                }
                if (sdv.advanceExact(localId)) {
                    final BytesRef bRef = sdv.binaryValue();
                    // Special handling for Boolean fields since they're stored as 'T' and 'F'.
                    if (schemaField.getType() instanceof BoolField) {
                        doc.addField(fieldName, schemaField.getType().toObject(schemaField, bRef));
                    } else {
                        doc.addField(fieldName, bRef.utf8ToString());
                    }
                }
                break;
            case SORTED_NUMERIC:
                final SortedNumericDocValues numericDv = leafReader.getSortedNumericDocValues(fieldName);
                NumberType type = schemaField.getType().getNumberType();
                if (numericDv != null) {
                    if (numericDv.advance(localId) == localId) {
                        final List<Object> outValues = new ArrayList<Object>(numericDv.docValueCount());
                        for (int i = 0; i < numericDv.docValueCount(); i++) {
                            long number = numericDv.nextValue();
                            switch(type) {
                                case INTEGER:
                                    outValues.add((int) number);
                                    break;
                                case LONG:
                                    outValues.add(number);
                                    break;
                                case FLOAT:
                                    outValues.add(NumericUtils.sortableIntToFloat((int) number));
                                    break;
                                case DOUBLE:
                                    outValues.add(NumericUtils.sortableLongToDouble(number));
                                    break;
                                case DATE:
                                    outValues.add(new Date(number));
                                    break;
                                default:
                                    throw new AssertionError("Unexpected PointType: " + type);
                            }
                        }
                        assert outValues.size() > 0;
                        doc.addField(fieldName, outValues);
                    }
                }
            case SORTED_SET:
                final SortedSetDocValues values = leafReader.getSortedSetDocValues(fieldName);
                if (values != null && values.getValueCount() > 0) {
                    if (values.advance(localId) == localId) {
                        final List<Object> outValues = new LinkedList<>();
                        for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
                            value = values.lookupOrd(ord);
                            outValues.add(schemaField.getType().toObject(schemaField, value));
                        }
                        assert outValues.size() > 0;
                        doc.addField(fieldName, outValues);
                    }
                }
            case NONE:
                break;
        }
    }
}
Also used : NumericDocValues(org.apache.lucene.index.NumericDocValues) SortedNumericDocValues(org.apache.lucene.index.SortedNumericDocValues) SortedNumericDocValues(org.apache.lucene.index.SortedNumericDocValues) TrieIntField(org.apache.solr.schema.TrieIntField) ArrayList(java.util.ArrayList) TrieDateField(org.apache.solr.schema.TrieDateField) BinaryDocValues(org.apache.lucene.index.BinaryDocValues) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) DocValuesType(org.apache.lucene.index.DocValuesType) TrieFloatField(org.apache.solr.schema.TrieFloatField) BytesRef(org.apache.lucene.util.BytesRef) TrieDoubleField(org.apache.solr.schema.TrieDoubleField) EnumField(org.apache.solr.schema.EnumField) BoolField(org.apache.solr.schema.BoolField) LeafReader(org.apache.lucene.index.LeafReader) Date(java.util.Date) SortedDocValues(org.apache.lucene.index.SortedDocValues) LinkedList(java.util.LinkedList) SchemaField(org.apache.solr.schema.SchemaField) NumberType(org.apache.solr.schema.NumberType) SortedSetDocValues(org.apache.lucene.index.SortedSetDocValues) FieldInfo(org.apache.lucene.index.FieldInfo)

Aggregations

SortedDocValues (org.apache.lucene.index.SortedDocValues)66 BytesRef (org.apache.lucene.util.BytesRef)32 LeafReaderContext (org.apache.lucene.index.LeafReaderContext)27 SortedSetDocValues (org.apache.lucene.index.SortedSetDocValues)23 LeafReader (org.apache.lucene.index.LeafReader)22 Document (org.apache.lucene.document.Document)21 NumericDocValues (org.apache.lucene.index.NumericDocValues)15 Directory (org.apache.lucene.store.Directory)15 RandomIndexWriter (org.apache.lucene.index.RandomIndexWriter)14 SortedDocValuesField (org.apache.lucene.document.SortedDocValuesField)13 IOException (java.io.IOException)12 BinaryDocValues (org.apache.lucene.index.BinaryDocValues)12 MockAnalyzer (org.apache.lucene.analysis.MockAnalyzer)10 MultiDocValues (org.apache.lucene.index.MultiDocValues)10 ArrayList (java.util.ArrayList)9 IndexReader (org.apache.lucene.index.IndexReader)9 OrdinalMap (org.apache.lucene.index.MultiDocValues.OrdinalMap)9 DoublePoint (org.apache.lucene.document.DoublePoint)8 FloatPoint (org.apache.lucene.document.FloatPoint)8 IntPoint (org.apache.lucene.document.IntPoint)8