Examples with SortedNumericDocValues org.apache.lucene.index.SortedNumericDocValues used on opensource projects

Example 16 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class ToParentBlockJoinSortField method getLongComparator.

private FieldComparator<?> getLongComparator(int numHits) {
    return new FieldComparator.LongComparator(numHits, getField(), (Long) missingValue) {

        @Override
        protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
            SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
            final BlockJoinSelector.Type type = order ? BlockJoinSelector.Type.MAX : BlockJoinSelector.Type.MIN;
            final BitSet parents = parentFilter.getBitSet(context);
            final BitSet children = childFilter.getBitSet(context);
            if (children == null) {
                return DocValues.emptyNumeric();
            }
            return BlockJoinSelector.wrap(sortedNumeric, type, parents, children);
        }
    };
}

Example 17 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class Lucene70DocValuesConsumer method addNumericField.

@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
    meta.writeInt(field.number);
    meta.writeByte(Lucene70DocValuesFormat.NUMERIC);
    writeValues(field, new EmptyDocValuesProducer() {

        @Override
        public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
            return DocValues.singleton(valuesProducer.getNumeric(field));
        }
    });
}

Example 18 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class TestLucene54DocValuesFormat method testSortedNumericAroundBlockSize.

@Slow
public void testSortedNumericAroundBlockSize() throws IOException {
    final int frontier = 1 << Lucene54DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
    for (int maxDoc = frontier - 1; maxDoc <= frontier + 1; ++maxDoc) {
        final Directory dir = newDirectory();
        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
        RAMFile buffer = new RAMFile();
        RAMOutputStream out = new RAMOutputStream(buffer, false);
        Document doc = new Document();
        SortedNumericDocValuesField field1 = new SortedNumericDocValuesField("snum", 0L);
        doc.add(field1);
        SortedNumericDocValuesField field2 = new SortedNumericDocValuesField("snum", 0L);
        doc.add(field2);
        for (int i = 0; i < maxDoc; ++i) {
            long s1 = random().nextInt(100);
            long s2 = random().nextInt(100);
            field1.setLongValue(s1);
            field2.setLongValue(s2);
            w.addDocument(doc);
            out.writeVLong(Math.min(s1, s2));
            out.writeVLong(Math.max(s1, s2));
        }
        out.close();
        w.forceMerge(1);
        DirectoryReader r = DirectoryReader.open(w);
        w.close();
        LeafReader sr = getOnlyLeafReader(r);
        assertEquals(maxDoc, sr.maxDoc());
        SortedNumericDocValues values = sr.getSortedNumericDocValues("snum");
        assertNotNull(values);
        RAMInputStream in = new RAMInputStream("", buffer);
        for (int i = 0; i < maxDoc; ++i) {
            assertEquals(i, values.nextDoc());
            assertEquals(2, values.docValueCount());
            assertEquals(in.readVLong(), values.nextValue());
            assertEquals(in.readVLong(), values.nextValue());
        }
        r.close();
        dir.close();
    }
}

Example 19 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class TestLucene54DocValuesFormat method doTestSparseDocValuesVsStoredFields.

private void doTestSparseDocValuesVsStoredFields() throws Exception {
    final long[] values = new long[TestUtil.nextInt(random(), 1, 500)];
    for (int i = 0; i < values.length; ++i) {
        values[i] = random().nextLong();
    }
    Directory dir = newFSDirectory(createTempDir());
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    conf.setMergeScheduler(new SerialMergeScheduler());
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
    // sparse compression is only enabled if less than 1% of docs have a value
    final int avgGap = 100;
    final int numDocs = atLeast(200);
    for (int i = random().nextInt(avgGap * 2); i >= 0; --i) {
        writer.addDocument(new Document());
    }
    final int maxNumValuesPerDoc = random().nextBoolean() ? 1 : TestUtil.nextInt(random(), 2, 5);
    for (int i = 0; i < numDocs; ++i) {
        Document doc = new Document();
        // single-valued
        long docValue = values[random().nextInt(values.length)];
        doc.add(new NumericDocValuesField("numeric", docValue));
        doc.add(new SortedDocValuesField("sorted", new BytesRef(Long.toString(docValue))));
        doc.add(new BinaryDocValuesField("binary", new BytesRef(Long.toString(docValue))));
        doc.add(new StoredField("value", docValue));
        // multi-valued
        final int numValues = TestUtil.nextInt(random(), 1, maxNumValuesPerDoc);
        for (int j = 0; j < numValues; ++j) {
            docValue = values[random().nextInt(values.length)];
            doc.add(new SortedNumericDocValuesField("sorted_numeric", docValue));
            doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef(Long.toString(docValue))));
            doc.add(new StoredField("values", docValue));
        }
        writer.addDocument(doc);
        // add a gap
        for (int j = TestUtil.nextInt(random(), 0, avgGap * 2); j >= 0; --j) {
            writer.addDocument(new Document());
        }
    }
    if (random().nextBoolean()) {
        writer.forceMerge(1);
    }
    final IndexReader indexReader = writer.getReader();
    TestUtil.checkReader(indexReader);
    writer.close();
    for (LeafReaderContext context : indexReader.leaves()) {
        final LeafReader reader = context.reader();
        final NumericDocValues numeric = DocValues.getNumeric(reader, "numeric");
        final SortedDocValues sorted = DocValues.getSorted(reader, "sorted");
        final BinaryDocValues binary = DocValues.getBinary(reader, "binary");
        final SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(reader, "sorted_numeric");
        final SortedSetDocValues sortedSet = DocValues.getSortedSet(reader, "sorted_set");
        for (int i = 0; i < reader.maxDoc(); ++i) {
            final Document doc = reader.document(i);
            final IndexableField valueField = doc.getField("value");
            final Long value = valueField == null ? null : valueField.numericValue().longValue();
            if (value == null) {
                assertTrue(numeric.docID() + " vs " + i, numeric.docID() < i);
            } else {
                assertEquals(i, numeric.nextDoc());
                assertEquals(i, binary.nextDoc());
                assertEquals(i, sorted.nextDoc());
                assertEquals(value.longValue(), numeric.longValue());
                assertTrue(sorted.ordValue() >= 0);
                assertEquals(new BytesRef(Long.toString(value)), sorted.lookupOrd(sorted.ordValue()));
                assertEquals(new BytesRef(Long.toString(value)), binary.binaryValue());
            }
            final IndexableField[] valuesFields = doc.getFields("values");
            if (valuesFields.length == 0) {
                assertTrue(sortedNumeric.docID() + " vs " + i, sortedNumeric.docID() < i);
            } else {
                final Set<Long> valueSet = new HashSet<>();
                for (IndexableField sf : valuesFields) {
                    valueSet.add(sf.numericValue().longValue());
                }
                assertEquals(i, sortedNumeric.nextDoc());
                assertEquals(valuesFields.length, sortedNumeric.docValueCount());
                for (int j = 0; j < sortedNumeric.docValueCount(); ++j) {
                    assertTrue(valueSet.contains(sortedNumeric.nextValue()));
                }
                assertEquals(i, sortedSet.nextDoc());
                int sortedSetCount = 0;
                while (true) {
                    long ord = sortedSet.nextOrd();
                    if (ord == SortedSetDocValues.NO_MORE_ORDS) {
                        break;
                    }
                    assertTrue(valueSet.contains(Long.parseLong(sortedSet.lookupOrd(ord).utf8ToString())));
                    sortedSetCount++;
                }
                assertEquals(valueSet.size(), sortedSetCount);
            }
        }
    }
    indexReader.close();
    dir.close();
}

Example 20 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class SortedNumericDocValuesRangeQuery method createWeight.

@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
    return new ConstantScoreWeight(this, boost) {

        @Override
        public Scorer scorer(LeafReaderContext context) throws IOException {
            SortedNumericDocValues values = getValues(context.reader(), field);
            if (values == null) {
                return null;
            }
            final NumericDocValues singleton = DocValues.unwrapSingleton(values);
            final TwoPhaseIterator iterator;
            if (singleton != null) {
                iterator = new TwoPhaseIterator(singleton) {

                    @Override
                    public boolean matches() throws IOException {
                        final long value = singleton.longValue();
                        return value >= lowerValue && value <= upperValue;
                    }

                    @Override
                    public float matchCost() {
                        // 2 comparisons
                        return 2;
                    }
                };
            } else {
                iterator = new TwoPhaseIterator(values) {

                    @Override
                    public boolean matches() throws IOException {
                        for (int i = 0, count = values.docValueCount(); i < count; ++i) {
                            final long value = values.nextValue();
                            if (value < lowerValue) {
                                continue;
                            }
                            // Values are sorted, so the first value that is >= lowerValue is our best candidate
                            return value <= upperValue;
                        }
                        // all values were < lowerValue
                        return false;
                    }

                    @Override
                    public float matchCost() {
                        // 2 comparisons
                        return 2;
                    }
                };
            }
            return new ConstantScoreScorer(this, score(), iterator);
        }
    };
}