Examples with SortedNumericDocValues org.apache.lucene.index.SortedNumericDocValues used on opensource projects

Example 31 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project elasticsearch by elastic.

the class MultiValueModeTests method testSingleValuedLongs.

public void testSingleValuedLongs() throws Exception {
    final int numDocs = scaledRandomIntBetween(1, 100);
    final long[] array = new long[numDocs];
    final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
    for (int i = 0; i < array.length; ++i) {
        if (randomBoolean()) {
            array[i] = randomLong();
            if (docsWithValue != null) {
                docsWithValue.set(i);
            }
        } else if (docsWithValue != null && randomBoolean()) {
            docsWithValue.set(i);
        }
    }
    final NumericDocValues singleValues = new NumericDocValues() {

        @Override
        public long get(int docID) {
            return array[docID];
        }
    };
    final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue);
    verify(multiValues, numDocs);
    final FixedBitSet rootDocs = randomRootDocs(numDocs);
    final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
    verify(multiValues, numDocs, rootDocs, innerDocs);
}

Example 32 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project elasticsearch by elastic.

the class MultiValueModeTests method verify.

private void verify(SortedNumericDocValues values, int maxDoc) {
    for (long missingValue : new long[] { 0, randomLong() }) {
        for (MultiValueMode mode : MultiValueMode.values()) {
            final NumericDocValues selected = mode.select(values, missingValue);
            for (int i = 0; i < maxDoc; ++i) {
                final long actual = selected.get(i);
                long expected = 0;
                values.setDocument(i);
                int numValues = values.count();
                if (numValues == 0) {
                    expected = missingValue;
                } else {
                    if (mode == MultiValueMode.MAX) {
                        expected = Long.MIN_VALUE;
                    } else if (mode == MultiValueMode.MIN) {
                        expected = Long.MAX_VALUE;
                    }
                    for (int j = 0; j < numValues; ++j) {
                        if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) {
                            expected += values.valueAt(j);
                        } else if (mode == MultiValueMode.MIN) {
                            expected = Math.min(expected, values.valueAt(j));
                        } else if (mode == MultiValueMode.MAX) {
                            expected = Math.max(expected, values.valueAt(j));
                        }
                    }
                    if (mode == MultiValueMode.AVG) {
                        expected = numValues > 1 ? Math.round((double) expected / (double) numValues) : expected;
                    } else if (mode == MultiValueMode.MEDIAN) {
                        int value = numValues / 2;
                        if (numValues % 2 == 0) {
                            expected = Math.round((values.valueAt(value - 1) + values.valueAt(value)) / 2.0);
                        } else {
                            expected = values.valueAt(value);
                        }
                    }
                }
                assertEquals(mode.toString() + " docId=" + i, expected, actual);
            }
        }
    }
}

Example 33 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class LegacyDocValuesIterables method sortedNumericToDocCount.

/** Converts number-of-values per document from {@link SortedNumericDocValues} into {@code Iterable<Number>}.
   *
   * @deprecated Consume {@link SortedDocValues} instead. */
@Deprecated
public static Iterable<Number> sortedNumericToDocCount(final DocValuesProducer valuesProducer, final FieldInfo fieldInfo, int maxDoc) {
    return new Iterable<Number>() {

        @Override
        public Iterator<Number> iterator() {
            final SortedNumericDocValues values;
            try {
                values = valuesProducer.getSortedNumeric(fieldInfo);
            } catch (IOException ioe) {
                throw new RuntimeException(ioe);
            }
            return new Iterator<Number>() {

                private int nextDocID;

                @Override
                public boolean hasNext() {
                    return nextDocID < maxDoc;
                }

                @Override
                public Number next() {
                    try {
                        if (nextDocID > values.docID()) {
                            values.nextDoc();
                        }
                        int result;
                        if (nextDocID == values.docID()) {
                            result = values.docValueCount();
                        } else {
                            result = 0;
                        }
                        nextDocID++;
                        return result;
                    } catch (IOException ioe) {
                        throw new RuntimeException(ioe);
                    }
                }
            };
        }
    };
}

Example 34 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class TestLucene70DocValuesFormat method testSortedNumericAroundBlockSize.

@Slow
public void testSortedNumericAroundBlockSize() throws IOException {
    final int frontier = 1 << Lucene70DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
    for (int maxDoc = frontier - 1; maxDoc <= frontier + 1; ++maxDoc) {
        final Directory dir = newDirectory();
        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
        RAMFile buffer = new RAMFile();
        RAMOutputStream out = new RAMOutputStream(buffer, false);
        Document doc = new Document();
        SortedNumericDocValuesField field1 = new SortedNumericDocValuesField("snum", 0L);
        doc.add(field1);
        SortedNumericDocValuesField field2 = new SortedNumericDocValuesField("snum", 0L);
        doc.add(field2);
        for (int i = 0; i < maxDoc; ++i) {
            long s1 = random().nextInt(100);
            long s2 = random().nextInt(100);
            field1.setLongValue(s1);
            field2.setLongValue(s2);
            w.addDocument(doc);
            out.writeVLong(Math.min(s1, s2));
            out.writeVLong(Math.max(s1, s2));
        }
        out.close();
        w.forceMerge(1);
        DirectoryReader r = DirectoryReader.open(w);
        w.close();
        LeafReader sr = getOnlyLeafReader(r);
        assertEquals(maxDoc, sr.maxDoc());
        SortedNumericDocValues values = sr.getSortedNumericDocValues("snum");
        assertNotNull(values);
        RAMInputStream in = new RAMInputStream("", buffer);
        for (int i = 0; i < maxDoc; ++i) {
            assertEquals(i, values.nextDoc());
            assertEquals(2, values.docValueCount());
            assertEquals(in.readVLong(), values.nextValue());
            assertEquals(in.readVLong(), values.nextValue());
        }
        r.close();
        dir.close();
    }
}

Example 35 with SortedNumericDocValues

use of org.apache.lucene.index.SortedNumericDocValues in project lucene-solr by apache.

the class TestLucene70DocValuesFormat method doTestSortedNumericBlocksOfVariousBitsPerValue.

private void doTestSortedNumericBlocksOfVariousBitsPerValue(LongSupplier counts) throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    conf.setMaxBufferedDocs(atLeast(Lucene70DocValuesFormat.NUMERIC_BLOCK_SIZE));
    conf.setRAMBufferSizeMB(-1);
    conf.setMergePolicy(newLogMergePolicy(random().nextBoolean()));
    IndexWriter writer = new IndexWriter(dir, conf);
    final int numDocs = atLeast(Lucene70DocValuesFormat.NUMERIC_BLOCK_SIZE * 3);
    final LongSupplier values = blocksOfVariousBPV();
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        int valueCount = (int) counts.getAsLong();
        long[] valueArray = new long[valueCount];
        for (int j = 0; j < valueCount; j++) {
            long value = values.getAsLong();
            valueArray[j] = value;
            doc.add(new SortedNumericDocValuesField("dv", value));
        }
        Arrays.sort(valueArray);
        for (int j = 0; j < valueCount; j++) {
            doc.add(new StoredField("stored", Long.toString(valueArray[j])));
        }
        writer.addDocument(doc);
        if (random().nextInt(31) == 0) {
            writer.commit();
        }
    }
    writer.forceMerge(1);
    writer.close();
    // compare
    DirectoryReader ir = DirectoryReader.open(dir);
    TestUtil.checkReader(ir);
    for (LeafReaderContext context : ir.leaves()) {
        LeafReader r = context.reader();
        SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");
        for (int i = 0; i < r.maxDoc(); i++) {
            if (i > docValues.docID()) {
                docValues.nextDoc();
            }
            String[] expected = r.document(i).getValues("stored");
            if (i < docValues.docID()) {
                assertEquals(0, expected.length);
            } else {
                String[] actual = new String[docValues.docValueCount()];
                for (int j = 0; j < actual.length; j++) {
                    actual[j] = Long.toString(docValues.nextValue());
                }
                assertArrayEquals(expected, actual);
            }
        }
    }
    ir.close();
    dir.close();
}