Examples with PackedLongValues org.apache.lucene.util.packed.PackedLongValues used on opensource projects

Example 1 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class TestSegmentMerger method testBuildDocMap.

public void testBuildDocMap() {
    final int maxDoc = TestUtil.nextInt(random(), 1, 128);
    final int numDocs = TestUtil.nextInt(random(), 0, maxDoc);
    final FixedBitSet liveDocs = new FixedBitSet(maxDoc);
    for (int i = 0; i < numDocs; ++i) {
        while (true) {
            final int docID = random().nextInt(maxDoc);
            if (!liveDocs.get(docID)) {
                liveDocs.set(docID);
                break;
            }
        }
    }
    final PackedLongValues docMap = MergeState.removeDeletes(maxDoc, liveDocs);
    // assert the mapping is compact
    for (int i = 0, del = 0; i < maxDoc; ++i) {
        if (liveDocs.get(i) == false) {
            ++del;
        } else {
            assertEquals(i - del, docMap.get(i));
        }
    }
}

Example 2 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class NumericDocValuesWriter method flush.

@Override
public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException {
    final PackedLongValues values;
    if (finalValues == null) {
        values = pending.build();
    } else {
        values = finalValues;
    }
    final SortingLeafReader.CachedNumericDVs sorted;
    if (sortMap != null) {
        NumericDocValues oldValues = new BufferedNumericDocValues(values, docsWithField.iterator());
        sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, oldValues);
    } else {
        sorted = null;
    }
    dvConsumer.addNumericField(fieldInfo, new EmptyDocValuesProducer() {

        @Override
        public NumericDocValues getNumeric(FieldInfo fieldInfo) {
            if (fieldInfo != NumericDocValuesWriter.this.fieldInfo) {
                throw new IllegalArgumentException("wrong fieldInfo");
            }
            if (sorted == null) {
                return new BufferedNumericDocValues(values, docsWithField.iterator());
            } else {
                return new SortingLeafReader.SortingNumericDocValues(sorted);
            }
        }
    });
}

Example 3 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class SortedNumericDocValuesWriter method flush.

@Override
public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException {
    final PackedLongValues values;
    final PackedLongValues valueCounts;
    if (finalValues == null) {
        values = pending.build();
        valueCounts = pendingCounts.build();
    } else {
        values = finalValues;
        valueCounts = finalValuesCount;
    }
    final long[][] sorted;
    if (sortMap != null) {
        sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, new BufferedSortedNumericDocValues(values, valueCounts, docsWithField.iterator()));
    } else {
        sorted = null;
    }
    dvConsumer.addSortedNumericField(fieldInfo, new EmptyDocValuesProducer() {

        @Override
        public SortedNumericDocValues getSortedNumeric(FieldInfo fieldInfoIn) {
            if (fieldInfoIn != fieldInfo) {
                throw new IllegalArgumentException("wrong fieldInfo");
            }
            final SortedNumericDocValues buf = new BufferedSortedNumericDocValues(values, valueCounts, docsWithField.iterator());
            if (sorted == null) {
                return buf;
            } else {
                return new SortingLeafReader.SortingSortedNumericDocValues(buf, sorted);
            }
        }
    });
}

Example 4 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class BinaryDocValuesWriter method flush.

@Override
public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException {
    bytes.freeze(false);
    final PackedLongValues lengths = this.lengths.build();
    final SortingLeafReader.CachedBinaryDVs sorted;
    if (sortMap != null) {
        sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, new BufferedBinaryDocValues(lengths, maxLength, bytes.getDataInput(), docsWithField.iterator()));
    } else {
        sorted = null;
    }
    dvConsumer.addBinaryField(fieldInfo, new EmptyDocValuesProducer() {

        @Override
        public BinaryDocValues getBinary(FieldInfo fieldInfoIn) {
            if (fieldInfoIn != fieldInfo) {
                throw new IllegalArgumentException("wrong fieldInfo");
            }
            if (sorted == null) {
                return new BufferedBinaryDocValues(lengths, maxLength, bytes.getDataInput(), docsWithField.iterator());
            } else {
                return new SortingLeafReader.SortingBinaryDocValues(sorted);
            }
        }
    });
}

Example 5 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class Sorter method sort.

/** Computes the old-to-new permutation over the given comparator. */
private static Sorter.DocMap sort(final int maxDoc, DocComparator comparator) {
    // check if the index is sorted
    boolean sorted = true;
    for (int i = 1; i < maxDoc; ++i) {
        if (comparator.compare(i - 1, i) > 0) {
            sorted = false;
            break;
        }
    }
    if (sorted) {
        return null;
    }
    // sort doc IDs
    final int[] docs = new int[maxDoc];
    for (int i = 0; i < maxDoc; i++) {
        docs[i] = i;
    }
    DocValueSorter sorter = new DocValueSorter(docs, comparator);
    // It can be common to sort a reader, add docs, sort it again, ... and in
    // that case timSort can save a lot of time
    // docs is now the newToOld mapping
    sorter.sort(0, docs.length);
    // The reason why we use MonotonicAppendingLongBuffer here is that it
    // wastes very little memory if the index is in random order but can save
    // a lot of memory if the index is already "almost" sorted
    final PackedLongValues.Builder newToOldBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
    for (int i = 0; i < maxDoc; ++i) {
        newToOldBuilder.add(docs[i]);
    }
    final PackedLongValues newToOld = newToOldBuilder.build();
    // invert the docs mapping:
    for (int i = 0; i < maxDoc; ++i) {
        docs[(int) newToOld.get(i)] = i;
    }
    // docs is now the oldToNew mapping
    final PackedLongValues.Builder oldToNewBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
    for (int i = 0; i < maxDoc; ++i) {
        oldToNewBuilder.add(docs[i]);
    }
    final PackedLongValues oldToNew = oldToNewBuilder.build();
    return new Sorter.DocMap() {

        @Override
        public int oldToNew(int docID) {
            return (int) oldToNew.get(docID);
        }

        @Override
        public int newToOld(int docID) {
            return (int) newToOld.get(docID);
        }

        @Override
        public int size() {
            return maxDoc;
        }
    };
}