Examples with PackedLongValues org.apache.lucene.util.packed.PackedLongValues used on opensource projects

Example 6 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class MultiSorter method sort.

/** Does a merge sort of the leaves of the incoming reader, returning {@link DocMap} to map each leaf's
   *  documents into the merged segment.  The documents for each incoming leaf reader must already be sorted by the same sort!
   *  Returns null if the merge sort is not needed (segments are already in index sort order).
   **/
static MergeState.DocMap[] sort(Sort sort, List<CodecReader> readers) throws IOException {
    // TODO: optimize if only 1 reader is incoming, though that's a rare case
    SortField[] fields = sort.getSort();
    final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
    for (int i = 0; i < fields.length; i++) {
        comparables[i] = getComparableProviders(readers, fields[i]);
    }
    int leafCount = readers.size();
    PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {

        @Override
        public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
            for (int i = 0; i < comparables.length; i++) {
                int cmp = a.values[i].compareTo(b.values[i]);
                if (cmp != 0) {
                    return cmp < 0;
                }
            }
            // tie-break by docID natural order:
            if (a.readerIndex != b.readerIndex) {
                return a.readerIndex < b.readerIndex;
            } else {
                return a.docID < b.docID;
            }
        }
    };
    PackedLongValues.Builder[] builders = new PackedLongValues.Builder[leafCount];
    for (int i = 0; i < leafCount; i++) {
        CodecReader reader = readers.get(i);
        LeafAndDocID leaf = new LeafAndDocID(i, reader.getLiveDocs(), reader.maxDoc(), comparables.length);
        for (int j = 0; j < comparables.length; j++) {
            leaf.values[j] = comparables[j][i].getComparable(leaf.docID);
            assert leaf.values[j] != null;
        }
        queue.add(leaf);
        builders[i] = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
    }
    // merge sort:
    int mappedDocID = 0;
    int lastReaderIndex = 0;
    boolean isSorted = true;
    while (queue.size() != 0) {
        LeafAndDocID top = queue.top();
        if (lastReaderIndex > top.readerIndex) {
            // merge sort is needed
            isSorted = false;
        }
        lastReaderIndex = top.readerIndex;
        builders[top.readerIndex].add(mappedDocID);
        if (top.liveDocs == null || top.liveDocs.get(top.docID)) {
            mappedDocID++;
        }
        top.docID++;
        if (top.docID < top.maxDoc) {
            for (int j = 0; j < comparables.length; j++) {
                top.values[j] = comparables[j][top.readerIndex].getComparable(top.docID);
                assert top.values[j] != null;
            }
            queue.updateTop();
        } else {
            queue.pop();
        }
    }
    if (isSorted) {
        return null;
    }
    MergeState.DocMap[] docMaps = new MergeState.DocMap[leafCount];
    for (int i = 0; i < leafCount; i++) {
        final PackedLongValues remapped = builders[i].build();
        final Bits liveDocs = readers.get(i).getLiveDocs();
        docMaps[i] = new MergeState.DocMap() {

            @Override
            public int get(int docID) {
                if (liveDocs == null || liveDocs.get(docID)) {
                    return (int) remapped.get(docID);
                } else {
                    return -1;
                }
            }
        };
    }
    return docMaps;
}

Example 7 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class NormValuesWriter method flush.

public void flush(SegmentWriteState state, Sorter.DocMap sortMap, NormsConsumer normsConsumer) throws IOException {
    final PackedLongValues values = pending.build();
    final SortingLeafReader.CachedNumericDVs sorted;
    if (sortMap != null) {
        sorted = NumericDocValuesWriter.sortDocValues(state.segmentInfo.maxDoc(), sortMap, new BufferedNorms(values, docsWithField.iterator()));
    } else {
        sorted = null;
    }
    normsConsumer.addNormsField(fieldInfo, new NormsProducer() {

        @Override
        public NumericDocValues getNorms(FieldInfo fieldInfo2) {
            if (fieldInfo != NormValuesWriter.this.fieldInfo) {
                throw new IllegalArgumentException("wrong fieldInfo");
            }
            if (sorted == null) {
                return new BufferedNorms(values, docsWithField.iterator());
            } else {
                return new SortingLeafReader.SortingNumericDocValues(sorted);
            }
        }

        @Override
        public void checkIntegrity() {
        }

        @Override
        public void close() {
        }

        @Override
        public long ramBytesUsed() {
            return 0;
        }
    });
}

Example 8 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class SortedSetDocValuesWriter method flush.

@Override
public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException {
    final int valueCount = hash.size();
    final PackedLongValues ords;
    final PackedLongValues ordCounts;
    final int[] sortedValues;
    final int[] ordMap;
    if (finalOrdCounts == null) {
        ords = pending.build();
        ordCounts = pendingCounts.build();
        sortedValues = hash.sort();
        ordMap = new int[valueCount];
        for (int ord = 0; ord < valueCount; ord++) {
            ordMap[sortedValues[ord]] = ord;
        }
    } else {
        ords = finalOrds;
        ordCounts = finalOrdCounts;
        sortedValues = finalSortedValues;
        ordMap = finalOrdMap;
    }
    final long[][] sorted;
    if (sortMap != null) {
        sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, new BufferedSortedSetDocValues(sortedValues, ordMap, hash, ords, ordCounts, maxCount, docsWithField.iterator()));
    } else {
        sorted = null;
    }
    dvConsumer.addSortedSetField(fieldInfo, new EmptyDocValuesProducer() {

        @Override
        public SortedSetDocValues getSortedSet(FieldInfo fieldInfoIn) {
            if (fieldInfoIn != fieldInfo) {
                throw new IllegalArgumentException("wrong fieldInfo");
            }
            final SortedSetDocValues buf = new BufferedSortedSetDocValues(sortedValues, ordMap, hash, ords, ordCounts, maxCount, docsWithField.iterator());
            if (sorted == null) {
                return buf;
            } else {
                return new SortingLeafReader.SortingSortedSetDocValues(buf, sorted);
            }
        }
    });
}

Example 9 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class MergeState method buildDeletionDocMaps.

// Remap docIDs around deletions
private DocMap[] buildDeletionDocMaps(List<CodecReader> readers) {
    int totalDocs = 0;
    int numReaders = readers.size();
    DocMap[] docMaps = new DocMap[numReaders];
    for (int i = 0; i < numReaders; i++) {
        LeafReader reader = readers.get(i);
        Bits liveDocs = reader.getLiveDocs();
        final PackedLongValues delDocMap;
        if (liveDocs != null) {
            delDocMap = removeDeletes(reader.maxDoc(), liveDocs);
        } else {
            delDocMap = null;
        }
        final int docBase = totalDocs;
        docMaps[i] = new DocMap() {

            @Override
            public int get(int docID) {
                if (liveDocs == null) {
                    return docBase + docID;
                } else if (liveDocs.get(docID)) {
                    return docBase + (int) delDocMap.get(docID);
                } else {
                    return -1;
                }
            }
        };
        totalDocs += reader.numDocs();
    }
    return docMaps;
}

Example 10 with PackedLongValues

use of org.apache.lucene.util.packed.PackedLongValues in project lucene-solr by apache.

the class BinaryDocValuesWriter method flush.

@Override
public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException {
    bytes.freeze(false);
    final PackedLongValues lengths = this.lengths.build();
    final SortingLeafReader.CachedBinaryDVs sorted;
    if (sortMap != null) {
        sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, new BufferedBinaryDocValues(lengths, maxLength, bytes.getDataInput(), docsWithField.iterator()));
    } else {
        sorted = null;
    }
    dvConsumer.addBinaryField(fieldInfo, new EmptyDocValuesProducer() {

        @Override
        public BinaryDocValues getBinary(FieldInfo fieldInfoIn) {
            if (fieldInfoIn != fieldInfo) {
                throw new IllegalArgumentException("wrong fieldInfo");
            }
            if (sorted == null) {
                return new BufferedBinaryDocValues(lengths, maxLength, bytes.getDataInput(), docsWithField.iterator());
            } else {
                return new SortingLeafReader.SortingBinaryDocValues(sorted);
            }
        }
    });
}