Examples with DirectMonotonicWriter org.apache.lucene.util.packed.DirectMonotonicWriter used on opensource projects

Example 6 with DirectMonotonicWriter

use of org.apache.lucene.util.packed.DirectMonotonicWriter in project lucene-solr by apache.

the class Lucene70DocValuesConsumer method addBinaryField.

@Override
public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
    meta.writeInt(field.number);
    meta.writeByte(Lucene70DocValuesFormat.BINARY);
    BinaryDocValues values = valuesProducer.getBinary(field);
    long start = data.getFilePointer();
    meta.writeLong(start);
    int numDocsWithField = 0;
    int minLength = Integer.MAX_VALUE;
    int maxLength = 0;
    for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
        numDocsWithField++;
        BytesRef v = values.binaryValue();
        int length = v.length;
        data.writeBytes(v.bytes, v.offset, v.length);
        minLength = Math.min(length, minLength);
        maxLength = Math.max(length, maxLength);
    }
    assert numDocsWithField <= maxDoc;
    meta.writeLong(data.getFilePointer() - start);
    if (numDocsWithField == 0) {
        meta.writeLong(-2);
        meta.writeLong(0L);
    } else if (numDocsWithField == maxDoc) {
        meta.writeLong(-1);
        meta.writeLong(0L);
    } else {
        long offset = data.getFilePointer();
        meta.writeLong(offset);
        values = valuesProducer.getBinary(field);
        IndexedDISI.writeBitSet(values, data);
        meta.writeLong(data.getFilePointer() - offset);
    }
    meta.writeInt(numDocsWithField);
    meta.writeInt(minLength);
    meta.writeInt(maxLength);
    if (maxLength > minLength) {
        start = data.getFilePointer();
        meta.writeLong(start);
        meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
        final DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1, DIRECT_MONOTONIC_BLOCK_SHIFT);
        long addr = 0;
        writer.add(addr);
        values = valuesProducer.getBinary(field);
        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
            addr += values.binaryValue().length;
            writer.add(addr);
        }
        writer.finish();
        meta.writeLong(data.getFilePointer() - start);
    }
}

Example 7 with DirectMonotonicWriter

use of org.apache.lucene.util.packed.DirectMonotonicWriter in project lucene-solr by apache.

the class Lucene70DocValuesConsumer method addSortedNumericField.

@Override
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
    meta.writeInt(field.number);
    meta.writeByte(Lucene70DocValuesFormat.SORTED_NUMERIC);
    long[] stats = writeValues(field, valuesProducer);
    int numDocsWithField = Math.toIntExact(stats[0]);
    long numValues = stats[1];
    assert numValues >= numDocsWithField;
    meta.writeInt(numDocsWithField);
    if (numValues > numDocsWithField) {
        long start = data.getFilePointer();
        meta.writeLong(start);
        meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
        final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT);
        long addr = 0;
        addressesWriter.add(addr);
        SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
            addr += values.docValueCount();
            addressesWriter.add(addr);
        }
        addressesWriter.finish();
        meta.writeLong(data.getFilePointer() - start);
    }
}

Example 8 with DirectMonotonicWriter

use of org.apache.lucene.util.packed.DirectMonotonicWriter in project lucene-solr by apache.

the class Lucene70DocValuesConsumer method writeTermsIndex.

private void writeTermsIndex(SortedSetDocValues values) throws IOException {
    final long size = values.getValueCount();
    meta.writeInt(Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
    long start = data.getFilePointer();
    long numBlocks = 1L + ((size + Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) >>> Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
    RAMOutputStream addressBuffer = new RAMOutputStream();
    DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, addressBuffer, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT);
    TermsEnum iterator = values.termsEnum();
    BytesRefBuilder previous = new BytesRefBuilder();
    long offset = 0;
    long ord = 0;
    for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
        if ((ord & Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) {
            writer.add(offset);
            int sortKeyLength = StringHelper.sortKeyLength(previous.get(), term);
            offset += sortKeyLength;
            data.writeBytes(term.bytes, term.offset, sortKeyLength);
        } else if ((ord & Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) {
            previous.copyBytes(term);
        }
        ++ord;
    }
    writer.add(offset);
    writer.finish();
    meta.writeLong(start);
    meta.writeLong(data.getFilePointer() - start);
    start = data.getFilePointer();
    addressBuffer.writeTo(data);
    meta.writeLong(start);
    meta.writeLong(data.getFilePointer() - start);
}