Examples with BytesRefIterator org.apache.lucene.util.BytesRefIterator used on opensource projects

Example 16 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class PagedBytesReference method iterator.

@Override
public final BytesRefIterator iterator() {
    final int offset = this.offset;
    final int length = this.length;
    // this iteration is page aligned to ensure we do NOT materialize the pages from the ByteArray
    // we calculate the initial fragment size here to ensure that if this reference is a slice we are still page aligned
    // across the entire iteration. The first page is smaller if our offset != 0 then we start in the middle of the page
    // otherwise we iterate full pages until we reach the last chunk which also might end within a page.
    final int initialFragmentSize = offset != 0 ? PAGE_SIZE - (offset % PAGE_SIZE) : PAGE_SIZE;
    return new BytesRefIterator() {

        int position = 0;

        int nextFragmentSize = Math.min(length, initialFragmentSize);

        // this BytesRef is reused across the iteration on purpose - BytesRefIterator interface was designed for this
        final BytesRef slice = new BytesRef();

        @Override
        public BytesRef next() throws IOException {
            if (nextFragmentSize != 0) {
                final boolean materialized = byteArray.get(offset + position, nextFragmentSize, slice);
                assert materialized == false : "iteration should be page aligned but array got materialized";
                position += nextFragmentSize;
                final int remaining = length - position;
                nextFragmentSize = Math.min(remaining, PAGE_SIZE);
                return slice;
            } else {
                assert nextFragmentSize == 0 : "fragmentSize expected [0] but was: [" + nextFragmentSize + "]";
                // we are done with this iteration
                return null;
            }
        }
    };
}

Example 17 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class TransportDecompressor method decompress.

public int decompress(BytesReference bytesReference) throws IOException {
    int bytesConsumed = 0;
    if (hasReadHeader == false) {
        if (CompressorFactory.COMPRESSOR.isCompressed(bytesReference) == false) {
            int maxToRead = Math.min(bytesReference.length(), 10);
            StringBuilder sb = new StringBuilder("stream marked as compressed, but no compressor found, first [").append(maxToRead).append("] content bytes out of [").append(bytesReference.length()).append("] readable bytes with message size [").append(bytesReference.length()).append("] ").append("] are [");
            for (int i = 0; i < maxToRead; i++) {
                sb.append(bytesReference.get(i)).append(",");
            }
            sb.append("]");
            throw new IllegalStateException(sb.toString());
        }
        hasReadHeader = true;
        int headerLength = CompressorFactory.COMPRESSOR.headerLength();
        bytesReference = bytesReference.slice(headerLength, bytesReference.length() - headerLength);
        bytesConsumed += headerLength;
    }
    BytesRefIterator refIterator = bytesReference.iterator();
    BytesRef ref;
    while ((ref = refIterator.next()) != null) {
        inflater.setInput(ref.bytes, ref.offset, ref.length);
        bytesConsumed += ref.length;
        boolean continueInflating = true;
        while (continueInflating) {
            final Recycler.V<byte[]> page;
            final boolean isNewPage = pageOffset == PageCacheRecycler.BYTE_PAGE_SIZE;
            if (isNewPage) {
                pageOffset = 0;
                page = recycler.bytePage(false);
            } else {
                page = pages.getLast();
            }
            byte[] output = page.v();
            try {
                int bytesInflated = inflater.inflate(output, pageOffset, PageCacheRecycler.BYTE_PAGE_SIZE - pageOffset);
                pageOffset += bytesInflated;
                if (isNewPage) {
                    if (bytesInflated == 0) {
                        page.close();
                        pageOffset = PageCacheRecycler.BYTE_PAGE_SIZE;
                    } else {
                        pages.add(page);
                    }
                }
            } catch (DataFormatException e) {
                throw new IOException("Exception while inflating bytes", e);
            }
            if (inflater.needsInput()) {
                continueInflating = false;
            }
            if (inflater.finished()) {
                bytesConsumed -= inflater.getRemaining();
                continueInflating = false;
            }
            assert inflater.needsDictionary() == false;
        }
    }
    return bytesConsumed;
}

Example 18 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class Netty4Utils method toByteBuf.

/**
 * Turns the given BytesReference into a ByteBuf. Note: the returned ByteBuf will reference the internal
 * pages of the BytesReference. Don't free the bytes of reference before the ByteBuf goes out of scope.
 */
public static ByteBuf toByteBuf(final BytesReference reference) {
    if (reference.length() == 0) {
        return Unpooled.EMPTY_BUFFER;
    }
    if (reference instanceof ByteBufBytesReference) {
        return ((ByteBufBytesReference) reference).toByteBuf();
    } else {
        final BytesRefIterator iterator = reference.iterator();
        // usually we have one, two, or three components from the header, the message, and a buffer
        final List<ByteBuf> buffers = new ArrayList<>(3);
        try {
            BytesRef slice;
            while ((slice = iterator.next()) != null) {
                buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length));
            }
            final CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size());
            composite.addComponents(true, buffers);
            return composite;
        } catch (IOException ex) {
            throw new AssertionError("no IO happens here", ex);
        }
    }
}

Example 19 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class AbstractBytesReferenceTestCase method testIteratorRandom.

public void testIteratorRandom() throws IOException {
    int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
    BytesReference pbr = newBytesReference(length);
    if (randomBoolean()) {
        int sliceOffset = randomIntBetween(0, pbr.length());
        int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
        pbr = pbr.slice(sliceOffset, sliceLength);
    }
    if (randomBoolean()) {
        pbr = new BytesArray(pbr.toBytesRef());
    }
    BytesRefIterator iterator = pbr.iterator();
    BytesRef ref = null;
    BytesRefBuilder builder = new BytesRefBuilder();
    while ((ref = iterator.next()) != null) {
        builder.append(ref);
    }
    assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
}

Example 20 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class AbstractBytesReferenceTestCase method testIterator.

public void testIterator() throws IOException {
    int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
    BytesReference pbr = newBytesReference(length);
    BytesRefIterator iterator = pbr.iterator();
    BytesRef ref;
    BytesRefBuilder builder = new BytesRefBuilder();
    while ((ref = iterator.next()) != null) {
        builder.append(ref);
    }
    assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
}