Examples with BytesRefIterator org.apache.lucene.util.BytesRefIterator used on opensource projects

Example 6 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project elasticsearch by elastic.

the class CompositeBytesReference method iterator.

@Override
public BytesRefIterator iterator() {
    if (references.length > 0) {
        return new BytesRefIterator() {

            int index = 0;

            private BytesRefIterator current = references[index++].iterator();

            @Override
            public BytesRef next() throws IOException {
                BytesRef next = current.next();
                if (next == null) {
                    while (index < references.length) {
                        current = references[index++].iterator();
                        next = current.next();
                        if (next != null) {
                            break;
                        }
                    }
                }
                return next;
            }
        };
    } else {
        return () -> null;
    }
}

Example 7 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project elasticsearch by elastic.

the class PagedBytesReference method iterator.

@Override
public final BytesRefIterator iterator() {
    final int offset = this.offset;
    final int length = this.length;
    // this iteration is page aligned to ensure we do NOT materialize the pages from the ByteArray
    // we calculate the initial fragment size here to ensure that if this reference is a slice we are still page aligned
    // across the entire iteration. The first page is smaller if our offset != 0 then we start in the middle of the page
    // otherwise we iterate full pages until we reach the last chunk which also might end within a page.
    final int initialFragmentSize = offset != 0 ? PAGE_SIZE - (offset % PAGE_SIZE) : PAGE_SIZE;
    return new BytesRefIterator() {

        int position = 0;

        int nextFragmentSize = Math.min(length, initialFragmentSize);

        // this BytesRef is reused across the iteration on purpose - BytesRefIterator interface was designed for this
        final BytesRef slice = new BytesRef();

        @Override
        public BytesRef next() throws IOException {
            if (nextFragmentSize != 0) {
                final boolean materialized = byteArray.get(offset + position, nextFragmentSize, slice);
                assert materialized == false : "iteration should be page aligned but array got materialized";
                position += nextFragmentSize;
                final int remaining = length - position;
                nextFragmentSize = Math.min(remaining, PAGE_SIZE);
                return slice;
            } else {
                assert nextFragmentSize == 0 : "fragmentSize expected [0] but was: [" + nextFragmentSize + "]";
                // we are done with this iteration
                return null;
            }
        }
    };
}

Example 8 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project elasticsearch by elastic.

the class CompositeBytesReferenceTests method testCompositeBuffer.

public void testCompositeBuffer() throws IOException {
    List<BytesReference> referenceList = newRefList(randomIntBetween(1, PAGE_SIZE * 2));
    BytesReference ref = new CompositeBytesReference(referenceList.toArray(new BytesReference[0]));
    BytesRefIterator iterator = ref.iterator();
    BytesRefBuilder builder = new BytesRefBuilder();
    for (BytesReference reference : referenceList) {
        // sometimes we have a paged ref - pull an iter and walk all pages!
        BytesRefIterator innerIter = reference.iterator();
        BytesRef scratch;
        while ((scratch = innerIter.next()) != null) {
            BytesRef next = iterator.next();
            assertNotNull(next);
            assertEquals(next, scratch);
            builder.append(next);
        }
    }
    assertNull(iterator.next());
    int offset = 0;
    for (BytesReference reference : referenceList) {
        assertEquals(reference, ref.slice(offset, reference.length()));
        int probes = randomIntBetween(Math.min(10, reference.length()), reference.length());
        for (int i = 0; i < probes; i++) {
            int index = randomIntBetween(0, reference.length() - 1);
            assertEquals(ref.get(offset + index), reference.get(index));
        }
        offset += reference.length();
    }
    BytesArray array = new BytesArray(builder.toBytesRef());
    assertEquals(array, ref);
    assertEquals(array.hashCode(), ref.hashCode());
    BytesStreamOutput output = new BytesStreamOutput();
    ref.writeTo(output);
    assertEquals(array, output.bytes());
}

Example 9 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project lucene-solr by apache.

the class BytesRefSortersTest method check.

private void check(BytesRefSorter sorter) throws Exception {
    for (int i = 0; i < 100; i++) {
        byte[] current = new byte[random().nextInt(256)];
        random().nextBytes(current);
        sorter.add(new BytesRef(current));
    }
    // Create two iterators and check that they're aligned with each other.
    BytesRefIterator i1 = sorter.iterator();
    BytesRefIterator i2 = sorter.iterator();
    // Verify sorter contract.
    expectThrows(IllegalStateException.class, () -> {
        sorter.add(new BytesRef(new byte[1]));
    });
    while (true) {
        BytesRef spare1 = i1.next();
        BytesRef spare2 = i2.next();
        assertEquals(spare1, spare2);
        if (spare1 == null) {
            break;
        }
    }
}

Example 10 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project lucene-solr by apache.

the class TestBytesRefArray method testSort.

public void testSort() throws IOException {
    Random random = random();
    BytesRefArray list = new BytesRefArray(Counter.newCounter());
    List<String> stringList = new ArrayList<>();
    for (int j = 0; j < 2; j++) {
        if (j > 0 && random.nextBoolean()) {
            list.clear();
            stringList.clear();
        }
        int entries = atLeast(500);
        BytesRefBuilder spare = new BytesRefBuilder();
        final int initSize = list.size();
        for (int i = 0; i < entries; i++) {
            String randomRealisticUnicodeString = TestUtil.randomRealisticUnicodeString(random);
            spare.copyChars(randomRealisticUnicodeString);
            assertEquals(initSize + i, list.append(spare.get()));
            stringList.add(randomRealisticUnicodeString);
        }
        Collections.sort(stringList, TestUtil.STRING_CODEPOINT_COMPARATOR);
        BytesRefIterator iter = list.iterator(Comparator.naturalOrder());
        int i = 0;
        BytesRef next;
        while ((next = iter.next()) != null) {
            assertEquals("entry " + i + " doesn't match", stringList.get(i), next.utf8ToString());
            i++;
        }
        assertNull(iter.next());
        assertEquals(i, stringList.size());
    }
}