Examples with BytesRefIterator org.apache.lucene.util.BytesRefIterator used on opensource projects

Example 11 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project lucene-solr by apache.

the class FSTCompletionBuilder method buildAutomaton.

/**
   * Builds the final automaton from a list of entries.
   */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
    // Build the automaton.
    final Outputs<Object> outputs = NoOutputs.getSingleton();
    final Object empty = outputs.getNoOutput();
    final Builder<Object> builder = new Builder<>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, shareMaxTailLength, outputs, true, 15);
    BytesRefBuilder scratch = new BytesRefBuilder();
    BytesRef entry;
    final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
    int count = 0;
    BytesRefIterator iter = sorter.iterator();
    while ((entry = iter.next()) != null) {
        count++;
        if (scratch.get().compareTo(entry) != 0) {
            builder.add(Util.toIntsRef(entry, scratchIntsRef), empty);
            scratch.copyBytes(entry);
        }
    }
    return count == 0 ? null : builder.finish();
}

Example 12 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project lucene-solr by apache.

the class TestHighFrequencyDictionary method testEmpty.

public void testEmpty() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    writer.commit();
    writer.close();
    IndexReader ir = DirectoryReader.open(dir);
    Dictionary dictionary = new HighFrequencyDictionary(ir, "bogus", 0.1f);
    BytesRefIterator tf = dictionary.getEntryIterator();
    assertNull(tf.next());
    dir.close();
}

Example 13 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class AbstractBytesReference method iterator.

@Override
public BytesRefIterator iterator() {
    return new BytesRefIterator() {

        BytesRef ref = length() == 0 ? null : toBytesRef();

        @Override
        public BytesRef next() throws IOException {
            BytesRef r = ref;
            // only return it once...
            ref = null;
            return r;
        }
    };
}

Example 14 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class AbstractBytesReference method compareIterators.

/**
 * Compares the two references using the given int function.
 */
private static int compareIterators(final BytesReference a, final BytesReference b, final ToIntBiFunction<BytesRef, BytesRef> f) {
    try {
        // we use the iterators since it's a 0-copy comparison where possible!
        final long lengthToCompare = Math.min(a.length(), b.length());
        final BytesRefIterator aIter = a.iterator();
        final BytesRefIterator bIter = b.iterator();
        BytesRef aRef = aIter.next();
        BytesRef bRef = bIter.next();
        if (aRef != null && bRef != null) {
            // do we have any data?
            // we clone since we modify the offsets and length in the iteration below
            aRef = aRef.clone();
            bRef = bRef.clone();
            if (aRef.length == a.length() && bRef.length == b.length()) {
                // is it only one array slice we are comparing?
                return f.applyAsInt(aRef, bRef);
            } else {
                for (int i = 0; i < lengthToCompare; ) {
                    if (aRef.length == 0) {
                        // must be non null otherwise we have a bug
                        aRef = aIter.next().clone();
                    }
                    if (bRef.length == 0) {
                        // must be non null otherwise we have a bug
                        bRef = bIter.next().clone();
                    }
                    final int aLength = aRef.length;
                    final int bLength = bRef.length;
                    // shrink to the same length and use the fast compare in lucene
                    final int length = Math.min(aLength, bLength);
                    aRef.length = bRef.length = length;
                    // now we move to the fast comparison - this is the hot part of the loop
                    int diff = f.applyAsInt(aRef, bRef);
                    aRef.length = aLength;
                    bRef.length = bLength;
                    if (diff != 0) {
                        return diff;
                    }
                    advance(aRef, length);
                    advance(bRef, length);
                    i += length;
                }
            }
        }
        // One is a prefix of the other, or, they are equal:
        return a.length() - b.length();
    } catch (IOException ex) {
        throw new AssertionError("can not happen", ex);
    }
}

Example 15 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class AbstractBytesReference method writeTo.

@Override
public void writeTo(OutputStream os) throws IOException {
    final BytesRefIterator iterator = iterator();
    BytesRef ref;
    while ((ref = iterator.next()) != null) {
        os.write(ref.bytes, ref.offset, ref.length);
    }
}