Examples with BytesRefIterator org.apache.lucene.util.BytesRefIterator used on opensource projects

Example 31 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class BytesReference method toByteBuffers.

/**
 * Returns an array of byte buffers from the given BytesReference.
 */
static ByteBuffer[] toByteBuffers(BytesReference reference) {
    BytesRefIterator byteRefIterator = reference.iterator();
    BytesRef r;
    try {
        ArrayList<ByteBuffer> buffers = new ArrayList<>();
        while ((r = byteRefIterator.next()) != null) {
            buffers.add(ByteBuffer.wrap(r.bytes, r.offset, r.length));
        }
        return buffers.toArray(new ByteBuffer[buffers.size()]);
    } catch (IOException e) {
        // this is really an error since we don't do IO in our bytesreferences
        throw new AssertionError("won't happen", e);
    }
}

Example 32 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class CompositeBytesReference method iterator.

@Override
public BytesRefIterator iterator() {
    if (references.length > 0) {
        return new BytesRefIterator() {

            int index = 0;

            private BytesRefIterator current = references[index++].iterator();

            @Override
            public BytesRef next() throws IOException {
                BytesRef next = current.next();
                if (next == null) {
                    while (index < references.length) {
                        current = references[index++].iterator();
                        next = current.next();
                        if (next != null) {
                            break;
                        }
                    }
                }
                return next;
            }
        };
    } else {
        return () -> null;
    }
}

Example 33 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class MultiFileWriter method innerWriteFileChunk.

private void innerWriteFileChunk(StoreFileMetadata fileMetadata, long position, BytesReference content, boolean lastChunk) throws IOException {
    final String name = fileMetadata.name();
    IndexOutput indexOutput;
    if (position == 0) {
        indexOutput = openAndPutIndexOutput(name, fileMetadata, store);
    } else {
        indexOutput = getOpenIndexOutput(name);
    }
    assert indexOutput.getFilePointer() == position : "file-pointer " + indexOutput.getFilePointer() + " != " + position;
    BytesRefIterator iterator = content.iterator();
    BytesRef scratch;
    while ((scratch = iterator.next()) != null) {
        // we iterate over all pages - this is a 0-copy for all core impls
        indexOutput.writeBytes(scratch.bytes, scratch.offset, scratch.length);
    }
    indexState.addRecoveredBytesToFile(name, content.length());
    if (indexOutput.getFilePointer() >= fileMetadata.length() || lastChunk) {
        try {
            Store.verify(indexOutput);
        } finally {
            // we are done
            indexOutput.close();
        }
        final String temporaryFileName = getTempNameForFile(name);
        assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName) : "expected: [" + temporaryFileName + "] in " + Arrays.toString(store.directory().listAll());
        store.directory().sync(Collections.singleton(temporaryFileName));
        IndexOutput remove = removeOpenIndexOutputs(name);
        // remove maybe null if we got finished
        assert remove == null || remove == indexOutput;
    }
}

Example 34 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class AbstractBytesReference method hashCode.

@Override
public int hashCode() {
    if (hash == null) {
        final BytesRefIterator iterator = iterator();
        BytesRef ref;
        int result = 1;
        try {
            while ((ref = iterator.next()) != null) {
                for (int i = 0; i < ref.length; i++) {
                    result = 31 * result + ref.bytes[ref.offset + i];
                }
            }
        } catch (IOException ex) {
            throw new AssertionError("wont happen", ex);
        }
        return hash = result;
    } else {
        return hash.intValue();
    }
}

Example 35 with BytesRefIterator

use of org.apache.lucene.util.BytesRefIterator in project crate by crate.

the class TranslogWriter method writeAndReleaseOps.

private void writeAndReleaseOps(final ArrayDeque<ReleasableBytesReference> operationsToWrite) throws IOException {
    try {
        assert writeLock.isHeldByCurrentThread();
        ByteBuffer ioBuffer = DiskIoBufferPool.getIoBuffer();
        ReleasableBytesReference operation;
        while ((operation = operationsToWrite.pollFirst()) != null) {
            try (Releasable toClose = operation) {
                BytesRefIterator iterator = operation.iterator();
                BytesRef current;
                while ((current = iterator.next()) != null) {
                    int currentBytesConsumed = 0;
                    while (currentBytesConsumed != current.length) {
                        int nBytesToWrite = Math.min(current.length - currentBytesConsumed, ioBuffer.remaining());
                        ioBuffer.put(current.bytes, current.offset + currentBytesConsumed, nBytesToWrite);
                        currentBytesConsumed += nBytesToWrite;
                        if (ioBuffer.hasRemaining() == false) {
                            ioBuffer.flip();
                            writeToFile(ioBuffer);
                            ioBuffer.clear();
                        }
                    }
                }
            }
        }
        ioBuffer.flip();
        writeToFile(ioBuffer);
    } finally {
        Releasables.close(operationsToWrite);
    }
}