Examples with ReleasableBytesReference org.elasticsearch.common.bytes.ReleasableBytesReference used on opensource projects

Example 11 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class TransportDecompressorTests method testMultiPageCompression.

public void testMultiPageCompression() throws IOException {
    try (BytesStreamOutput output = new BytesStreamOutput()) {
        StreamOutput deflateStream = CompressorFactory.COMPRESSOR.streamOutput(Streams.flushOnCloseStream(output));
        for (int i = 0; i < 10000; ++i) {
            deflateStream.writeInt(i);
        }
        deflateStream.close();
        BytesReference bytes = output.bytes();
        TransportDecompressor decompressor = new TransportDecompressor(PageCacheRecycler.NON_RECYCLING_INSTANCE);
        int bytesConsumed = decompressor.decompress(bytes);
        assertEquals(bytes.length(), bytesConsumed);
        assertTrue(decompressor.isEOS());
        ReleasableBytesReference reference1 = decompressor.pollDecompressedPage();
        ReleasableBytesReference reference2 = decompressor.pollDecompressedPage();
        ReleasableBytesReference reference3 = decompressor.pollDecompressedPage();
        assertNull(decompressor.pollDecompressedPage());
        CompositeBytesReference composite = new CompositeBytesReference(reference1, reference2, reference3);
        assertEquals(4 * 10000, composite.length());
        StreamInput streamInput = composite.streamInput();
        for (int i = 0; i < 10000; ++i) {
            assertEquals(i, streamInput.readInt());
        }
        Releasables.close(reference1, reference2, reference3);
    }
}

Example 12 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class Netty4MessageChannelHandler method channelRead.

@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
    assert Transports.assertTransportThread();
    assert msg instanceof ByteBuf : "Expected message type ByteBuf, found: " + msg.getClass();
    final ByteBuf buffer = (ByteBuf) msg;
    Netty4TcpChannel channel = ctx.channel().attr(Netty4Transport.CHANNEL_KEY).get();
    final BytesReference wrapped = Netty4Utils.toBytesReference(buffer);
    try (ReleasableBytesReference reference = new ReleasableBytesReference(wrapped, buffer::release)) {
        pipeline.handleBytes(channel, reference);
    }
}

Example 13 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class Translog method add.

/**
 * Adds an operation to the transaction log.
 *
 * @param operation the operation to add
 * @return the location of the operation in the translog
 * @throws IOException if adding the operation to the translog resulted in an I/O exception
 */
public Location add(final Operation operation) throws IOException {
    final ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(bigArrays);
    boolean successfullySerialized = false;
    try {
        final long start = out.position();
        out.skip(Integer.BYTES);
        writeOperationNoSize(new BufferedChecksumStreamOutput(out), operation);
        final long end = out.position();
        final int operationSize = (int) (end - Integer.BYTES - start);
        out.seek(start);
        out.writeInt(operationSize);
        out.seek(end);
        successfullySerialized = true;
        try (ReleasableBytesReference bytes = new ReleasableBytesReference(out.bytes(), out);
            ReleasableLock ignored = readLock.acquire()) {
            ensureOpen();
            if (operation.primaryTerm() > current.getPrimaryTerm()) {
                assert false : "Operation term is newer than the current term; " + "current term[" + current.getPrimaryTerm() + "], operation term[" + operation + "]";
                throw new IllegalArgumentException("Operation term is newer than the current term; " + "current term[" + current.getPrimaryTerm() + "], operation term[" + operation + "]");
            }
            return current.add(bytes, operation.seqNo());
        }
    } catch (final AlreadyClosedException | IOException ex) {
        closeOnTragicEvent(ex);
        throw ex;
    } catch (final Exception ex) {
        closeOnTragicEvent(ex);
        throw new TranslogException(shardId, "Failed to write operation [" + operation + "]", ex);
    } finally {
        if (successfullySerialized == false) {
            Releasables.close(out);
        }
    }
}

Example 14 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class TranslogWriter method syncUpTo.

/**
 * Syncs the translog up to at least the given offset unless already synced
 *
 * @return <code>true</code> if this call caused an actual sync operation
 */
public boolean syncUpTo(long offset) throws IOException {
    if (lastSyncedCheckpoint.offset < offset && syncNeeded()) {
        synchronized (syncLock) {
            // only one sync/checkpoint should happen concurrently but we wait
            if (lastSyncedCheckpoint.offset < offset && syncNeeded()) {
                // double checked locking - we don't want to fsync unless we have to and now that we have
                // the lock we should check again since if this code is busy we might have fsynced enough already
                final Checkpoint checkpointToSync;
                final LongArrayList flushedSequenceNumbers;
                final ArrayDeque<ReleasableBytesReference> toWrite;
                try (ReleasableLock toClose = writeLock.acquire()) {
                    synchronized (this) {
                        ensureOpen();
                        checkpointToSync = getCheckpoint();
                        toWrite = pollOpsToWrite();
                        flushedSequenceNumbers = nonFsyncedSequenceNumbers;
                        nonFsyncedSequenceNumbers = new LongArrayList(64);
                    }
                    try {
                        // Write ops will release operations.
                        writeAndReleaseOps(toWrite);
                    } catch (final Exception ex) {
                        closeWithTragicEvent(ex);
                        throw ex;
                    }
                }
                // we can continue writing to the buffer etc.
                try {
                    channel.force(false);
                    writeCheckpoint(checkpointChannel, checkpointPath, checkpointToSync);
                } catch (final Exception ex) {
                    closeWithTragicEvent(ex);
                    throw ex;
                }
                flushedSequenceNumbers.forEach((LongProcedure) persistedSequenceNumberConsumer::accept);
                assert lastSyncedCheckpoint.offset <= checkpointToSync.offset : "illegal state: " + lastSyncedCheckpoint.offset + " <= " + checkpointToSync.offset;
                // write protected by syncLock
                lastSyncedCheckpoint = checkpointToSync;
                return true;
            }
        }
    }
    return false;
}

Example 15 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class TranslogWriter method writeAndReleaseOps.

private void writeAndReleaseOps(final ArrayDeque<ReleasableBytesReference> operationsToWrite) throws IOException {
    try {
        assert writeLock.isHeldByCurrentThread();
        ByteBuffer ioBuffer = DiskIoBufferPool.getIoBuffer();
        ReleasableBytesReference operation;
        while ((operation = operationsToWrite.pollFirst()) != null) {
            try (Releasable toClose = operation) {
                BytesRefIterator iterator = operation.iterator();
                BytesRef current;
                while ((current = iterator.next()) != null) {
                    int currentBytesConsumed = 0;
                    while (currentBytesConsumed != current.length) {
                        int nBytesToWrite = Math.min(current.length - currentBytesConsumed, ioBuffer.remaining());
                        ioBuffer.put(current.bytes, current.offset + currentBytesConsumed, nBytesToWrite);
                        currentBytesConsumed += nBytesToWrite;
                        if (ioBuffer.hasRemaining() == false) {
                            ioBuffer.flip();
                            writeToFile(ioBuffer);
                            ioBuffer.clear();
                        }
                    }
                }
            }
        }
        ioBuffer.flip();
        writeToFile(ioBuffer);
    } finally {
        Releasables.close(operationsToWrite);
    }
}