Examples with ReleasableBytesReference org.elasticsearch.common.bytes.ReleasableBytesReference used on opensource projects

Example 1 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class InboundPipeline method releasePendingBytes.

private void releasePendingBytes(int bytesConsumed) {
    int bytesToRelease = bytesConsumed;
    while (bytesToRelease != 0) {
        try (ReleasableBytesReference reference = pending.pollFirst()) {
            assert reference != null;
            if (bytesToRelease < reference.length()) {
                pending.addFirst(reference.retainedSlice(bytesToRelease, reference.length() - bytesToRelease));
                bytesToRelease -= bytesToRelease;
            } else {
                bytesToRelease -= reference.length();
            }
        }
    }
}

Example 2 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class InboundDecoderTests method testDecodeHandshakeCompatibility.

public void testDecodeHandshakeCompatibility() throws IOException {
    String action = "test-request";
    long requestId = randomNonNegativeLong();
    Version handshakeCompat = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
    OutboundMessage message = new OutboundMessage.Request(new TestRequest(randomAlphaOfLength(100)), handshakeCompat, action, requestId, true, false);
    final BytesReference bytes = message.serialize(new BytesStreamOutput());
    int totalHeaderSize = TcpHeader.headerSize(handshakeCompat);
    InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
    final ArrayList<Object> fragments = new ArrayList<>();
    final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(bytes);
    int bytesConsumed = decoder.decode(releasable1, fragments::add);
    assertEquals(totalHeaderSize, bytesConsumed);
    assertEquals(1, releasable1.refCount());
    final Header header = (Header) fragments.get(0);
    assertEquals(requestId, header.getRequestId());
    assertEquals(handshakeCompat, header.getVersion());
    assertFalse(header.isCompressed());
    assertTrue(header.isHandshake());
    assertTrue(header.isRequest());
    // TODO: On 9.0 this will be true because all compatible versions with contain the variable header int
    assertTrue(header.needsToReadVariableHeader());
    fragments.clear();
}

Example 3 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class InboundDecoderTests method testDecodePreHeaderSizeVariableInt.

public void testDecodePreHeaderSizeVariableInt() throws IOException {
    // TODO: Can delete test on 9.0
    boolean isCompressed = randomBoolean();
    String action = "test-request";
    long requestId = randomNonNegativeLong();
    final Version preHeaderVariableInt = Version.V_4_4_0;
    final String contentValue = randomAlphaOfLength(100);
    final OutboundMessage message = new OutboundMessage.Request(new TestRequest(contentValue), preHeaderVariableInt, action, requestId, true, isCompressed);
    final BytesReference totalBytes = message.serialize(new BytesStreamOutput());
    int partialHeaderSize = TcpHeader.headerSize(preHeaderVariableInt);
    InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
    final ArrayList<Object> fragments = new ArrayList<>();
    final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(totalBytes);
    int bytesConsumed = decoder.decode(releasable1, fragments::add);
    assertEquals(partialHeaderSize, bytesConsumed);
    assertEquals(1, releasable1.refCount());
    final Header header = (Header) fragments.get(0);
    assertEquals(requestId, header.getRequestId());
    assertEquals(preHeaderVariableInt, header.getVersion());
    assertEquals(isCompressed, header.isCompressed());
    assertTrue(header.isHandshake());
    assertTrue(header.isRequest());
    assertTrue(header.needsToReadVariableHeader());
    fragments.clear();
    final BytesReference bytes2 = totalBytes.slice(bytesConsumed, totalBytes.length() - bytesConsumed);
    final ReleasableBytesReference releasable2 = ReleasableBytesReference.wrap(bytes2);
    int bytesConsumed2 = decoder.decode(releasable2, fragments::add);
    assertEquals(2, fragments.size());
    assertEquals(InboundDecoder.END_CONTENT, fragments.get(fragments.size() - 1));
    assertEquals(totalBytes.length() - bytesConsumed, bytesConsumed2);
}

Example 4 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class InboundDecoderTests method testVersionIncompatibilityDecodeException.

public void testVersionIncompatibilityDecodeException() throws IOException {
    String action = "test-request";
    long requestId = randomNonNegativeLong();
    Version incompatibleVersion = Version.V_3_2_0;
    OutboundMessage message = new OutboundMessage.Request(new TestRequest(randomAlphaOfLength(100)), incompatibleVersion, action, requestId, false, true);
    final BytesReference bytes = message.serialize(new BytesStreamOutput());
    InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
    final ArrayList<Object> fragments = new ArrayList<>();
    final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(bytes);
    expectThrows(IllegalStateException.class, () -> decoder.decode(releasable1, fragments::add));
    // No bytes are retained
    assertEquals(1, releasable1.refCount());
}

Example 5 with ReleasableBytesReference

use of org.elasticsearch.common.bytes.ReleasableBytesReference in project crate by crate.

the class InboundPipelineTests method testPipelineHandling.

public void testPipelineHandling() throws IOException {
    final List<Tuple<MessageData, Exception>> expected = new ArrayList<>();
    final List<Tuple<MessageData, Exception>> actual = new ArrayList<>();
    final List<ReleasableBytesReference> toRelease = new ArrayList<>();
    final BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {
        try {
            final Header header = m.getHeader();
            final MessageData actualData;
            final Version version = header.getVersion();
            final boolean isRequest = header.isRequest();
            final long requestId = header.getRequestId();
            final boolean isCompressed = header.isCompressed();
            if (m.isShortCircuit()) {
                actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), null);
            } else if (isRequest) {
                final TestRequest request = new TestRequest(m.openOrGetStreamInput());
                actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), request.value);
            } else {
                final TestResponse response = new TestResponse(m.openOrGetStreamInput());
                actualData = new MessageData(version, requestId, isRequest, isCompressed, null, response.value);
            }
            actual.add(new Tuple<>(actualData, m.getException()));
        } catch (IOException e) {
            throw new AssertionError(e);
        }
    };
    final StatsTracker statsTracker = new StatsTracker();
    final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
    final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
    final String breakThisAction = "break_this_action";
    final String actionName = "actionName";
    final Predicate<String> canTripBreaker = breakThisAction::equals;
    final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker();
    circuitBreaker.startBreaking();
    final InboundAggregator aggregator = new InboundAggregator(() -> circuitBreaker, canTripBreaker);
    final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler);
    final FakeTcpChannel channel = new FakeTcpChannel();
    final int iterations = randomIntBetween(100, 500);
    long totalMessages = 0;
    long bytesReceived = 0;
    for (int i = 0; i < iterations; ++i) {
        actual.clear();
        expected.clear();
        toRelease.clear();
        try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
            while (streamOutput.size() < BYTE_THRESHOLD) {
                final Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
                final String value = randomAlphaOfLength(randomIntBetween(10, 200));
                final boolean isRequest = randomBoolean();
                final boolean isCompressed = randomBoolean();
                final long requestId = totalMessages++;
                final MessageData messageData;
                Exception expectedExceptionClass = null;
                OutboundMessage message;
                if (isRequest) {
                    if (rarely()) {
                        messageData = new MessageData(version, requestId, true, isCompressed, breakThisAction, null);
                        message = new OutboundMessage.Request(new TestRequest(value), version, breakThisAction, requestId, false, isCompressed);
                        expectedExceptionClass = new CircuitBreakingException("");
                    } else {
                        messageData = new MessageData(version, requestId, true, isCompressed, actionName, value);
                        message = new OutboundMessage.Request(new TestRequest(value), version, actionName, requestId, false, isCompressed);
                    }
                } else {
                    messageData = new MessageData(version, requestId, false, isCompressed, null, value);
                    message = new OutboundMessage.Response(new TestResponse(value), version, requestId, false, isCompressed);
                }
                expected.add(new Tuple<>(messageData, expectedExceptionClass));
                final BytesReference reference = message.serialize(new BytesStreamOutput());
                Streams.copy(reference.streamInput(), streamOutput);
            }
            final BytesReference networkBytes = streamOutput.bytes();
            int currentOffset = 0;
            while (currentOffset != networkBytes.length()) {
                final int remainingBytes = networkBytes.length() - currentOffset;
                final int bytesToRead = Math.min(randomIntBetween(1, 32 * 1024), remainingBytes);
                final BytesReference slice = networkBytes.slice(currentOffset, bytesToRead);
                try (ReleasableBytesReference reference = new ReleasableBytesReference(slice, () -> {
                })) {
                    toRelease.add(reference);
                    bytesReceived += reference.length();
                    pipeline.handleBytes(channel, reference);
                    currentOffset += bytesToRead;
                }
            }
            final int messages = expected.size();
            for (int j = 0; j < messages; ++j) {
                final Tuple<MessageData, Exception> expectedTuple = expected.get(j);
                final Tuple<MessageData, Exception> actualTuple = actual.get(j);
                final MessageData expectedMessageData = expectedTuple.v1();
                final MessageData actualMessageData = actualTuple.v1();
                assertEquals(expectedMessageData.requestId, actualMessageData.requestId);
                assertEquals(expectedMessageData.isRequest, actualMessageData.isRequest);
                assertEquals(expectedMessageData.isCompressed, actualMessageData.isCompressed);
                assertEquals(expectedMessageData.actionName, actualMessageData.actionName);
                assertEquals(expectedMessageData.value, actualMessageData.value);
                if (expectedTuple.v2() != null) {
                    assertNotNull(actualTuple.v2());
                    assertThat(actualTuple.v2(), instanceOf(expectedTuple.v2().getClass()));
                }
            }
            for (ReleasableBytesReference released : toRelease) {
                assertEquals(0, released.refCount());
            }
        }
        assertEquals(bytesReceived, statsTracker.getBytesRead());
        assertEquals(totalMessages, statsTracker.getMessagesReceived());
    }
}