Examples with Append io.pravega.shared.protocol.netty.Append used on opensource projects

Example 36 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testDelayedDataAppended.

/**
 * Test to ensure newer appends are processed only after successfully sending the DataAppended acknowledgement
 * back to client. This test tests the following:
 * - If sending first DataAppended is blocked, ensure future appends are not written to store.
 * - Once the first DataAppended is sent ensure the remaining appends are written to store and DataAppended ack'ed
 * back.
 */
@Test(timeout = 15 * 1000)
public void testDelayedDataAppended() throws Exception {
    ReusableLatch firstStoreAppendInvoked = new ReusableLatch();
    ReusableLatch completeFirstDataAppendedAck = new ReusableLatch();
    ReusableLatch secondStoreAppendInvoked = new ReusableLatch();
    @Cleanup("shutdownNow") ScheduledExecutorService nettyExecutor = ExecutorServiceHelpers.newScheduledThreadPool(1, "Netty-threadPool");
    String streamSegmentName = "testDelayedAppend";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    // Ensure the first DataAppended is hung/delayed.
    doAnswer(invocation -> {
        firstStoreAppendInvoked.release();
        // wait, simulating a hung/delayed dataAppended acknowledgement.
        completeFirstDataAppendedAck.await();
        return null;
    }).doAnswer(invocation -> {
        secondStoreAppendInvoked.release();
        return null;
    }).when(connection).send(any(DataAppended.class));
    AppendProcessor processor = new AppendProcessor(store, connection, new FailingRequestProcessor(), null);
    CompletableFuture<SegmentProperties> propsFuture = CompletableFuture.completedFuture(StreamSegmentInformation.builder().name(streamSegmentName).build());
    when(store.getStreamSegmentInfo(streamSegmentName, true, AppendProcessor.TIMEOUT)).thenReturn(propsFuture);
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    verify(store).getStreamSegmentInfo(streamSegmentName, true, AppendProcessor.TIMEOUT);
    CompletableFuture<Void> result = CompletableFuture.completedFuture(null);
    int eventCount = 100;
    when(store.append(streamSegmentName, data, updateEventNumber(clientId, 100, SegmentMetadata.NULL_ATTRIBUTE_VALUE, eventCount), AppendProcessor.TIMEOUT)).thenReturn(result);
    // Trigger the first append, here the sending of DataAppended ack will be delayed/hung.
    nettyExecutor.submit(() -> processor.append(new Append(streamSegmentName, clientId, 100, eventCount, Unpooled.wrappedBuffer(data), null)));
    firstStoreAppendInvoked.await();
    verify(store).append(streamSegmentName, data, updateEventNumber(clientId, 100, SegmentMetadata.NULL_ATTRIBUTE_VALUE, eventCount), AppendProcessor.TIMEOUT);
    /* Trigger the next append. This should be completed immediately and should not cause a store.append to be
        invoked as the previous DataAppended ack is still not sent. */
    processor.append(new Append(streamSegmentName, clientId, 200, eventCount, Unpooled.wrappedBuffer(data), null));
    // Since the first Ack was never sent the next append should not be written to the store.
    verifyNoMoreInteractions(store);
    // Setup mock for check behaviour after the delayed/hung dataAppended completes.
    when(store.append(streamSegmentName, data, updateEventNumber(clientId, 200, 100, eventCount), AppendProcessor.TIMEOUT)).thenReturn(result);
    // Now ensure the dataAppended sent
    completeFirstDataAppendedAck.release();
    // wait until the next store append is invoked.
    secondStoreAppendInvoked.await();
    // Verify that the next store append invoked.
    verify(store).append(streamSegmentName, data, updateEventNumber(clientId, 200, 100, eventCount), AppendProcessor.TIMEOUT);
    // Verify two DataAppended acks are sent out.
    verify(connection, times(2)).send(any(DataAppended.class));
    verify(connection).send(new DataAppended(clientId, 100, Long.MIN_VALUE));
    verify(connection).send(new DataAppended(clientId, 200, 100));
}

Example 37 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class SegmentOutputStreamTest method sendAndVerifyEvent.

private void sendAndVerifyEvent(UUID cid, ClientConnection connection, SegmentOutputStreamImpl output, ByteBuffer data, int num, Long expectedLength) throws SegmentSealedException, ConnectionFailedException {
    CompletableFuture<Boolean> acked = new CompletableFuture<>();
    output.write(new PendingEvent(null, data, acked, expectedLength));
    verify(connection).send(new Append(SEGMENT, cid, num, Unpooled.wrappedBuffer(data), expectedLength));
    assertEquals(false, acked.isDone());
}

Example 38 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendTest method sendReceivingAppend.

@Test(timeout = 10000)
public void sendReceivingAppend() throws Exception {
    String segment = "sendReceivingAppend";
    ByteBuf data = Unpooled.wrappedBuffer("Hello world\n".getBytes());
    StreamSegmentStore store = SERVICE_BUILDER.createStreamSegmentService();
    @Cleanup EmbeddedChannel channel = createChannel(store);
    SegmentCreated created = (SegmentCreated) sendRequest(channel, new CreateSegment(1, segment, CreateSegment.NO_SCALE, 0, "", 1024L));
    assertEquals(segment, created.getSegment());
    UUID uuid = UUID.randomUUID();
    AppendSetup setup = (AppendSetup) sendRequest(channel, new SetupAppend(2, uuid, segment, ""));
    assertEquals(segment, setup.getSegment());
    assertEquals(uuid, setup.getWriterId());
    DataAppended ack = (DataAppended) sendRequest(channel, new Append(segment, uuid, data.readableBytes(), new Event(data), 1L));
    assertEquals(uuid, ack.getWriterId());
    assertEquals(data.readableBytes(), ack.getEventNumber());
    assertEquals(Long.MIN_VALUE, ack.getPreviousEventNumber());
}

Example 39 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testAppendPipeliningWithSeal.

@Test(timeout = 15 * 1000)
public void testAppendPipeliningWithSeal() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data1 = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    byte[] data2 = new byte[] { 1, 2, 3, 4, 5 };
    byte[] data3 = new byte[] { 1, 2, 3 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    InOrder connectionVerifier = Mockito.inOrder(connection);
    setupGetAttributes(streamSegmentName, clientId, store);
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    connectionVerifier.verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    // Initiate one append.
    val store1 = new CompletableFuture<Long>();
    val ac1 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 1, 0, 1), store1);
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data1), null, requestId));
    verifyStoreAppend(ac1, data1);
    verify(tracker).updateOutstandingBytes(connection, data1.length, data1.length);
    // Second append will fail with StreamSegmentSealedException (this simulates having one append, immediately followed
    // by a Seal, then another append).
    val ac2 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 2, 1, 1), Futures.failedFuture(new StreamSegmentSealedException(streamSegmentName)));
    processor.append(new Append(streamSegmentName, clientId, 2, 1, Unpooled.wrappedBuffer(data2), null, requestId));
    verifyStoreAppend(ac2, data2);
    verify(tracker).updateOutstandingBytes(connection, data2.length, data1.length + data2.length);
    verify(tracker).updateOutstandingBytes(connection, -data2.length, data1.length);
    // Third append should fail just like the second one. However this will have a higher event number than that one
    // and we use it to verify it won't prevent sending the SegmentIsSealed message or send it multiple times.
    val ac3 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 3, 2, 1), Futures.failedFuture(new StreamSegmentSealedException(streamSegmentName)));
    processor.append(new Append(streamSegmentName, clientId, 3, 1, Unpooled.wrappedBuffer(data3), null, requestId));
    verifyStoreAppend(ac3, data3);
    verify(tracker).updateOutstandingBytes(connection, data3.length, data1.length + data3.length);
    // Complete the first one and verify it is acked properly.
    store1.complete(100L);
    connectionVerifier.verify(connection).send(new DataAppended(requestId, clientId, 1, 0L, 100L));
    // Verify that a SegmentIsSealed message is sent AFTER the ack from the first one (for the second append).
    connectionVerifier.verify(connection).send(new WireCommands.SegmentIsSealed(requestId, streamSegmentName, "", 2L));
    // Verify that a second SegmentIsSealed is sent (for the third append).
    connectionVerifier.verify(connection).send(new WireCommands.SegmentIsSealed(requestId, streamSegmentName, "", 3L));
    verify(tracker).updateOutstandingBytes(connection, -data3.length, data1.length);
    verify(tracker).updateOutstandingBytes(connection, -data1.length, 0);
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 4, 3, 1), Futures.failedFuture(new IntentionalException(streamSegmentName)));
    AssertExtensions.assertThrows("append() accepted a new request after sending a SegmentIsSealed message.", () -> processor.append(new Append(streamSegmentName, clientId, 4, 1, Unpooled.wrappedBuffer(data1), null, requestId)), ex -> ex instanceof IllegalStateException);
    // Verify no more messages are sent over the connection.
    connectionVerifier.verifyNoMoreInteractions();
    verifyNoMoreInteractions(store);
}

Example 40 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testAppendFails.

@Test
public void testAppendFails() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    val mockedRecorder = mock(SegmentStatsRecorder.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(new IntentionalException()));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    try {
        processor.append(new Append(streamSegmentName, clientId, data.length * 2, 1, Unpooled.wrappedBuffer(data), null, requestId));
        fail();
    } catch (IllegalStateException e) {
    // Expected
    }
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).close();
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verify(store, atMost(1)).append(any(), any(), any(), any());
    verifyNoMoreInteractions(connection);
    verify(mockedRecorder, never()).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
}