Examples with Append io.pravega.shared.protocol.netty.Append used on opensource projects

Example 61 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class SegmentOutputStreamTest method testSegmentSealedFollowedbyConnectionDrop.

@Test(timeout = 10000)
public void testSegmentSealedFollowedbyConnectionDrop() throws Exception {
    @Cleanup("shutdownNow") ScheduledExecutorService executor = ExecutorServiceHelpers.newScheduledThreadPool(2, "netty-callback");
    // Segment sealed callback will finish execution only when the releaseCallbackLatch is released;
    ReusableLatch releaseCallbackLatch = new ReusableLatch(false);
    ReusableLatch callBackInvokedLatch = new ReusableLatch(false);
    final Consumer<Segment> segmentSealedCallback = segment -> Exceptions.handleInterrupted(() -> {
        callBackInvokedLatch.release();
        releaseCallbackLatch.await();
    });
    // Setup mocks.
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    // Mock client connection that is returned for every invocation of ConnectionFactory#establishConnection.
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    // Create a Segment writer.
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    // trigger establishment of connection.
    output.reconnect();
    // Verify if SetupAppend is sent over the connection.
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    // Write an event and ensure inflight has an event.
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> ack = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, ack));
    order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertFalse(ack.isDone());
    // Simulate a SegmentIsSealed WireCommand from SegmentStore.
    executor.submit(() -> cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(output.getRequestId(), SEGMENT, "SomeException", 1)));
    // Wait until callback invocation has been triggered, but has not completed.
    // If the callback is not invoked the test will fail due to a timeout.
    callBackInvokedLatch.await();
    // Now trigger a connection drop netty callback and wait until it is executed.
    executor.submit(() -> cf.getProcessor(uri).connectionDropped()).get();
    // close is invoked on the connection.
    order.verify(connection).close();
    // Verify no further reconnection attempts which involves sending of SetupAppend wire command.
    order.verifyNoMoreInteractions();
    // Release latch to ensure the callback is completed.
    releaseCallbackLatch.release();
    // Verify no further reconnection attempts which involves sending of SetupAppend wire command.
    order.verifyNoMoreInteractions();
    // Trigger a reconnect again and verify if any new connections are initiated.
    output.reconnect();
    // Reconnect operation will be executed on the executor service.
    ScheduledExecutorService service = executorService();
    service.shutdown();
    // Wait until all the tasks for reconnect have been completed.
    service.awaitTermination(10, TimeUnit.SECONDS);
    // Verify no further reconnection attempts which involves sending of SetupAppend wire command.
    order.verifyNoMoreInteractions();
}

Example 62 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class SegmentOutputStreamTest method testReconnectOnBadAcks.

@Test(timeout = 10000)
public void testReconnectOnBadAcks() throws ConnectionFailedException {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = spy(new MockConnectionFactoryImpl());
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    // Ensure task submitted to executor is run inline.
    implementAsDirectExecutor(executor);
    cf.setExecutor(executor);
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0));
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> acked1 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, acked1));
    order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertEquals(false, acked1.isDone());
    AssertExtensions.assertBlocks(() -> output.flush(), () -> cf.getProcessor(uri).dataAppended(new WireCommands.DataAppended(output.getRequestId(), cid, 1, 0, -1)));
    assertEquals(false, acked1.isCompletedExceptionally());
    assertEquals(true, acked1.isDone());
    order.verify(connection).send(new WireCommands.KeepAlive());
    // simulate bad ack
    CompletableFuture<Void> acked2 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, acked2));
    order.verify(connection).send(new Append(SEGMENT, cid, 2, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertEquals(false, acked2.isDone());
    cf.getProcessor(uri).dataAppended(new WireCommands.DataAppended(output.getRequestId(), cid, 2, 3, -1));
    // check that client reconnected
    verify(cf, times(2)).establishConnection(any(), any());
}

Example 63 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testConditionalAppendFailure.

@Test
public void testConditionalAppendFailure() {
    String streamSegmentName = "scope/stream/testConditionalAppendFailure";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    val mockedRecorder = Mockito.mock(SegmentStatsRecorder.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    val ac1 = interceptAppend(store, streamSegmentName, 0, updateEventNumber(clientId, 1), CompletableFuture.completedFuture((long) data.length));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data), 0L, requestId));
    val ac2 = interceptAppend(store, streamSegmentName, 0, updateEventNumber(clientId, 2, 1, 1), Futures.failedFuture(new BadOffsetException(streamSegmentName, data.length, 0)));
    processor.append(new Append(streamSegmentName, clientId, 2, 1, Unpooled.wrappedBuffer(data), 0L, requestId));
    val ac3 = interceptAppend(store, streamSegmentName, 0, updateEventNumber(clientId, 3, 1, 1), Futures.failedFuture(new BadAttributeUpdateException(streamSegmentName, new AttributeUpdate(AttributeId.fromUUID(clientId), null, 0), true, "test")));
    processor.append(new Append(streamSegmentName, clientId, 3, 1, Unpooled.wrappedBuffer(data), 0L, requestId));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac1, data);
    verifyStoreAppend(ac2, data);
    verifyStoreAppend(ac3, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker, times(3)).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new DataAppended(requestId, clientId, 1, 0, data.length));
    verify(connection).send(new ConditionalCheckFailed(clientId, 2, requestId));
    verify(connection).send(new InvalidEventNumber(clientId, requestId, "test"));
    verify(connection).close();
    verify(tracker, times(3)).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
    verify(mockedRecorder).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
}

Example 64 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testAppendFailChannelClose.

@Test(timeout = 5000)
public void testAppendFailChannelClose() throws Exception {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    // Setup mocks.
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    setupGetAttributes(streamSegmentName, clientId, store);
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(new IntentionalException()));
    @Cleanup EmbeddedChannel channel = createChannel(store);
    // Send a setup append WireCommand.
    Reply reply = sendRequest(channel, new SetupAppend(requestId, clientId, streamSegmentName, ""));
    assertEquals(new AppendSetup(requestId, streamSegmentName, clientId, 0), reply);
    // Send an append which will cause a RuntimeException to be thrown by the store.
    reply = sendRequest(channel, new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    assertNull("No WireCommand reply is expected", reply);
    // Verify that the channel is closed by the AppendProcessor.
    assertEventuallyEquals(false, channel::isOpen, 3000);
}

Example 65 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testBadAttributeException.

@Test(timeout = 5000)
public void testBadAttributeException() throws Exception {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    // Setup mocks.
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    setupGetAttributes(streamSegmentName, clientId, store);
    val ex = new BadAttributeUpdateException(streamSegmentName, new AttributeUpdate(AttributeId.randomUUID(), AttributeUpdateType.ReplaceIfEquals, 100, 101), false, "error");
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(ex));
    @Cleanup EmbeddedChannel channel = createChannel(store);
    // Send a setup append WireCommand.
    Reply reply = sendRequest(channel, new SetupAppend(requestId, clientId, streamSegmentName, ""));
    assertEquals(new AppendSetup(requestId, streamSegmentName, clientId, 0), reply);
    // Send an append which will cause a RuntimeException to be thrown by the store.
    reply = sendRequest(channel, new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    // validate InvalidEventNumber Wirecommand is sent before closing the Channel.
    assertNotNull("Invalid Event WireCommand is expected", reply);
    assertEquals(WireCommandType.INVALID_EVENT_NUMBER.getCode(), ((WireCommand) reply).getType().getCode());
    assertEquals(requestId, ((InvalidEventNumber) reply).getRequestId());
    // Verify that the channel is closed by the AppendProcessor.
    assertEventuallyEquals(false, channel::isOpen, 3000);
}