Examples with AppendSetup io.pravega.shared.protocol.netty.WireCommands.AppendSetup used on opensource projects

Example 36 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testMultipleSetupAppendDueToConnectionFailure.

@Test
public void testMultipleSetupAppendDueToConnectionFailure() {
    String streamSegmentName = "scope/stream/testMultipleSetupAppend";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    val mockedRecorder = Mockito.mock(SegmentStatsRecorder.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    // Setup mock to enusure both the SetupAppends return the newer values.
    val clientIdAttribute = AttributeId.fromUUID(clientId);
    when(store.getAttributes(streamSegmentName, Collections.singleton(clientIdAttribute), true, AppendProcessor.TIMEOUT)).thenReturn(CompletableFuture.completedFuture(Collections.singletonMap(clientIdAttribute, 0L))).thenReturn(CompletableFuture.completedFuture(Collections.singletonMap(clientIdAttribute, 1L)));
    val ac1 = interceptAppend(store, streamSegmentName, 0, updateEventNumber(clientId, 1), CompletableFuture.completedFuture((long) data.length));
    // First conditional Append.
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data), 0L, 1L));
    // Simulate a connection drop on the client side and the client resends a SetupAppend followed by conditional Append.
    val ac2 = interceptAppend(store, streamSegmentName, data.length, updateEventNumber(clientId, 2, 1, 1), CompletableFuture.completedFuture((long) data.length * 2));
    processor.setupAppend(new SetupAppend(2, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, 2, 1, Unpooled.wrappedBuffer(data), (long) data.length, 2));
    verify(store, times(2)).getAttributes(anyString(), eq(Collections.singleton(clientIdAttribute)), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac1, data);
    verifyStoreAppend(ac2, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker, times(2)).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new DataAppended(1, clientId, 1, 0, data.length));
    // verify that the second SetupAppend is responded by the AppendProcessor.
    verify(connection).send(new AppendSetup(2, streamSegmentName, clientId, 1));
    verify(connection).send(new DataAppended(2, clientId, 2, 1, data.length * 2));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
    verify(mockedRecorder, times(2)).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
}

Example 37 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testUnsupportedOperation.

@Test
public void testUnsupportedOperation() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    val ac = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(new UnsupportedOperationException()));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new OperationUnsupported(requestId, "appending data", ""));
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 38 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class SegmentOutputStreamTest method testFlushWithMultipleConnectFailures.

@Test(timeout = 10000)
public void testFlushWithMultipleConnectFailures() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    RetryWithBackoff retryConfig = Retry.withExpBackoff(1, 1, 1);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    // Ensure task submitted to executor is run inline.
    implementAsDirectExecutor(executor);
    cf.setExecutor(executor);
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, retryConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    try {
        output.reconnect();
        verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
        // Simulate a successful connection setup.
        cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
        // try sending an event.
        byte[] eventData = "test data".getBytes();
        CompletableFuture<Void> acked = new CompletableFuture<>();
        // this is an inflight event and the client will track it until there is a response from SSS.
        output.write(PendingEvent.withoutHeader(null, ByteBuffer.wrap(eventData), acked));
        verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(eventData), null, output.getRequestId()));
        reset(connection);
        // simulate a connection drop and verify if the writer tries to establish a new connection.
        cf.getProcessor(uri).connectionDropped();
        verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
        reset(connection);
        // Verify flush blocks until there is a response from SSS. Incase of connection error the client retries. If the
        // retry count more than the configuration ensure flush returns exceptionally.
        AssertExtensions.assertBlocks(() -> AssertExtensions.assertThrows(RetriesExhaustedException.class, output::flush), () -> cf.getProcessor(uri).connectionDropped());
        assertTrue("Connection is  exceptionally closed with RetriesExhaustedException", output.getConnection().isCompletedExceptionally());
        AssertExtensions.assertThrows(RetriesExhaustedException.class, () -> Futures.getThrowingException(output.getConnection()));
        // Verify that the inflight event future is completed exceptionally.
        AssertExtensions.assertThrows(RetriesExhaustedException.class, () -> Futures.getThrowingException(acked));
    } finally {
        // Verify that a close on the SegmentOutputStream does throw a RetriesExhaustedException.
        AssertExtensions.assertThrows(RetriesExhaustedException.class, output::close);
    }
}

Example 39 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class SegmentOutputStreamTest method testAlreadySealedSegment.

@Test(timeout = 10000)
public void testAlreadySealedSegment() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(output.getRequestId(), SEGMENT, "SomeException", 1));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    order.verifyNoMoreInteractions();
}

Example 40 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class SegmentOutputStreamTest method testConnectionFailureWithSegmentSealed.

@Test(timeout = 10000)
public void testConnectionFailureWithSegmentSealed() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    InOrder inOrder = Mockito.inOrder(connection);
    inOrder.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    // connection is setup .
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> acked = new CompletableFuture<>();
    Append append = new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId());
    CompletableFuture<Void> acked2 = new CompletableFuture<>();
    Append append2 = new Append(SEGMENT, cid, 2, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId());
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            // simulate in a race with segment is sealed callback and a connection drop.
            cf.getProcessor(uri).connectionDropped();
            // wait until the writer reattempts to establish a connection to simulate the race.
            verify(connection, times(2)).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
            // the connection object throws a throws a
            throw new ConnectionFailedException();
        }
    }).when(connection).send(append);
    // trigger the first write
    output.write(PendingEvent.withoutHeader(null, data, acked));
    AssertExtensions.assertBlocks(() -> {
        // the below write will be blocked since the connection setup is not complete.
        output.write(PendingEvent.withoutHeader(null, data, acked2));
    }, () -> {
        // simulate a race between segmentIsSealed response and appendSetup.
        cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(output.getRequestId(), SEGMENT, "Segment is sealed", 1));
        // invoke the segment is sealed to ensure
        cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
        // ensure the reconnect is invoked inline.
        output.reconnect();
    });
    CompletableFuture<Void> acked3 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, acked3));
    inOrder.verify(connection).send(append);
    inOrder.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    inOrder.verify(connection).send(eq(append2));
    // the setup append should not transmit the inflight events given that the segment is sealed.
    inOrder.verifyNoMoreInteractions();
    assertFalse(acked.isDone());
    assertFalse(acked2.isDone());
    assertFalse(acked3.isDone());
}