Examples with AppendSetup io.pravega.shared.protocol.netty.WireCommands.AppendSetup used on opensource projects

Example 46 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testAppendPipelining.

/**
 * Verifies that appends are "pipelined" into the underlying store and that acks are sent as appropriate via the
 * connection when they complete.
 */
@Test(timeout = 15 * 1000)
public void testAppendPipelining() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data1 = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    byte[] data2 = new byte[] { 1, 2, 3, 4, 5 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    // Initiate two appends in short sequence, and simulate the Store blocking on both of them.
    val store1 = new CompletableFuture<Long>();
    val ac1 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 1, 0, 1), store1);
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data1), null, requestId));
    val store2 = new CompletableFuture<Long>();
    val ac2 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 2, 1, 1), store2);
    processor.append(new Append(streamSegmentName, clientId, 2, 1, Unpooled.wrappedBuffer(data2), null, requestId));
    verifyStoreAppend(ac1, data1);
    verifyStoreAppend(ac2, data2);
    verify(tracker).updateOutstandingBytes(connection, data1.length, data1.length);
    verify(tracker).updateOutstandingBytes(connection, data2.length, data1.length + data2.length);
    // Complete the second one (this simulates acks arriving out of order from the store).
    store2.complete(100L);
    // Verify an ack is sent for both appends (because of pipelining guarantees), but only the second append's length
    // is subtracted from the outstanding bytes.
    verify(connection).send(new DataAppended(requestId, clientId, 2, 0L, 100L));
    verify(tracker).updateOutstandingBytes(connection, -data2.length, data1.length);
    verifyNoMoreInteractions(connection);
    // Complete the first one, and verify that no additional acks are sent via the connection, but the first append's
    // length is subtracted from the outstanding bytes.
    store1.complete(50L);
    verify(tracker).updateOutstandingBytes(connection, -data1.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 47 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testMultipleSetupAppendDueToConnectionFailure.

@Test
public void testMultipleSetupAppendDueToConnectionFailure() {
    String streamSegmentName = "scope/stream/testMultipleSetupAppend";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    val mockedRecorder = Mockito.mock(SegmentStatsRecorder.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    // Setup mock to enusure both the SetupAppends return the newer values.
    val clientIdAttribute = AttributeId.fromUUID(clientId);
    when(store.getAttributes(streamSegmentName, Collections.singleton(clientIdAttribute), true, AppendProcessor.TIMEOUT)).thenReturn(CompletableFuture.completedFuture(Collections.singletonMap(clientIdAttribute, 0L))).thenReturn(CompletableFuture.completedFuture(Collections.singletonMap(clientIdAttribute, 1L)));
    val ac1 = interceptAppend(store, streamSegmentName, 0, updateEventNumber(clientId, 1), CompletableFuture.completedFuture((long) data.length));
    // First conditional Append.
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data), 0L, 1L));
    // Simulate a connection drop on the client side and the client resends a SetupAppend followed by conditional Append.
    val ac2 = interceptAppend(store, streamSegmentName, data.length, updateEventNumber(clientId, 2, 1, 1), CompletableFuture.completedFuture((long) data.length * 2));
    processor.setupAppend(new SetupAppend(2, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, 2, 1, Unpooled.wrappedBuffer(data), (long) data.length, 2));
    verify(store, times(2)).getAttributes(anyString(), eq(Collections.singleton(clientIdAttribute)), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac1, data);
    verifyStoreAppend(ac2, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker, times(2)).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new DataAppended(1, clientId, 1, 0, data.length));
    // verify that the second SetupAppend is responded by the AppendProcessor.
    verify(connection).send(new AppendSetup(2, streamSegmentName, clientId, 1));
    verify(connection).send(new DataAppended(2, clientId, 2, 1, data.length * 2));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
    verify(mockedRecorder, times(2)).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
}

Example 48 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testUnsupportedOperation.

@Test
public void testUnsupportedOperation() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    val ac = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(new UnsupportedOperationException()));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new OperationUnsupported(requestId, "appending data", ""));
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 49 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testAppendPipeliningWithSeal.

@Test(timeout = 15 * 1000)
public void testAppendPipeliningWithSeal() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data1 = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    byte[] data2 = new byte[] { 1, 2, 3, 4, 5 };
    byte[] data3 = new byte[] { 1, 2, 3 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    InOrder connectionVerifier = Mockito.inOrder(connection);
    setupGetAttributes(streamSegmentName, clientId, store);
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    connectionVerifier.verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    // Initiate one append.
    val store1 = new CompletableFuture<Long>();
    val ac1 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 1, 0, 1), store1);
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data1), null, requestId));
    verifyStoreAppend(ac1, data1);
    verify(tracker).updateOutstandingBytes(connection, data1.length, data1.length);
    // Second append will fail with StreamSegmentSealedException (this simulates having one append, immediately followed
    // by a Seal, then another append).
    val ac2 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 2, 1, 1), Futures.failedFuture(new StreamSegmentSealedException(streamSegmentName)));
    processor.append(new Append(streamSegmentName, clientId, 2, 1, Unpooled.wrappedBuffer(data2), null, requestId));
    verifyStoreAppend(ac2, data2);
    verify(tracker).updateOutstandingBytes(connection, data2.length, data1.length + data2.length);
    verify(tracker).updateOutstandingBytes(connection, -data2.length, data1.length);
    // Third append should fail just like the second one. However this will have a higher event number than that one
    // and we use it to verify it won't prevent sending the SegmentIsSealed message or send it multiple times.
    val ac3 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 3, 2, 1), Futures.failedFuture(new StreamSegmentSealedException(streamSegmentName)));
    processor.append(new Append(streamSegmentName, clientId, 3, 1, Unpooled.wrappedBuffer(data3), null, requestId));
    verifyStoreAppend(ac3, data3);
    verify(tracker).updateOutstandingBytes(connection, data3.length, data1.length + data3.length);
    // Complete the first one and verify it is acked properly.
    store1.complete(100L);
    connectionVerifier.verify(connection).send(new DataAppended(requestId, clientId, 1, 0L, 100L));
    // Verify that a SegmentIsSealed message is sent AFTER the ack from the first one (for the second append).
    connectionVerifier.verify(connection).send(new WireCommands.SegmentIsSealed(requestId, streamSegmentName, "", 2L));
    // Verify that a second SegmentIsSealed is sent (for the third append).
    connectionVerifier.verify(connection).send(new WireCommands.SegmentIsSealed(requestId, streamSegmentName, "", 3L));
    verify(tracker).updateOutstandingBytes(connection, -data3.length, data1.length);
    verify(tracker).updateOutstandingBytes(connection, -data1.length, 0);
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 4, 3, 1), Futures.failedFuture(new IntentionalException(streamSegmentName)));
    AssertExtensions.assertThrows("append() accepted a new request after sending a SegmentIsSealed message.", () -> processor.append(new Append(streamSegmentName, clientId, 4, 1, Unpooled.wrappedBuffer(data1), null, requestId)), ex -> ex instanceof IllegalStateException);
    // Verify no more messages are sent over the connection.
    connectionVerifier.verifyNoMoreInteractions();
    verifyNoMoreInteractions(store);
}

Example 50 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testAppendFails.

@Test
public void testAppendFails() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    val mockedRecorder = mock(SegmentStatsRecorder.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(new IntentionalException()));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    try {
        processor.append(new Append(streamSegmentName, clientId, data.length * 2, 1, Unpooled.wrappedBuffer(data), null, requestId));
        fail();
    } catch (IllegalStateException e) {
    // Expected
    }
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).close();
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verify(store, atMost(1)).append(any(), any(), any(), any());
    verifyNoMoreInteractions(connection);
    verify(mockedRecorder, never()).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
}