Examples with Append io.pravega.shared.protocol.netty.Append used on opensource projects

Example 31 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testOutstandingByteTracking.

/**
 * Simulates multiple connections being set up and all sending appends (conditional or unconditional). Some may be
 * failed by the store. Verifies that {@link ConnectionTracker#getTotalOutstanding()} does not drift with time,
 * regardless of append outcome.
 */
@Test
public void testOutstandingByteTracking() throws Exception {
    final int connectionCount = 5;
    final int writersCount = 5;
    final int segmentCount = 5;
    val tracker = new ConnectionTracker();
    val context = mock(ChannelHandlerContext.class);
    val channel = mock(Channel.class);
    val channelConfig = mock(ChannelConfig.class);
    when(context.channel()).thenReturn(channel);
    when(channel.config()).thenReturn(channelConfig);
    val eventLoop = mock(EventLoop.class);
    when(channel.eventLoop()).thenReturn(eventLoop);
    when(eventLoop.inEventLoop()).thenReturn(false);
    val channelFuture = mock(ChannelFuture.class);
    when(channel.writeAndFlush(any())).thenReturn(channelFuture);
    val segments = IntStream.range(0, segmentCount).mapToObj(Integer::toString).collect(Collectors.toList());
    val writers = IntStream.range(0, writersCount).mapToObj(i -> UUID.randomUUID()).collect(Collectors.toList());
    val store = mock(StreamSegmentStore.class);
    val processors = new ArrayList<AppendProcessor>();
    for (int i = 0; i < connectionCount; i++) {
        val h = new ServerConnectionInboundHandler();
        h.channelRegistered(context);
        val p = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(h, tracker)).build();
        processors.add(p);
    }
    // Setup appends.
    for (int connectionId = 0; connectionId < processors.size(); connectionId++) {
        for (val s : segments) {
            for (val w : writers) {
                when(store.getAttributes(s, Collections.singleton(AttributeId.fromUUID(w)), true, AppendProcessor.TIMEOUT)).thenReturn(CompletableFuture.completedFuture(Collections.singletonMap(AttributeId.fromUUID(w), 0L)));
                processors.get(connectionId).setupAppend(new WireCommands.SetupAppend(0, w, s, null));
            }
        }
    }
    // Divide the segments into conditional and unconditional.
    val conditionalSegments = segments.subList(0, segments.size() / 2);
    val unconditionalSegments = segments.subList(conditionalSegments.size(), segments.size() - 1);
    // Send a few appends to each connection from each writer.
    val appendData = Unpooled.wrappedBuffer(new byte[1]);
    when(store.append(any(), any(), any(), any())).thenReturn(delayedResponse(0L));
    for (val s : unconditionalSegments) {
        for (val p : processors) {
            for (val w : writers) {
                p.append(new Append(s, w, 1, new WireCommands.Event(appendData.retain()), 0));
            }
        }
    }
    // Send a few conditional appends to each connection from each writer. Fail some along the way.
    int appendOffset = 0;
    for (val s : conditionalSegments) {
        for (val p : processors) {
            for (val w : writers) {
                boolean fail = appendOffset % 3 == 0;
                if (fail) {
                    when(store.append(any(), any(long.class), any(), any(), any())).thenReturn(delayedFailure(new BadOffsetException(s, appendOffset, appendOffset)));
                } else {
                    when(store.append(any(), any(long.class), any(), any(), any())).thenReturn(delayedResponse(0L));
                }
                p.append(new Append(s, w, 1, new WireCommands.Event(appendData.retain()), appendOffset, 0));
                appendOffset++;
            }
        }
    }
    // Fail (attributes) all connections.
    when(store.append(any(), any(), any(), any())).thenReturn(delayedFailure(new BadAttributeUpdateException("s", null, false, "intentional")));
    for (val s : conditionalSegments) {
        for (val p : processors) {
            for (val w : writers) {
                p.append(new Append(s, w, 1, new WireCommands.Event(appendData.retain()), 0));
            }
        }
    }
    // Verify that there is no drift in the ConnectionTracker#getTotalOutstanding value. Due to the async nature
    // of the calls, this value may not immediately be updated.
    AssertExtensions.assertEventuallyEquals(0L, tracker::getTotalOutstanding, 10000);
}

Example 32 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testTransactionAppend.

@Test
public void testTransactionAppend() {
    String streamSegmentName = "scope/stream/transactionSegment#transaction.01234567890123456789012345678901";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    val mockedRecorder = Mockito.mock(SegmentStatsRecorder.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    val ac = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), CompletableFuture.completedFuture(21L));
    processor.setupAppend(new SetupAppend(requestId, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac, data);
    verify(connection).send(new AppendSetup(requestId, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new DataAppended(requestId, clientId, data.length, 0L, 21L));
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
    verify(mockedRecorder).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
}

Example 33 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testAppend.

@Test
public void testAppend() {
    String streamSegmentName = "scope/stream/0.#epoch.0";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    val mockedRecorder = Mockito.mock(SegmentStatsRecorder.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    val ac = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), CompletableFuture.completedFuture((long) data.length));
    SetupAppend setupAppendCommand = new SetupAppend(1, clientId, streamSegmentName, "");
    processor.setupAppend(setupAppendCommand);
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new DataAppended(requestId, clientId, data.length, 0L, data.length));
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
    verify(mockedRecorder).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
    assertTrue(processor.isSetupAppendCompleted(setupAppendCommand.getSegment(), setupAppendCommand.getWriterId()));
}

Example 34 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testConditionalAppendFailureOnUnconditionalAppend.

@Test
public void testConditionalAppendFailureOnUnconditionalAppend() {
    String streamSegmentName = "scope/stream/testConditionalAppendFailureOnUnconditionalAppend";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    val mockedRecorder = Mockito.mock(SegmentStatsRecorder.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    val ac1 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 1), CompletableFuture.completedFuture((long) data.length));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data), null, requestId));
    val ac2 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 1, 1, 1), Futures.failedFuture(new BadOffsetException(streamSegmentName, data.length, 0)));
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data), null, requestId));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac1, data);
    verifyStoreAppend(ac2, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker, times(2)).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new DataAppended(requestId, clientId, 1, 0, data.length));
    verify(connection).close();
    verify(tracker, times(2)).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 35 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class AppendProcessorTest method testSwitchingStream.

@Test
public void testSwitchingStream() {
    String segment1 = "scope/stream/segment1";
    String segment2 = "scope/stream/segment2";
    UUID clientId1 = UUID.randomUUID();
    UUID clientId2 = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    setupGetAttributes(segment1, clientId1, store);
    val ac1 = interceptAppend(store, segment1, updateEventNumber(clientId1, data.length), CompletableFuture.completedFuture((long) data.length));
    setupGetAttributes(segment2, clientId2, store);
    val ac2 = interceptAppend(store, segment2, updateEventNumber(clientId2, data.length), CompletableFuture.completedFuture((long) data.length));
    processor.setupAppend(new SetupAppend(1, clientId1, segment1, ""));
    processor.append(new Append(segment1, clientId1, data.length, 1, Unpooled.wrappedBuffer(data), null, 2));
    processor.setupAppend(new SetupAppend(3, clientId2, segment2, ""));
    processor.append(new Append(segment2, clientId2, data.length, 1, Unpooled.wrappedBuffer(data), null, 4));
    verify(store).getAttributes(eq(segment1), eq(Collections.singleton(AttributeId.fromUUID(clientId1))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac1, data);
    verify(store).getAttributes(eq(segment2), eq(Collections.singleton(AttributeId.fromUUID(clientId2))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac2, data);
    verify(tracker, times(2)).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).send(new AppendSetup(1, segment1, clientId1, 0));
    verify(connection).send(new DataAppended(2, clientId1, data.length, 0, data.length));
    verify(connection).send(new AppendSetup(3, segment2, clientId2, 0));
    verify(connection).send(new DataAppended(4, clientId2, data.length, 0, data.length));
    verify(tracker, times(2)).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}