Examples with Append io.pravega.shared.protocol.netty.Append used on opensource projects

Example 46 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class ClientConnectionTest method tcpClientConnectionBatchSizeTrackerUniquenessTest.

/**
 * This test verifies that the number of entries in the {@link FlowToBatchSizeTracker} object of a connection is
 * driven by the number of flows, not by the different ids from the requests.
 *
 * @throws Exception
 */
@Test
public void tcpClientConnectionBatchSizeTrackerUniquenessTest() throws Exception {
    byte[] payload = new byte[100];
    ClientConnectionTest.ReplyProcessor processor = new ClientConnectionTest.ReplyProcessor();
    @Cleanup MockServer server = new MockServer();
    server.start();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    @Cleanup TcpClientConnection clientConnection = TcpClientConnection.connect(server.getUri(), ClientConfig.builder().build(), processor, executor, null).join();
    UUID writerId = new UUID(1, 1);
    clientConnection.send(new WireCommands.SetupAppend(1, writerId, "segment", ""));
    // clear setup.
    server.getReadCommands().take();
    // Check that the map is initially empty.
    Assert.assertEquals(0, clientConnection.getConnectionReaderFlowToBatchSizeTracker().getFlowToBatchSizeTrackerMap().size());
    // Create one flow and increase its sequence number on each append.
    Flow firstFlow = Flow.create();
    for (int i = 0; i < 100; i++) {
        clientConnection.send(new Append("segment", writerId, i, new Event(Unpooled.wrappedBuffer(payload)), firstFlow.asLong()));
        // Assert that the map always contains one entry, as there is only one flow (with increasing sequence number).
        Assert.assertEquals(1, clientConnection.getConnectionReaderFlowToBatchSizeTracker().getFlowToBatchSizeTrackerMap().size());
        firstFlow.getNextSequenceNumber();
    }
    // Start another writer on the same connection.
    UUID writerId2 = new UUID(1, 2);
    clientConnection.send(new WireCommands.SetupAppend(1, writerId2, "segment2", ""));
    // Create another flow.
    Flow secondFlow = Flow.create();
    // Check that with another flow appending, there are just 2 flows in the batch size tracker map.
    clientConnection.send(new Append("segment2", writerId2, 0, new Event(Unpooled.wrappedBuffer(payload)), secondFlow.getNextSequenceNumber()));
    Assert.assertEquals(2, clientConnection.getConnectionReaderFlowToBatchSizeTracker().getFlowToBatchSizeTrackerMap().size());
    clientConnection.send(new Append("segment2", writerId2, 1, new Event(Unpooled.wrappedBuffer(payload)), secondFlow.getNextSequenceNumber()));
    Assert.assertEquals(2, clientConnection.getConnectionReaderFlowToBatchSizeTracker().getFlowToBatchSizeTrackerMap().size());
}

Example 47 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class CommandEncoderTest method testBatchTimeout.

@Test
public void testBatchTimeout() throws IOException {
    AppendBatchSizeTracker batchSizeTracker = new FixedBatchSizeTracker(100);
    DecodingOutputStream output = new DecodingOutputStream();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, null, endpoint);
    UUID writerId = UUID.randomUUID();
    WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append1 = new Append("seg", writerId, 1, event, 0);
    commandEncoder.write(append1);
    long l = commandEncoder.batchTimeout(0);
    // Triggers a timeout
    commandEncoder.batchTimeout(l);
    AppendBlock block = (AppendBlock) output.decoded.remove(0);
    assertEquals(108, block.getData().readableBytes());
    AppendBlockEnd blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId, blockEnd.getWriterId());
    assertEquals(48, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(1, blockEnd.getNumEvents());
    assertTrue(output.decoded.isEmpty());
}

Example 48 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class CommandEncoderTest method testShutdown.

@Test
public void testShutdown() throws IOException {
    AppendBatchSizeTrackerImpl batchSizeTracker = new AppendBatchSizeTrackerImpl();
    DecodingOutputStream output = new DecodingOutputStream();
    AtomicInteger counter = new AtomicInteger(0);
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, new FailingReplyProcessor() {

        @Override
        public void connectionDropped() {
            counter.getAndAdd(1);
        }

        @Override
        public void processingFailure(Exception error) {
        }
    }, endpoint);
    // maximum setup requests
    for (int i = 0; i < CommandEncoder.MAX_SETUP_SEGMENTS_SIZE; i++) {
        UUID writerId = UUID.randomUUID();
        WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId, "seg", "");
        commandEncoder.write(setupAppend);
    }
    // further setup request should throw IOException
    UUID writerId = UUID.randomUUID();
    final WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId, "seg", "");
    assertThrows(IOException.class, () -> commandEncoder.write(setupAppend));
    // then connection is closed, ObjectClosedException should be thrown
    writerId = UUID.randomUUID();
    final WireCommand setupAppend2 = new WireCommands.SetupAppend(0, writerId, "seg", "");
    assertThrows(ObjectClosedException.class, () -> commandEncoder.write(setupAppend2));
    writerId = UUID.randomUUID();
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append = new Append("", writerId, 1, event, 1);
    assertThrows(ObjectClosedException.class, () -> commandEncoder.write(append));
    assertEquals(counter.get(), 1);
}

Example 49 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class CommandEncoderTest method testAppendsQueued.

@Test
public void testAppendsQueued() throws IOException {
    AppendBatchSizeTracker batchSizeTracker = new FixedBatchSizeTracker(100);
    DecodingOutputStream output = new DecodingOutputStream();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, null, endpoint);
    UUID writerId1 = UUID.randomUUID();
    WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId1, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    UUID writerId2 = UUID.randomUUID();
    setupAppend = new WireCommands.SetupAppend(0, writerId2, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append1 = new Append("seg", writerId1, 1, event, 0);
    commandEncoder.write(append1);
    Append appendOther = new Append("seg", writerId2, 100, event, 0);
    commandEncoder.write(appendOther);
    Append append2 = new Append("seg", writerId1, 2, event, 0);
    Append append3 = new Append("seg", writerId1, 3, event, 0);
    commandEncoder.write(append2);
    commandEncoder.write(append3);
    AppendBlock block = (AppendBlock) output.decoded.remove(0);
    assertEquals(108, block.getData().readableBytes());
    AppendBlockEnd blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId1, blockEnd.getWriterId());
    assertEquals(80 + 16, blockEnd.getSizeOfWholeEvents());
    assertEquals(40 + 4, blockEnd.getData().readableBytes());
    assertEquals(3, blockEnd.getNumEvents());
    block = (AppendBlock) output.decoded.remove(0);
    assertEquals(48, block.getData().readableBytes());
    blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId2, blockEnd.getWriterId());
    assertEquals(48, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(1, blockEnd.getNumEvents());
}

Example 50 with Append

use of io.pravega.shared.protocol.netty.Append in project pravega by pravega.

the class CommandEncoderTest method testAppendsQueuedBreak.

@Test
public void testAppendsQueuedBreak() throws IOException {
    AppendBatchSizeTracker batchSizeTracker = new FixedBatchSizeTracker(100);
    DecodingOutputStream output = new DecodingOutputStream();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, null, endpoint);
    UUID writerId1 = UUID.randomUUID();
    WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId1, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    UUID writerId2 = UUID.randomUUID();
    setupAppend = new WireCommands.SetupAppend(0, writerId2, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append1 = new Append("seg", writerId1, 1, event, 0);
    commandEncoder.write(append1);
    Append appendOther1 = new Append("seg", writerId2, 101, event, 0);
    Append appendOther2 = new Append("seg", writerId2, 102, event, 0);
    Append appendOther3 = new Append("seg", writerId2, 103, event, 0);
    commandEncoder.write(appendOther1);
    commandEncoder.write(appendOther2);
    commandEncoder.write(appendOther3);
    commandEncoder.write(new WireCommands.KeepAlive());
    AppendBlock block = (AppendBlock) output.decoded.remove(0);
    assertEquals(108, block.getData().readableBytes());
    AppendBlockEnd blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId1, blockEnd.getWriterId());
    assertEquals(48, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(1, blockEnd.getNumEvents());
    block = (AppendBlock) output.decoded.remove(0);
    assertEquals(48 * 3, block.getData().readableBytes());
    blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId2, blockEnd.getWriterId());
    assertEquals(48 * 3, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(3, blockEnd.getNumEvents());
}