Examples with WireCommand io.pravega.shared.protocol.netty.WireCommand used on opensource projects

Example 16 with WireCommand

use of io.pravega.shared.protocol.netty.WireCommand in project pravega by pravega.

the class PravegaRequestProcessorTest method testUpdateEntries.

@Test(timeout = 20000)
public void testUpdateEntries() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testUpdateEntries";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection, tracker), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    // Execute and Verify createSegment calling stack is executed as design.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    // Test with unversioned data.
    TableEntry e1 = TableEntry.unversioned(keys.get(0), generateValue(rnd));
    WireCommands.TableEntries cmd = getTableEntries(singletonList(e1));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", cmd, WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableEntriesUpdated(2, singletonList(0L)));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(1), eq(false), any());
    verifyConnectionTracker(e1, connection, tracker);
    // Test with versioned data.
    e1 = TableEntry.versioned(keys.get(0), generateValue(rnd), 0L);
    cmd = getTableEntries(singletonList(e1));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(3, tableSegmentName, "", cmd, WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    ArgumentCaptor<WireCommand> wireCommandsCaptor = ArgumentCaptor.forClass(WireCommand.class);
    order.verify(connection).send(wireCommandsCaptor.capture());
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(1), eq(true), any());
    verifyConnectionTracker(e1, connection, tracker);
    List<Long> keyVersions = ((WireCommands.TableEntriesUpdated) wireCommandsCaptor.getAllValues().get(0)).getUpdatedVersions();
    assertTrue(keyVersions.size() == 1);
    // Test with key not present. The table store throws KeyNotExistsException.
    TableEntry e2 = TableEntry.versioned(keys.get(1), generateValue(rnd), 0L);
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(4, tableSegmentName, "", getTableEntries(singletonList(e2)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableKeyDoesNotExist(4, tableSegmentName, ""));
    verifyNoMoreInteractions(recorderMock);
    verifyConnectionTracker(e2, connection, tracker);
    // Test with invalid key version. The table store throws BadKeyVersionException.
    TableEntry e3 = TableEntry.versioned(keys.get(0), generateValue(rnd), 10L);
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(5, tableSegmentName, "", getTableEntries(singletonList(e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableKeyBadVersion(5, tableSegmentName, ""));
    verifyNoMoreInteractions(recorderMock);
    verifyConnectionTracker(e3, connection, tracker);
    // Test with valid tableSegmentOffset.
    long tableSegmentOffset = store.getStreamSegmentInfo(tableSegmentName, Duration.ofMinutes(1)).get().getLength();
    TableEntry e4 = TableEntry.versioned(keys.get(0), generateValue(rnd), keyVersions.get(0));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(6, tableSegmentName, "", getTableEntries(singletonList(e4)), tableSegmentOffset));
    order.verify(connection).send(wireCommandsCaptor.capture());
    verify(recorderMock, times(2)).updateEntries(eq(tableSegmentName), eq(1), eq(true), any());
    verifyConnectionTracker(e4, connection, tracker);
    keyVersions = ((WireCommands.TableEntriesUpdated) wireCommandsCaptor.getAllValues().get(1)).getUpdatedVersions();
    assertTrue(keyVersions.size() == 1);
    // Test with invalid tableSegmentOffset.
    TableEntry e5 = TableEntry.versioned(keys.get(0), generateValue(rnd), keyVersions.get(0));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(7, tableSegmentName, "", getTableEntries(singletonList(e5)), tableSegmentOffset - 1));
    long length = store.getStreamSegmentInfo(tableSegmentName, Duration.ofMinutes(1)).get().getLength();
    order.verify(connection).send(new WireCommands.SegmentIsTruncated(7, tableSegmentName, length, "", tableSegmentOffset - 1));
    verify(recorderMock, times(2)).updateEntries(eq(tableSegmentName), eq(1), eq(true), any());
    verifyConnectionTracker(e5, connection, tracker);
}

Example 17 with WireCommand

use of io.pravega.shared.protocol.netty.WireCommand in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadTableEntriesDelta.

@Test
public void testReadTableEntriesDelta() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testReadTableEntriesDelta";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    ArrayView testValue = generateValue(rnd);
    TableEntry e1 = TableEntry.unversioned(keys.get(0), testValue);
    TableEntry e2 = TableEntry.unversioned(keys.get(1), testValue);
    TableEntry e3 = TableEntry.unversioned(keys.get(2), testValue);
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(asList(e1, e2, e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(3), eq(false), any());
    // 1. Now read the table entries where suggestedEntryCount is equal to number of entries in the Table Store.
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(3, tableSegmentName, "", 0, 3));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommand> wireCommandsCaptor = ArgumentCaptor.forClass(WireCommand.class);
    order.verify(connection, times(2)).send(wireCommandsCaptor.capture());
    verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(3), any());
    // Verify the WireCommands.
    List<Long> keyVersions = ((WireCommands.TableEntriesUpdated) wireCommandsCaptor.getAllValues().get(0)).getUpdatedVersions();
    WireCommands.TableEntriesDeltaRead getTableEntriesIteratorsResp = (WireCommands.TableEntriesDeltaRead) wireCommandsCaptor.getAllValues().get(1);
    assertTrue(getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> e.getKey().getKeyVersion()).collect(Collectors.toList()).containsAll(keyVersions));
    // Verify if the value is correct.
    assertTrue(getTableEntriesIteratorsResp.getEntries().getEntries().stream().allMatch(e -> {
        ByteBuf buf = e.getValue().getData();
        byte[] bytes = new byte[buf.readableBytes()];
        buf.getBytes(buf.readerIndex(), bytes);
        return testValue.equals(new ByteArraySegment(bytes));
    }));
    // 2. Now read the table entries where suggestedEntryCount is less than the number of entries in the Table Store.
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(3, tableSegmentName, "", 0L, 1));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommands.TableEntriesDeltaRead> tableEntriesCaptor = ArgumentCaptor.forClass(WireCommands.TableEntriesDeltaRead.class);
    order.verify(connection, times(1)).send(tableEntriesCaptor.capture());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(0);
    assertEquals(1, getTableEntriesIteratorsResp.getEntries().getEntries().size());
    assertTrue(keyVersions.contains(getTableEntriesIteratorsResp.getEntries().getEntries().get(0).getKey().getKeyVersion()));
    assertFalse(getTableEntriesIteratorsResp.isShouldClear());
    assertFalse(getTableEntriesIteratorsResp.isReachedEnd());
    // Get the last position.
    long lastPosition = getTableEntriesIteratorsResp.getLastPosition();
    // 3. Now read the remaining table entries by providing a higher suggestedKeyCount and the state to the iterator.
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(3, tableSegmentName, "", lastPosition, 3));
    // Capture the WireCommands sent.
    order.verify(connection, times(1)).send(tableEntriesCaptor.capture());
    verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(1), any());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(1);
    // We read through all the entries, so this should report as having reached the end.
    assertTrue(getTableEntriesIteratorsResp.isReachedEnd());
    assertEquals(2, getTableEntriesIteratorsResp.getEntries().getEntries().size());
    assertTrue(keyVersions.containsAll(getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> e.getKey().getKeyVersion()).collect(Collectors.toList())));
    // 4. Update some TableEntry.
    TableEntry e4 = TableEntry.versioned(keys.get(0), generateValue(rnd), keyVersions.get(0));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(4, tableSegmentName, "", getTableEntries(asList(e4)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(1), eq(true), any());
    // 5. Remove some TableEntry.
    TableEntry e5 = TableEntry.versioned(keys.get(1), generateValue(rnd), keyVersions.get(1));
    WireCommands.TableKey key = new WireCommands.TableKey(toByteBuf(e5.getKey().getKey()), e5.getKey().getVersion());
    processor.removeTableKeys(new WireCommands.RemoveTableKeys(5, tableSegmentName, "", singletonList(key), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableKeysRemoved(5, tableSegmentName));
    verify(recorderMock).removeKeys(eq(tableSegmentName), eq(1), eq(true), any());
    // // Now very the delta iteration returns a list with an updated key (4.) and a removed key (5.).
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(6, tableSegmentName, "", 0L, 4));
    // verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(3), any());
    // Capture the WireCommands sent.
    order.verify(connection).send(tableEntriesCaptor.capture());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(2);
    val results = getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> TableEntry.versioned(new ByteBufWrapper(e.getKey().getData()), new ByteBufWrapper(e.getValue().getData()), e.getKey().getKeyVersion())).collect(Collectors.toList());
    assertEquals("Expecting 2 entries left in the TableSegment", 2, results.size());
    // Does not container entry removed.
    assertFalse(results.contains(e5));
}

Example 18 with WireCommand

use of io.pravega.shared.protocol.netty.WireCommand in project pravega by pravega.

the class AppendProcessorTest method testBadAttributeException.

@Test(timeout = 5000)
public void testBadAttributeException() throws Exception {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    // Setup mocks.
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    setupGetAttributes(streamSegmentName, clientId, store);
    val ex = new BadAttributeUpdateException(streamSegmentName, new AttributeUpdate(AttributeId.randomUUID(), AttributeUpdateType.ReplaceIfEquals, 100, 101), false, "error");
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(ex));
    @Cleanup EmbeddedChannel channel = createChannel(store);
    // Send a setup append WireCommand.
    Reply reply = sendRequest(channel, new SetupAppend(requestId, clientId, streamSegmentName, ""));
    assertEquals(new AppendSetup(requestId, streamSegmentName, clientId, 0), reply);
    // Send an append which will cause a RuntimeException to be thrown by the store.
    reply = sendRequest(channel, new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    // validate InvalidEventNumber Wirecommand is sent before closing the Channel.
    assertNotNull("Invalid Event WireCommand is expected", reply);
    assertEquals(WireCommandType.INVALID_EVENT_NUMBER.getCode(), ((WireCommand) reply).getType().getCode());
    assertEquals(requestId, ((InvalidEventNumber) reply).getRequestId());
    // Verify that the channel is closed by the AppendProcessor.
    assertEventuallyEquals(false, channel::isOpen, 3000);
}

Example 19 with WireCommand

use of io.pravega.shared.protocol.netty.WireCommand in project pravega by pravega.

the class MockServer method listen.

private void listen() {
    try {
        @Cleanup ServerSocket ss = new ServerSocket(port);
        started.release();
        @Cleanup Socket s = ss.accept();
        outputStream.complete(s.getOutputStream());
        @Cleanup InputStream stream = s.getInputStream();
        IoBuffer buffer = new IoBuffer();
        while (!stop.get()) {
            WireCommand command = TcpClientConnection.ConnectionReader.readCommand(stream, buffer);
            readCommands.add(command);
        }
    } catch (IOException e) {
        stop.set(true);
    }
}

Example 20 with WireCommand

use of io.pravega.shared.protocol.netty.WireCommand in project pravega by pravega.

the class CommandEncoderTest method testBatchTimeout.

@Test
public void testBatchTimeout() throws IOException {
    AppendBatchSizeTracker batchSizeTracker = new FixedBatchSizeTracker(100);
    DecodingOutputStream output = new DecodingOutputStream();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, null, endpoint);
    UUID writerId = UUID.randomUUID();
    WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append1 = new Append("seg", writerId, 1, event, 0);
    commandEncoder.write(append1);
    long l = commandEncoder.batchTimeout(0);
    // Triggers a timeout
    commandEncoder.batchTimeout(l);
    AppendBlock block = (AppendBlock) output.decoded.remove(0);
    assertEquals(108, block.getData().readableBytes());
    AppendBlockEnd blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId, blockEnd.getWriterId());
    assertEquals(48, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(1, blockEnd.getNumEvents());
    assertTrue(output.decoded.isEmpty());
}