Examples with TableEntry io.pravega.segmentstore.contracts.tables.TableEntry used on opensource projects

Example 31 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testRemoveKeys.

@Test(timeout = 30000)
public void testRemoveKeys() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testRemoveEntries";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection, tracker), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    TableEntry e1 = TableEntry.unversioned(keys.get(0), generateValue(rnd));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(singletonList(e1)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableEntriesUpdated(2, singletonList(0L)));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(1), eq(false), any());
    verifyConnectionTracker(e1, connection, tracker);
    // Remove a Table Key
    WireCommands.TableKey key = new WireCommands.TableKey(toByteBuf(e1.getKey().getKey()), 0L);
    processor.removeTableKeys(new WireCommands.RemoveTableKeys(3, tableSegmentName, "", singletonList(key), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableKeysRemoved(3, tableSegmentName));
    verify(recorderMock).removeKeys(eq(tableSegmentName), eq(1), eq(true), any());
    verifyConnectionTracker(key, connection, tracker);
    // Test with non-existent key.
    key = new WireCommands.TableKey(toByteBuf(e1.getKey().getKey()), 0L);
    processor.removeTableKeys(new WireCommands.RemoveTableKeys(4, tableSegmentName, "", singletonList(key), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableKeyBadVersion(4, tableSegmentName, ""));
    verifyConnectionTracker(key, connection, tracker);
    long segmentLength = store.getStreamSegmentInfo(tableSegmentName, Duration.ofMinutes(1)).get().getLength();
    // Test when providing a matching tableSegmentOffset.
    TableEntry e2 = TableEntry.unversioned(keys.get(1), generateValue(rnd));
    key = new WireCommands.TableKey(toByteBuf(e2.getKey().getKey()), e2.getKey().getVersion());
    processor.removeTableKeys(new WireCommands.RemoveTableKeys(5, tableSegmentName, "", singletonList(key), segmentLength));
    order.verify(connection).send(new WireCommands.TableKeysRemoved(5, tableSegmentName));
    verify(recorderMock).removeKeys(eq(tableSegmentName), eq(1), eq(true), any());
    verifyConnectionTracker(key, connection, tracker);
    // // Test when providing a mismatching tableSegmentOffset.
    long badOffset = segmentLength - 1;
    TableEntry e3 = TableEntry.unversioned(keys.get(2), generateValue(rnd));
    key = new WireCommands.TableKey(toByteBuf(e3.getKey().getKey()), e3.getKey().getVersion());
    processor.removeTableKeys(new WireCommands.RemoveTableKeys(6, tableSegmentName, "", singletonList(key), badOffset));
    segmentLength = store.getStreamSegmentInfo(tableSegmentName, Duration.ofMinutes(1)).get().getLength();
    // BadOffsetException should be thrown, prompting a SegmentIsTruncated response.
    order.verify(connection).send(new WireCommands.SegmentIsTruncated(6, tableSegmentName, segmentLength, "", badOffset));
    verify(recorderMock).removeKeys(eq(tableSegmentName), eq(1), eq(true), any());
    verifyConnectionTracker(key, connection, tracker);
}

Example 32 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadTableEntriesDelta.

@Test
public void testReadTableEntriesDelta() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testReadTableEntriesDelta";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    ArrayView testValue = generateValue(rnd);
    TableEntry e1 = TableEntry.unversioned(keys.get(0), testValue);
    TableEntry e2 = TableEntry.unversioned(keys.get(1), testValue);
    TableEntry e3 = TableEntry.unversioned(keys.get(2), testValue);
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(asList(e1, e2, e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(3), eq(false), any());
    // 1. Now read the table entries where suggestedEntryCount is equal to number of entries in the Table Store.
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(3, tableSegmentName, "", 0, 3));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommand> wireCommandsCaptor = ArgumentCaptor.forClass(WireCommand.class);
    order.verify(connection, times(2)).send(wireCommandsCaptor.capture());
    verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(3), any());
    // Verify the WireCommands.
    List<Long> keyVersions = ((WireCommands.TableEntriesUpdated) wireCommandsCaptor.getAllValues().get(0)).getUpdatedVersions();
    WireCommands.TableEntriesDeltaRead getTableEntriesIteratorsResp = (WireCommands.TableEntriesDeltaRead) wireCommandsCaptor.getAllValues().get(1);
    assertTrue(getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> e.getKey().getKeyVersion()).collect(Collectors.toList()).containsAll(keyVersions));
    // Verify if the value is correct.
    assertTrue(getTableEntriesIteratorsResp.getEntries().getEntries().stream().allMatch(e -> {
        ByteBuf buf = e.getValue().getData();
        byte[] bytes = new byte[buf.readableBytes()];
        buf.getBytes(buf.readerIndex(), bytes);
        return testValue.equals(new ByteArraySegment(bytes));
    }));
    // 2. Now read the table entries where suggestedEntryCount is less than the number of entries in the Table Store.
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(3, tableSegmentName, "", 0L, 1));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommands.TableEntriesDeltaRead> tableEntriesCaptor = ArgumentCaptor.forClass(WireCommands.TableEntriesDeltaRead.class);
    order.verify(connection, times(1)).send(tableEntriesCaptor.capture());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(0);
    assertEquals(1, getTableEntriesIteratorsResp.getEntries().getEntries().size());
    assertTrue(keyVersions.contains(getTableEntriesIteratorsResp.getEntries().getEntries().get(0).getKey().getKeyVersion()));
    assertFalse(getTableEntriesIteratorsResp.isShouldClear());
    assertFalse(getTableEntriesIteratorsResp.isReachedEnd());
    // Get the last position.
    long lastPosition = getTableEntriesIteratorsResp.getLastPosition();
    // 3. Now read the remaining table entries by providing a higher suggestedKeyCount and the state to the iterator.
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(3, tableSegmentName, "", lastPosition, 3));
    // Capture the WireCommands sent.
    order.verify(connection, times(1)).send(tableEntriesCaptor.capture());
    verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(1), any());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(1);
    // We read through all the entries, so this should report as having reached the end.
    assertTrue(getTableEntriesIteratorsResp.isReachedEnd());
    assertEquals(2, getTableEntriesIteratorsResp.getEntries().getEntries().size());
    assertTrue(keyVersions.containsAll(getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> e.getKey().getKeyVersion()).collect(Collectors.toList())));
    // 4. Update some TableEntry.
    TableEntry e4 = TableEntry.versioned(keys.get(0), generateValue(rnd), keyVersions.get(0));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(4, tableSegmentName, "", getTableEntries(asList(e4)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(1), eq(true), any());
    // 5. Remove some TableEntry.
    TableEntry e5 = TableEntry.versioned(keys.get(1), generateValue(rnd), keyVersions.get(1));
    WireCommands.TableKey key = new WireCommands.TableKey(toByteBuf(e5.getKey().getKey()), e5.getKey().getVersion());
    processor.removeTableKeys(new WireCommands.RemoveTableKeys(5, tableSegmentName, "", singletonList(key), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableKeysRemoved(5, tableSegmentName));
    verify(recorderMock).removeKeys(eq(tableSegmentName), eq(1), eq(true), any());
    // // Now very the delta iteration returns a list with an updated key (4.) and a removed key (5.).
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(6, tableSegmentName, "", 0L, 4));
    // verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(3), any());
    // Capture the WireCommands sent.
    order.verify(connection).send(tableEntriesCaptor.capture());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(2);
    val results = getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> TableEntry.versioned(new ByteBufWrapper(e.getKey().getData()), new ByteBufWrapper(e.getValue().getData()), e.getKey().getKeyVersion())).collect(Collectors.toList());
    assertEquals("Expecting 2 entries left in the TableSegment", 2, results.size());
    // Does not container entry removed.
    assertFalse(results.contains(e5));
}

Example 33 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadTable.

@Test(timeout = 30000)
public void testReadTable() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testReadTable";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    val recorderMockOrder = inOrder(recorderMock);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(2, rnd);
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    recorderMockOrder.verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    TableEntry entry = TableEntry.unversioned(keys.get(0), generateValue(rnd));
    // Read value of a non-existent key.
    WireCommands.TableKey key = new WireCommands.TableKey(toByteBuf(entry.getKey().getKey()), TableKey.NO_VERSION);
    processor.readTable(new WireCommands.ReadTable(2, tableSegmentName, "", singletonList(key)));
    // expected result is Key (with key with version as NOT_EXISTS) and an empty TableValue.)
    WireCommands.TableKey keyResponse = new WireCommands.TableKey(toByteBuf(entry.getKey().getKey()), WireCommands.TableKey.NOT_EXISTS);
    order.verify(connection).send(new WireCommands.TableRead(2, tableSegmentName, new WireCommands.TableEntries(singletonList(new AbstractMap.SimpleImmutableEntry<>(keyResponse, WireCommands.TableValue.EMPTY)))));
    recorderMockOrder.verify(recorderMock).getKeys(eq(tableSegmentName), eq(1), any());
    // Update a value to a key.
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(3, tableSegmentName, "", getTableEntries(singletonList(entry)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableEntriesUpdated(3, singletonList(0L)));
    recorderMockOrder.verify(recorderMock).updateEntries(eq(tableSegmentName), eq(1), eq(false), any());
    // Read the value of the key.
    key = new WireCommands.TableKey(toByteBuf(entry.getKey().getKey()), 0L);
    TableEntry expectedEntry = TableEntry.versioned(entry.getKey().getKey(), entry.getValue(), 0L);
    processor.readTable(new WireCommands.ReadTable(4, tableSegmentName, "", singletonList(key)));
    order.verify(connection).send(new WireCommands.TableRead(4, tableSegmentName, getTableEntries(singletonList(expectedEntry))));
    recorderMockOrder.verify(recorderMock).getKeys(eq(tableSegmentName), eq(1), any());
}

Example 34 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testDeleteNonEmptyTable.

@Test(timeout = 30000)
public void testDeleteNonEmptyTable() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testTable";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(2, rnd);
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(3, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(3, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    TableEntry e1 = TableEntry.unversioned(keys.get(0), generateValue(rnd));
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(4, tableSegmentName, "", getTableEntries(singletonList(e1)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    order.verify(connection).send(new WireCommands.TableEntriesUpdated(4, singletonList(0L)));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(1), eq(false), any());
    // Delete a table segment which has data.
    processor.deleteTableSegment(new WireCommands.DeleteTableSegment(5, tableSegmentName, true, ""));
    order.verify(connection).send(new WireCommands.TableSegmentNotEmpty(5, tableSegmentName, ""));
    verifyNoMoreInteractions(recorderMock);
}

Example 35 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadTableEntriesDeltaOutOfBounds.

@Test
public void testReadTableEntriesDeltaOutOfBounds() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testReadTableEntriesDelta";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    ArrayView testValue = generateValue(rnd);
    TableEntry e1 = TableEntry.unversioned(keys.get(0), testValue);
    TableEntry e2 = TableEntry.unversioned(keys.get(1), testValue);
    TableEntry e3 = TableEntry.unversioned(keys.get(2), testValue);
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(asList(e1, e2, e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(3), eq(false), any());
    long length = store.getStreamSegmentInfo(tableSegmentName, Duration.ofMinutes(1)).get().getLength();
    processor.readTableEntriesDelta(new WireCommands.ReadTableEntriesDelta(1, tableSegmentName, "", length + 1, 3));
    order.verify(connection).send(new WireCommands.ErrorMessage(1, tableSegmentName, "fromPosition (652) can not exceed the length (651) of the TableSegment.", WireCommands.ErrorMessage.ErrorCode.valueOf(IllegalArgumentException.class)));
    verify(recorderMock, times(0)).iterateEntries(eq(tableSegmentName), eq(3), any());
}