Examples with TableEntry io.pravega.segmentstore.contracts.tables.TableEntry used on opensource projects

Example 1 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testGetTableKeys.

@Test
public void testGetTableKeys() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testGetTableKeys";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    TableEntry e1 = TableEntry.unversioned(keys.get(0), generateValue(rnd));
    TableEntry e2 = TableEntry.unversioned(keys.get(1), generateValue(rnd));
    TableEntry e3 = TableEntry.unversioned(keys.get(2), generateValue(rnd));
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(asList(e1, e2, e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(3), eq(false), any());
    // 1. Now read the table keys where suggestedKeyCount is equal to number of entries in the Table Store.
    WireCommands.TableIteratorArgs args = new WireCommands.TableIteratorArgs(Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER);
    processor.readTableKeys(new WireCommands.ReadTableKeys(3, tableSegmentName, "", 3, args));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommand> wireCommandsCaptor = ArgumentCaptor.forClass(WireCommand.class);
    order.verify(connection, times(2)).send(wireCommandsCaptor.capture());
    verify(recorderMock).iterateKeys(eq(tableSegmentName), eq(3), any());
    // Verify the WireCommands.
    List<Long> keyVersions = ((WireCommands.TableEntriesUpdated) wireCommandsCaptor.getAllValues().get(0)).getUpdatedVersions();
    WireCommands.TableKeysRead getTableKeysReadResponse = (WireCommands.TableKeysRead) wireCommandsCaptor.getAllValues().get(1);
    assertTrue(getTableKeysReadResponse.getKeys().stream().map(WireCommands.TableKey::getKeyVersion).collect(Collectors.toList()).containsAll(keyVersions));
    // 2. Now read the table keys where suggestedKeyCount is less than the number of keys in the Table Store.
    processor.readTableKeys(new WireCommands.ReadTableKeys(3, tableSegmentName, "", 1, args));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommands.TableKeysRead> tableKeysCaptor = ArgumentCaptor.forClass(WireCommands.TableKeysRead.class);
    order.verify(connection, times(1)).send(tableKeysCaptor.capture());
    verify(recorderMock).iterateKeys(eq(tableSegmentName), eq(1), any());
    // Verify the WireCommands.
    getTableKeysReadResponse = tableKeysCaptor.getAllValues().get(0);
    assertEquals(1, getTableKeysReadResponse.getKeys().size());
    assertTrue(keyVersions.contains(getTableKeysReadResponse.getKeys().get(0).getKeyVersion()));
    // Get the last state.
    ByteBuf state = getTableKeysReadResponse.getContinuationToken();
    args = new WireCommands.TableIteratorArgs(state, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER);
    // 3. Now read the remaining table keys by providing a higher suggestedKeyCount and the state to the iterator.
    processor.readTableKeys(new WireCommands.ReadTableKeys(3, tableSegmentName, "", 3, args));
    // Capture the WireCommands sent.
    tableKeysCaptor = ArgumentCaptor.forClass(WireCommands.TableKeysRead.class);
    order.verify(connection, times(1)).send(tableKeysCaptor.capture());
    verify(recorderMock).iterateKeys(eq(tableSegmentName), eq(1), any());
    // Verify the WireCommands.
    getTableKeysReadResponse = tableKeysCaptor.getAllValues().get(0);
    assertEquals(2, getTableKeysReadResponse.getKeys().size());
    assertTrue(keyVersions.containsAll(getTableKeysReadResponse.getKeys().stream().map(WireCommands.TableKey::getKeyVersion).collect(Collectors.toList())));
}

Example 2 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessor method readTableEntriesDelta.

@Override
public void readTableEntriesDelta(WireCommands.ReadTableEntriesDelta readTableEntriesDelta) {
    final String segment = readTableEntriesDelta.getSegment();
    final String operation = "readTableEntriesDelta";
    if (!verifyToken(segment, readTableEntriesDelta.getRequestId(), readTableEntriesDelta.getDelegationToken(), operation)) {
        return;
    }
    final int suggestedEntryCount = readTableEntriesDelta.getSuggestedEntryCount();
    final long fromPosition = readTableEntriesDelta.getFromPosition();
    log.info(readTableEntriesDelta.getRequestId(), "Iterate Table Entries Delta: Segment={} Count={} FromPositon={}.", readTableEntriesDelta.getSegment(), readTableEntriesDelta.getSuggestedEntryCount(), readTableEntriesDelta.getFromPosition());
    val timer = new Timer();
    val result = new DeltaIteratorResult<BufferView, Map.Entry<WireCommands.TableKey, WireCommands.TableValue>>(segment.getBytes().length + WireCommands.TableEntriesRead.HEADER_BYTES);
    tableStore.entryDeltaIterator(segment, fromPosition, TIMEOUT).thenCompose(itr -> itr.collectRemaining(e -> {
        if (result.getItemCount() >= suggestedEntryCount || result.getSizeBytes() >= MAX_READ_SIZE) {
            return false;
        }
        TableEntry entry = e.getEntries().iterator().next();
        DeltaIteratorState state = DeltaIteratorState.deserialize(e.getState());
        // Store all TableEntries.
        val k = new WireCommands.TableKey(toByteBuf(entry.getKey().getKey()), entry.getKey().getVersion());
        val v = new WireCommands.TableValue(toByteBuf(entry.getValue()));
        if (state.isDeletionRecord()) {
            result.remove(entry.getKey().getKey(), k.size() + v.size());
        } else {
            Map.Entry<WireCommands.TableKey, WireCommands.TableValue> old = result.getItem(entry.getKey().getKey());
            if (old != null && old.getKey().getKeyVersion() < entry.getKey().getVersion()) {
                int sizeBytes = (k.size() + v.size()) - (old.getKey().size() + old.getValue().size());
                result.add(entry.getKey().getKey(), new AbstractMap.SimpleImmutableEntry<>(k, v), sizeBytes);
            } else {
                result.add(entry.getKey().getKey(), new AbstractMap.SimpleImmutableEntry<>(k, v), k.size() + v.size());
            }
        }
        result.setState(state);
        // Update total read data.
        return true;
    })).thenAccept(v -> {
        log.debug(readTableEntriesDelta.getRequestId(), "Iterate Table Segment Entries Delta complete ({}).", result.getItemCount());
        connection.send(new WireCommands.TableEntriesDeltaRead(readTableEntriesDelta.getRequestId(), segment, new WireCommands.TableEntries(result.getItems()), result.getState().isShouldClear(), result.getState().isReachedEnd(), result.getState().getFromPosition()));
        this.tableStatsRecorder.iterateEntries(readTableEntriesDelta.getSegment(), result.getItemCount(), timer.getElapsed());
    }).exceptionally(e -> handleException(readTableEntriesDelta.getRequestId(), segment, operation, e));
}

Example 3 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class TableMetadataStore method createSegment.

@Override
protected CompletableFuture<Void> createSegment(String segmentName, ArrayView segmentInfo, TimeoutTimer timer) {
    ensureInitialized();
    TableEntry entry = TableEntry.notExists(getTableKey(segmentName), segmentInfo);
    return this.tableStore.put(this.metadataSegmentName, Collections.singletonList(entry), timer.getRemaining()).handle((ignored, ex) -> {
        if (ex != null) {
            if (Exceptions.unwrap(ex) instanceof BadKeyVersionException) {
                ex = new StreamSegmentExistsException(segmentName);
            }
            throw new CompletionException(ex);
        }
        return null;
    });
}

Example 4 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testGetTableEntries.

@Test
public void testGetTableEntries() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    val rnd = new Random(0);
    String tableSegmentName = "testGetTableEntries";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    val recorderMock = mock(TableSegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, tableStore, new TrackedConnection(connection), SegmentStatsRecorder.noOp(), recorderMock, new PassingTokenVerifier(), false);
    // Generate keys.
    ArrayList<ArrayView> keys = generateKeys(3, rnd);
    ArrayView testValue = generateValue(rnd);
    TableEntry e1 = TableEntry.unversioned(keys.get(0), testValue);
    TableEntry e2 = TableEntry.unversioned(keys.get(1), testValue);
    TableEntry e3 = TableEntry.unversioned(keys.get(2), testValue);
    // Create a table segment and add data.
    processor.createTableSegment(new WireCommands.CreateTableSegment(1, tableSegmentName, false, 0, "", 0));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, tableSegmentName));
    verify(recorderMock).createTableSegment(eq(tableSegmentName), any());
    processor.updateTableEntries(new WireCommands.UpdateTableEntries(2, tableSegmentName, "", getTableEntries(asList(e1, e2, e3)), WireCommands.NULL_TABLE_SEGMENT_OFFSET));
    verify(recorderMock).updateEntries(eq(tableSegmentName), eq(3), eq(false), any());
    // 1. Now read the table entries where suggestedEntryCount is equal to number of entries in the Table Store.
    WireCommands.TableIteratorArgs args = new WireCommands.TableIteratorArgs(Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER);
    processor.readTableEntries(new WireCommands.ReadTableEntries(3, tableSegmentName, "", 3, args));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommand> wireCommandsCaptor = ArgumentCaptor.forClass(WireCommand.class);
    order.verify(connection, times(2)).send(wireCommandsCaptor.capture());
    verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(3), any());
    // Verify the WireCommands.
    List<Long> keyVersions = ((WireCommands.TableEntriesUpdated) wireCommandsCaptor.getAllValues().get(0)).getUpdatedVersions();
    WireCommands.TableEntriesRead getTableEntriesIteratorsResp = (WireCommands.TableEntriesRead) wireCommandsCaptor.getAllValues().get(1);
    assertTrue(getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> e.getKey().getKeyVersion()).collect(Collectors.toList()).containsAll(keyVersions));
    // Verify if the value is correct.
    assertTrue(getTableEntriesIteratorsResp.getEntries().getEntries().stream().allMatch(e -> {
        ByteBuf buf = e.getValue().getData();
        byte[] bytes = new byte[buf.readableBytes()];
        buf.getBytes(buf.readerIndex(), bytes);
        return testValue.equals(new ByteArraySegment(bytes));
    }));
    // 2. Now read the table keys where suggestedEntryCount is less than the number of entries in the Table Store.
    processor.readTableEntries(new WireCommands.ReadTableEntries(3, tableSegmentName, "", 1, args));
    // Capture the WireCommands sent.
    ArgumentCaptor<WireCommands.TableEntriesRead> tableEntriesCaptor = ArgumentCaptor.forClass(WireCommands.TableEntriesRead.class);
    order.verify(connection, times(1)).send(tableEntriesCaptor.capture());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(0);
    assertEquals(1, getTableEntriesIteratorsResp.getEntries().getEntries().size());
    assertTrue(keyVersions.contains(getTableEntriesIteratorsResp.getEntries().getEntries().get(0).getKey().getKeyVersion()));
    // Get the last state.
    ByteBuf state = getTableEntriesIteratorsResp.getContinuationToken();
    args = new WireCommands.TableIteratorArgs(state, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER, Unpooled.EMPTY_BUFFER);
    // 3. Now read the remaining table entries by providing a higher suggestedKeyCount and the state to the iterator.
    processor.readTableEntries(new WireCommands.ReadTableEntries(3, tableSegmentName, "", 3, args));
    // Capture the WireCommands sent.
    tableEntriesCaptor = ArgumentCaptor.forClass(WireCommands.TableEntriesRead.class);
    order.verify(connection, times(1)).send(tableEntriesCaptor.capture());
    verify(recorderMock).iterateEntries(eq(tableSegmentName), eq(1), any());
    // Verify the WireCommands.
    getTableEntriesIteratorsResp = tableEntriesCaptor.getAllValues().get(0);
    assertEquals(2, getTableEntriesIteratorsResp.getEntries().getEntries().size());
    assertTrue(keyVersions.containsAll(getTableEntriesIteratorsResp.getEntries().getEntries().stream().map(e -> e.getKey().getKeyVersion()).collect(Collectors.toList())));
}

Example 5 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class HashTableSegmentLayout method put.

@Override
CompletableFuture<List<Long>> put(@NonNull DirectSegmentAccess segment, @NonNull List<TableEntry> entries, long tableSegmentOffset, TimeoutTimer timer) {
    val segmentInfo = segment.getInfo();
    ensureSegmentType(segmentInfo.getName(), segmentInfo.getType());
    // Generate an Update Batch for all the entries (since we need to know their Key Hashes and relative
    // offsets in the batch itself).
    val updateBatch = batch(entries, TableEntry::getKey, this.serializer::getUpdateLength, TableKeyBatch.update());
    logRequest("put", segmentInfo.getName(), updateBatch.isConditional(), tableSegmentOffset, entries.size(), updateBatch.getLength());
    return this.keyIndex.update(segment, updateBatch, () -> commit(entries, this.serializer::serializeUpdate, segment, tableSegmentOffset, timer.getRemaining()), timer);
}