Examples with IteratorArgs io.pravega.segmentstore.contracts.tables.IteratorArgs used on opensource projects

Example 1 with IteratorArgs

use of io.pravega.segmentstore.contracts.tables.IteratorArgs in project pravega by pravega.

the class HashTableSegmentLayout method newIterator.

private <T> CompletableFuture<AsyncIterator<IteratorItem<T>>> newIterator(@NonNull DirectSegmentAccess segment, @NonNull IteratorArgs args, @NonNull GetBucketReader<T> createBucketReader) {
    Preconditions.checkArgument(args.getFrom() == null && args.getTo() == null, "Range Iterators not supported for HashTableSegments.");
    UUID fromHash;
    BufferView serializedState = args.getContinuationToken();
    try {
        fromHash = KeyHasher.getNextHash(serializedState == null ? null : IteratorStateImpl.deserialize(serializedState).getKeyHash());
    } catch (IOException ex) {
        // Bad IteratorState serialization.
        throw new IllegalDataFormatException("Unable to deserialize `serializedState`.", ex);
    }
    if (fromHash == null) {
        // Nothing to iterate on.
        return CompletableFuture.completedFuture(TableIterator.empty());
    }
    // Create a converter that will use a TableBucketReader to fetch all requested items in the iterated Buckets.
    val bucketReader = createBucketReader.apply(segment, this.keyIndex::getBackpointerOffset, this.executor);
    TableIterator.ConvertResult<IteratorItem<T>> converter = bucket -> bucketReader.findAllExisting(bucket.getSegmentOffset(), new TimeoutTimer(args.getFetchTimeout())).thenApply(result -> new IteratorItemImpl<>(new IteratorStateImpl(bucket.getHash()).serialize(), result));
    // Fetch the Tail (Unindexed) Hashes, then create the TableIterator.
    return this.keyIndex.getUnindexedKeyHashes(segment).thenComposeAsync(cacheHashes -> TableIterator.<IteratorItem<T>>builder().segment(segment).cacheHashes(cacheHashes).firstHash(fromHash).executor(executor).resultConverter(converter).fetchTimeout(args.getFetchTimeout()).build(), this.executor);
}

Example 2 with IteratorArgs

use of io.pravega.segmentstore.contracts.tables.IteratorArgs in project pravega by pravega.

the class TableSegmentLayoutTestBase method checkIterators.

@SneakyThrows
protected void checkIterators(Map<BufferView, BufferView> expectedEntries, ContainerTableExtension ext) {
    val emptyIteratorArgs = createEmptyIteratorArgs();
    // Check that invalid serializer state is handled properly.
    val emptyEntryIterator = ext.entryIterator(SEGMENT_NAME, emptyIteratorArgs).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val actualEmptyEntries = collectIteratorItems(emptyEntryIterator);
    Assert.assertEquals("Unexpected entries returned.", 0, actualEmptyEntries.size());
    val iteratorArgs = IteratorArgs.builder().fetchTimeout(TIMEOUT).build();
    // Collect and verify all Table Entries.
    val entryIterator = ext.entryIterator(SEGMENT_NAME, iteratorArgs).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val actualEntries = collectIteratorItems(entryIterator);
    // When we check Entry Iterator, we order by Version and verify that versions match. Entry Iterators also return
    // versions so it's important that we check those.
    actualEntries.sort(Comparator.comparingLong(e -> e.getKey().getVersion()));
    // Get the existing keys. We will use this to check Key Versions.
    val existingEntries = ext.get(SEGMENT_NAME, new ArrayList<>(expectedEntries.keySet()), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    existingEntries.sort(Comparator.comparingLong(e -> e.getKey().getVersion()));
    AssertExtensions.assertListEquals("Unexpected Table Entries from entryIterator().", existingEntries, actualEntries, TableEntry::equals);
    // Collect and verify all Table Keys. We now need to sort by Key, as Key Iterators do not return Version.
    val c = BufferViewComparator.create();
    val existingKeys = existingEntries.stream().sorted((e1, e2) -> c.compare(e1.getKey().getKey(), e2.getKey().getKey())).map(TableEntry::getKey).collect(Collectors.toList());
    val keyIterator = ext.keyIterator(SEGMENT_NAME, iteratorArgs).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val actualKeys = collectIteratorItems(keyIterator);
    actualKeys.sort((e1, e2) -> c.compare(e1.getKey(), e2.getKey()));
    AssertExtensions.assertListEquals("Unexpected Table Keys from keyIterator().", existingKeys, actualKeys, (k1, k2) -> k1.getKey().equals(k2.getKey()));
}

Example 3 with IteratorArgs

use of io.pravega.segmentstore.contracts.tables.IteratorArgs in project pravega by pravega.

the class PravegaRequestProcessor method readTableEntries.

@Override
public void readTableEntries(WireCommands.ReadTableEntries readTableEntries) {
    final String segment = readTableEntries.getSegment();
    final String operation = "readTableEntries";
    if (!verifyToken(segment, readTableEntries.getRequestId(), readTableEntries.getDelegationToken(), operation)) {
        return;
    }
    log.debug(readTableEntries.getRequestId(), "Iterate Table Segment Entries: Segment={}, Count={}.", readTableEntries.getSegment(), readTableEntries.getSuggestedEntryCount());
    final int suggestedEntryCount = readTableEntries.getSuggestedEntryCount();
    final IteratorArgs args = getIteratorArgs(readTableEntries.getArgs());
    val result = new IteratorResult<Map.Entry<WireCommands.TableKey, WireCommands.TableValue>>(segment.getBytes().length + WireCommands.TableEntriesRead.HEADER_BYTES);
    val timer = new Timer();
    tableStore.entryIterator(segment, args).thenCompose(itr -> itr.collectRemaining(e -> {
        if (result.getItemCount() >= suggestedEntryCount || result.getSizeBytes() >= MAX_READ_SIZE) {
            return false;
        }
        // Store all TableEntries.
        for (val entry : e.getEntries()) {
            val k = new WireCommands.TableKey(toByteBuf(entry.getKey().getKey()), entry.getKey().getVersion());
            val v = new WireCommands.TableValue(toByteBuf(entry.getValue()));
            result.add(new AbstractMap.SimpleImmutableEntry<>(k, v), k.size() + v.size());
        }
        // Update the continuation token.
        result.setContinuationToken(e.getState());
        return true;
    })).thenAccept(v -> {
        log.debug(readTableEntries.getRequestId(), "Iterate Table Segment Entries complete ({}).", result.getItemCount());
        connection.send(new WireCommands.TableEntriesRead(readTableEntries.getRequestId(), segment, new WireCommands.TableEntries(result.getItems()), toByteBuf(result.getContinuationToken())));
        this.tableStatsRecorder.iterateEntries(readTableEntries.getSegment(), result.getItemCount(), timer.getElapsed());
    }).exceptionally(e -> handleException(readTableEntries.getRequestId(), segment, operation, e));
}

Example 4 with IteratorArgs

use of io.pravega.segmentstore.contracts.tables.IteratorArgs in project pravega by pravega.

the class PravegaRequestProcessor method readTableKeys.

@Override
public void readTableKeys(WireCommands.ReadTableKeys readTableKeys) {
    final String segment = readTableKeys.getSegment();
    final String operation = "readTableKeys";
    if (!verifyToken(segment, readTableKeys.getRequestId(), readTableKeys.getDelegationToken(), operation)) {
        return;
    }
    log.debug(readTableKeys.getRequestId(), "Iterate Table Segment Keys: Segment={}, Count={}.", readTableKeys.getSegment(), readTableKeys.getSuggestedKeyCount());
    final int suggestedKeyCount = readTableKeys.getSuggestedKeyCount();
    final IteratorArgs args = getIteratorArgs(readTableKeys.getArgs());
    val result = new IteratorResult<WireCommands.TableKey>(segment.getBytes().length + WireCommands.TableKeysRead.HEADER_BYTES);
    val timer = new Timer();
    tableStore.keyIterator(segment, args).thenCompose(itr -> itr.collectRemaining(e -> {
        synchronized (result) {
            if (result.getItemCount() >= suggestedKeyCount || result.getSizeBytes() >= MAX_READ_SIZE) {
                return false;
            }
            // Store all TableKeys.
            for (val key : e.getEntries()) {
                val k = new WireCommands.TableKey(toByteBuf(key.getKey()), key.getVersion());
                result.add(k, k.size());
            }
            // Update the continuation token.
            result.setContinuationToken(e.getState());
            return true;
        }
    })).thenAccept(v -> {
        log.debug(readTableKeys.getRequestId(), "Iterate Table Segment Keys complete ({}).", result.getItemCount());
        connection.send(new WireCommands.TableKeysRead(readTableKeys.getRequestId(), segment, result.getItems(), toByteBuf(result.getContinuationToken())));
        this.tableStatsRecorder.iterateKeys(readTableKeys.getSegment(), result.getItemCount(), timer.getElapsed());
    }).exceptionally(e -> handleException(readTableKeys.getRequestId(), segment, operation, e));
}