Examples with IteratorItem io.pravega.segmentstore.contracts.tables.IteratorItem used on opensource projects

Example 1 with IteratorItem

use of io.pravega.segmentstore.contracts.tables.IteratorItem in project pravega by pravega.

the class HashTableSegmentLayout method newIterator.

private <T> CompletableFuture<AsyncIterator<IteratorItem<T>>> newIterator(@NonNull DirectSegmentAccess segment, @NonNull IteratorArgs args, @NonNull GetBucketReader<T> createBucketReader) {
    Preconditions.checkArgument(args.getFrom() == null && args.getTo() == null, "Range Iterators not supported for HashTableSegments.");
    UUID fromHash;
    BufferView serializedState = args.getContinuationToken();
    try {
        fromHash = KeyHasher.getNextHash(serializedState == null ? null : IteratorStateImpl.deserialize(serializedState).getKeyHash());
    } catch (IOException ex) {
        // Bad IteratorState serialization.
        throw new IllegalDataFormatException("Unable to deserialize `serializedState`.", ex);
    }
    if (fromHash == null) {
        // Nothing to iterate on.
        return CompletableFuture.completedFuture(TableIterator.empty());
    }
    // Create a converter that will use a TableBucketReader to fetch all requested items in the iterated Buckets.
    val bucketReader = createBucketReader.apply(segment, this.keyIndex::getBackpointerOffset, this.executor);
    TableIterator.ConvertResult<IteratorItem<T>> converter = bucket -> bucketReader.findAllExisting(bucket.getSegmentOffset(), new TimeoutTimer(args.getFetchTimeout())).thenApply(result -> new IteratorItemImpl<>(new IteratorStateImpl(bucket.getHash()).serialize(), result));
    // Fetch the Tail (Unindexed) Hashes, then create the TableIterator.
    return this.keyIndex.getUnindexedKeyHashes(segment).thenComposeAsync(cacheHashes -> TableIterator.<IteratorItem<T>>builder().segment(segment).cacheHashes(cacheHashes).firstHash(fromHash).executor(executor).resultConverter(converter).fetchTimeout(args.getFetchTimeout()).build(), this.executor);
}

Example 2 with IteratorItem

use of io.pravega.segmentstore.contracts.tables.IteratorItem in project pravega by pravega.

the class TableSegmentLayoutTestBase method collectIteratorItems.

private <T> List<T> collectIteratorItems(AsyncIterator<IteratorItem<T>> iterator) throws Exception {
    val result = new ArrayList<T>();
    val hashes = new HashSet<BufferView>();
    iterator.forEachRemaining(item -> {
        Assert.assertTrue("Duplicate IteratorItem.getState().", hashes.add(item.getState()));
        result.addAll(item.getEntries());
    }, executorService()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    return result;
}

Example 3 with IteratorItem

use of io.pravega.segmentstore.contracts.tables.IteratorItem in project pravega by pravega.

the class TableServiceTests method check.

private void check(HashMap<BufferView, EntryData> keyInfo, TableStore tableStore) throws Exception {
    val bySegment = keyInfo.entrySet().stream().collect(Collectors.groupingBy(e -> e.getValue().segmentName));
    // Check inexistent keys.
    val searchFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    val iteratorFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    // Delta Iteration does not support fixed-key-length TableSegments.
    val unsortedIteratorFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    val offsetIteratorFutures = new ArrayList<CompletableFuture<List<IteratorItem<TableEntry>>>>();
    val expectedResult = new ArrayList<Map.Entry<BufferView, EntryData>>();
    for (val e : bySegment.entrySet()) {
        String segmentName = e.getKey();
        boolean fixedKeyLength = isFixedKeyLength(segmentName);
        val info = tableStore.getInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        Assert.assertEquals(segmentName, info.getName());
        AssertExtensions.assertGreaterThan("Unexpected length for " + segmentName, 0, info.getLength());
        val expectedKeyLength = isFixedKeyLength(segmentName) ? getFixedKeyLength(segmentName) : 0;
        Assert.assertEquals("Unexpected key length for " + segmentName, expectedKeyLength, info.getKeyLength());
        Assert.assertEquals(fixedKeyLength, info.getType().isFixedKeyLengthTableSegment());
        val keys = new ArrayList<BufferView>();
        for (val se : e.getValue()) {
            keys.add(se.getKey());
            expectedResult.add(se);
        }
        searchFutures.add(tableStore.get(segmentName, keys, TIMEOUT));
        CompletableFuture<List<TableEntry>> entryIteratorFuture = tableStore.entryIterator(segmentName, IteratorArgs.builder().fetchTimeout(TIMEOUT).build()).thenCompose(ei -> {
            val result = new ArrayList<TableEntry>();
            return ei.forEachRemaining(i -> result.addAll(i.getEntries()), executorService()).thenApply(v -> {
                if (fixedKeyLength) {
                    checkSortedOrder(result);
                }
                return result;
            });
        });
        iteratorFutures.add(entryIteratorFuture);
        if (!fixedKeyLength) {
            unsortedIteratorFutures.add(entryIteratorFuture);
            // For simplicity, always start from beginning of TableSegment.
            offsetIteratorFutures.add(tableStore.entryDeltaIterator(segmentName, 0L, TIMEOUT).thenCompose(ei -> {
                val result = new ArrayList<IteratorItem<TableEntry>>();
                return ei.forEachRemaining(result::add, executorService()).thenApply(v -> result);
            }));
        }
    }
    // Check search results.
    val actualResults = Futures.allOfWithResults(searchFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).collect(Collectors.toList());
    Assert.assertEquals("Unexpected number of search results.", expectedResult.size(), actualResults.size());
    for (int i = 0; i < expectedResult.size(); i++) {
        val expectedKey = expectedResult.get(i).getKey();
        val expectedEntry = expectedResult.get(i).getValue();
        val actual = actualResults.get(i);
        if (expectedEntry.isDeleted()) {
            // Deleted keys will be returned as nulls.
            if (actual != null) {
                val r2 = tableStore.get(expectedEntry.segmentName, Collections.singletonList(expectedKey), TIMEOUT).join();
            }
            Assert.assertNull("Not expecting a value for a deleted Key ", actual);
        } else {
            Assert.assertEquals("Unexpected value for non-deleted Key.", expectedEntry.getValue(), actual.getValue());
            Assert.assertEquals("Unexpected key for non-deleted Key.", expectedKey, actual.getKey().getKey());
            Assert.assertEquals("Unexpected TableKey.Version for non-deleted Key.", expectedEntry.getVersion(), actual.getKey().getVersion());
        }
    }
    // Check iterator results. We sort it (and actualResults) by Version/Offset to ease the comparison.
    val actualIteratorResults = Futures.allOfWithResults(iteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val expectedIteratorResults = actualResults.stream().filter(Objects::nonNull).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    // These lists are used to compare non-delta based iteration with delta based iteration.
    val actualUnsortedIteratorResults = Futures.allOfWithResults(unsortedIteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val expectedUnsortedIteratorResults = actualUnsortedIteratorResults.stream().filter(Objects::nonNull).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val actualOffsetIteratorList = Futures.allOfWithResults(offsetIteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).collect(Collectors.toList());
    val actualOffsetIteratorResults = processDeltaIteratorItems(actualOffsetIteratorList).stream().sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    AssertExtensions.assertListEquals("Unexpected result from entryIterator().", expectedIteratorResults, actualIteratorResults, TableEntry::equals);
    for (val entry : expectedUnsortedIteratorResults) {
        Assert.assertNotNull("Missing expected TableEntry from deltaEntryIterator()", actualOffsetIteratorResults.contains(entry));
    }
}

Example 4 with IteratorItem

use of io.pravega.segmentstore.contracts.tables.IteratorItem in project pravega by pravega.

the class HashTableSegmentLayout method entryDeltaIterator.

@Override
AsyncIterator<IteratorItem<TableEntry>> entryDeltaIterator(@NonNull DirectSegmentAccess segment, long fromPosition, Duration fetchTimeout) {
    val segmentInfo = segment.getInfo();
    ensureSegmentType(segmentInfo.getName(), segmentInfo.getType());
    Preconditions.checkArgument(fromPosition <= segmentInfo.getLength(), "fromPosition (%s) can not exceed the length (%s) of the TableSegment.", fromPosition, segmentInfo.getLength());
    logRequest("entryDeltaIterator", segment.getSegmentId(), fromPosition);
    long compactionOffset = segmentInfo.getAttributes().getOrDefault(TableAttributes.COMPACTION_OFFSET, 0L);
    // All of the most recent keys will exist beyond the compactionOffset.
    long startOffset = Math.max(fromPosition, compactionOffset);
    // We should clear if the starting position may have been truncated out due to compaction.
    boolean shouldClear = fromPosition < compactionOffset;
    // Maximum length of the TableSegment we want to read until.
    int maxBytesToRead = (int) (segmentInfo.getLength() - startOffset);
    TableEntryDeltaIterator.ConvertResult<IteratorItem<TableEntry>> converter = item -> CompletableFuture.completedFuture(new IteratorItemImpl<TableEntry>(item.getKey().serialize(), Collections.singletonList(item.getValue())));
    return TableEntryDeltaIterator.<IteratorItem<TableEntry>>builder().segment(segment).entrySerializer(serializer).executor(executor).maxBytesToRead(maxBytesToRead).startOffset(startOffset).currentBatchOffset(fromPosition).fetchTimeout(fetchTimeout).resultConverter(converter).shouldClear(shouldClear).build();
}