Examples with TableEntry io.pravega.segmentstore.contracts.tables.TableEntry used on opensource projects

Example 21 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class WriterTableProcessorTests method checkIndex.

private void checkIndex(HashMap<BufferView, TableEntry> existingEntries, HashMap<BufferView, UUID> allKeys, TestContext context) throws Exception {
    // Get all the buckets associated with the given keys.
    val timer = new TimeoutTimer(TIMEOUT);
    val bucketsByHash = context.indexReader.locateBuckets(context.segmentMock, allKeys.values(), timer).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Index the existing Keys by their current offsets.
    val keysByOffset = existingEntries.entrySet().stream().collect(Collectors.toMap(e -> e.getValue().getKey().getVersion(), Map.Entry::getKey));
    // Load up all the offsets for all buckets.
    val buckets = bucketsByHash.values().stream().distinct().collect(Collectors.toMap(b -> b, b -> context.indexReader.getBucketOffsets(context.segmentMock, b, timer).join()));
    // Loop through all the bucket's offsets and verify that those offsets do point to existing keys.
    for (val e : buckets.entrySet()) {
        val bucketOffsets = e.getValue();
        for (val offset : bucketOffsets) {
            Assert.assertTrue("Found Bucket Offset that points to non-existing key.", keysByOffset.containsKey(offset));
        }
    }
    // TableBucket, otherwise it is not included in any bucket.
    for (val e : allKeys.entrySet()) {
        val key = e.getKey();
        val tableEntry = existingEntries.get(key);
        val bucket = bucketsByHash.get(e.getValue());
        Assert.assertNotNull("Test error: no bucket found.", bucket);
        val bucketOffsets = buckets.get(bucket);
        if (tableEntry != null) {
            // This key should exist: just verify the TableEntry's offset (Key Version) exists in the Bucket's offset list.
            Assert.assertTrue("Non-deleted key was not included in a Table Bucket.", bucketOffsets.contains(tableEntry.getKey().getVersion()));
        } else {
            // Verify that all the keys that the Table Bucket points to do not match our key. Use our existing offset-key cache for that.
            for (val offset : bucketOffsets) {
                val keyAtOffset = keysByOffset.get(offset);
                Assert.assertNotEquals("Deleted key was still included in a Table Bucket.", key, keyAtOffset);
            }
        }
    }
}

Example 22 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class TableServiceTests method check.

private void check(HashMap<BufferView, EntryData> keyInfo, TableStore tableStore) throws Exception {
    val bySegment = keyInfo.entrySet().stream().collect(Collectors.groupingBy(e -> e.getValue().segmentName));
    // Check inexistent keys.
    val searchFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    val iteratorFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    // Delta Iteration does not support fixed-key-length TableSegments.
    val unsortedIteratorFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    val offsetIteratorFutures = new ArrayList<CompletableFuture<List<IteratorItem<TableEntry>>>>();
    val expectedResult = new ArrayList<Map.Entry<BufferView, EntryData>>();
    for (val e : bySegment.entrySet()) {
        String segmentName = e.getKey();
        boolean fixedKeyLength = isFixedKeyLength(segmentName);
        val info = tableStore.getInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        Assert.assertEquals(segmentName, info.getName());
        AssertExtensions.assertGreaterThan("Unexpected length for " + segmentName, 0, info.getLength());
        val expectedKeyLength = isFixedKeyLength(segmentName) ? getFixedKeyLength(segmentName) : 0;
        Assert.assertEquals("Unexpected key length for " + segmentName, expectedKeyLength, info.getKeyLength());
        Assert.assertEquals(fixedKeyLength, info.getType().isFixedKeyLengthTableSegment());
        val keys = new ArrayList<BufferView>();
        for (val se : e.getValue()) {
            keys.add(se.getKey());
            expectedResult.add(se);
        }
        searchFutures.add(tableStore.get(segmentName, keys, TIMEOUT));
        CompletableFuture<List<TableEntry>> entryIteratorFuture = tableStore.entryIterator(segmentName, IteratorArgs.builder().fetchTimeout(TIMEOUT).build()).thenCompose(ei -> {
            val result = new ArrayList<TableEntry>();
            return ei.forEachRemaining(i -> result.addAll(i.getEntries()), executorService()).thenApply(v -> {
                if (fixedKeyLength) {
                    checkSortedOrder(result);
                }
                return result;
            });
        });
        iteratorFutures.add(entryIteratorFuture);
        if (!fixedKeyLength) {
            unsortedIteratorFutures.add(entryIteratorFuture);
            // For simplicity, always start from beginning of TableSegment.
            offsetIteratorFutures.add(tableStore.entryDeltaIterator(segmentName, 0L, TIMEOUT).thenCompose(ei -> {
                val result = new ArrayList<IteratorItem<TableEntry>>();
                return ei.forEachRemaining(result::add, executorService()).thenApply(v -> result);
            }));
        }
    }
    // Check search results.
    val actualResults = Futures.allOfWithResults(searchFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).collect(Collectors.toList());
    Assert.assertEquals("Unexpected number of search results.", expectedResult.size(), actualResults.size());
    for (int i = 0; i < expectedResult.size(); i++) {
        val expectedKey = expectedResult.get(i).getKey();
        val expectedEntry = expectedResult.get(i).getValue();
        val actual = actualResults.get(i);
        if (expectedEntry.isDeleted()) {
            // Deleted keys will be returned as nulls.
            if (actual != null) {
                val r2 = tableStore.get(expectedEntry.segmentName, Collections.singletonList(expectedKey), TIMEOUT).join();
            }
            Assert.assertNull("Not expecting a value for a deleted Key ", actual);
        } else {
            Assert.assertEquals("Unexpected value for non-deleted Key.", expectedEntry.getValue(), actual.getValue());
            Assert.assertEquals("Unexpected key for non-deleted Key.", expectedKey, actual.getKey().getKey());
            Assert.assertEquals("Unexpected TableKey.Version for non-deleted Key.", expectedEntry.getVersion(), actual.getKey().getVersion());
        }
    }
    // Check iterator results. We sort it (and actualResults) by Version/Offset to ease the comparison.
    val actualIteratorResults = Futures.allOfWithResults(iteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val expectedIteratorResults = actualResults.stream().filter(Objects::nonNull).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    // These lists are used to compare non-delta based iteration with delta based iteration.
    val actualUnsortedIteratorResults = Futures.allOfWithResults(unsortedIteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val expectedUnsortedIteratorResults = actualUnsortedIteratorResults.stream().filter(Objects::nonNull).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val actualOffsetIteratorList = Futures.allOfWithResults(offsetIteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).collect(Collectors.toList());
    val actualOffsetIteratorResults = processDeltaIteratorItems(actualOffsetIteratorList).stream().sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    AssertExtensions.assertListEquals("Unexpected result from entryIterator().", expectedIteratorResults, actualIteratorResults, TableEntry::equals);
    for (val entry : expectedUnsortedIteratorResults) {
        Assert.assertNotNull("Missing expected TableEntry from deltaEntryIterator()", actualOffsetIteratorResults.contains(entry));
    }
}

Example 23 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class TableMetadataStore method updateSegmentInfo.

@Override
protected CompletableFuture<Void> updateSegmentInfo(String segmentName, ArrayView segmentInfo, Duration timeout) {
    ensureInitialized();
    TableEntry entry = TableEntry.unversioned(getTableKey(segmentName), segmentInfo);
    return this.tableStore.put(this.metadataSegmentName, Collections.singletonList(entry), timeout).thenRun(Runnables.doNothing());
}

Example 24 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class FixedKeyLengthTableSegmentLayout method get.

@Override
CompletableFuture<List<TableEntry>> get(@NonNull DirectSegmentAccess segment, @NonNull List<BufferView> keys, TimeoutTimer timer) {
    val segmentInfo = segment.getInfo();
    ensureSegmentType(segmentInfo.getName(), segmentInfo.getType());
    val attributes = keys.stream().map(key -> {
        ensureValidKeyLength(segmentInfo.getName(), key.getLength());
        return AttributeId.from(key.getCopy());
    }).collect(Collectors.toList());
    logRequest("get", segmentInfo.getName(), keys.size());
    return getByAttributes(segment, attributes, timer);
}

Example 25 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class FixedKeyLengthTableSegmentLayout method getByAttributes.

private CompletableFuture<List<TableEntry>> getByAttributes(DirectSegmentAccess segment, List<AttributeId> attributes, TimeoutTimer timer) {
    return segment.getAttributes(attributes, false, timer.getRemaining()).thenComposeAsync(attributeValues -> {
        val result = new ArrayList<CompletableFuture<TableEntry>>(attributes.size());
        for (val attributeId : attributes) {
            val segmentOffset = attributeValues.getOrDefault(attributeId, NO_OFFSET);
            result.add(getEntryWithSingleRetry(segment, attributeId, segmentOffset, timer));
        }
        return Futures.allOfWithResults(result);
    }, this.executor);
}