Examples with BufferView io.pravega.common.util.BufferView used on opensource projects

Example 46 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class SegmentAggregatorTests method getAppendData.

// endregion
// region Helpers
private void getAppendData(StorageOperation operation, OutputStream stream, TestContext context) {
    Assert.assertTrue("Not an append operation: " + operation, operation instanceof CachedStreamSegmentAppendOperation);
    BufferView result = context.dataSource.getAppendData(operation.getStreamSegmentId(), operation.getStreamSegmentOffset(), (int) operation.getLength());
    try {
        result.copyTo(stream);
    } catch (IOException ex) {
        Assert.fail("Not expecting this exception: " + ex);
    }
}

Example 47 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class WriterTableProcessorTests method checkIndex.

private void checkIndex(HashMap<BufferView, TableEntry> existingEntries, HashMap<BufferView, UUID> allKeys, TestContext context) throws Exception {
    // Get all the buckets associated with the given keys.
    val timer = new TimeoutTimer(TIMEOUT);
    val bucketsByHash = context.indexReader.locateBuckets(context.segmentMock, allKeys.values(), timer).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Index the existing Keys by their current offsets.
    val keysByOffset = existingEntries.entrySet().stream().collect(Collectors.toMap(e -> e.getValue().getKey().getVersion(), Map.Entry::getKey));
    // Load up all the offsets for all buckets.
    val buckets = bucketsByHash.values().stream().distinct().collect(Collectors.toMap(b -> b, b -> context.indexReader.getBucketOffsets(context.segmentMock, b, timer).join()));
    // Loop through all the bucket's offsets and verify that those offsets do point to existing keys.
    for (val e : buckets.entrySet()) {
        val bucketOffsets = e.getValue();
        for (val offset : bucketOffsets) {
            Assert.assertTrue("Found Bucket Offset that points to non-existing key.", keysByOffset.containsKey(offset));
        }
    }
    // TableBucket, otherwise it is not included in any bucket.
    for (val e : allKeys.entrySet()) {
        val key = e.getKey();
        val tableEntry = existingEntries.get(key);
        val bucket = bucketsByHash.get(e.getValue());
        Assert.assertNotNull("Test error: no bucket found.", bucket);
        val bucketOffsets = buckets.get(bucket);
        if (tableEntry != null) {
            // This key should exist: just verify the TableEntry's offset (Key Version) exists in the Bucket's offset list.
            Assert.assertTrue("Non-deleted key was not included in a Table Bucket.", bucketOffsets.contains(tableEntry.getKey().getVersion()));
        } else {
            // Verify that all the keys that the Table Bucket points to do not match our key. Use our existing offset-key cache for that.
            for (val offset : bucketOffsets) {
                val keyAtOffset = keysByOffset.get(offset);
                Assert.assertNotEquals("Deleted key was still included in a Table Bucket.", key, keyAtOffset);
            }
        }
    }
}

Example 48 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableSegmentLayoutTestBase method testSingleUpdates.

@SneakyThrows
protected void testSingleUpdates(KeyHasher hasher, EntryGenerator generateToUpdate, KeyGenerator generateToRemove) {
    @Cleanup val context = new TableContext(hasher, executorService());
    // Generate the keys.
    val keys = IntStream.range(0, SINGLE_UPDATE_COUNT).mapToObj(i -> createRandomKey(context)).collect(Collectors.toList());
    // Create the Segment.
    createSegment(context, SEGMENT_NAME);
    @Cleanup val processor = createWriterTableProcessor(context);
    // Update.
    val expectedEntries = new HashMap<BufferView, BufferView>();
    val removedKeys = new ArrayList<BufferView>();
    val keyVersions = new HashMap<BufferView, Long>();
    Function<BufferView, Long> getKeyVersion = k -> keyVersions.getOrDefault(k, TableKey.NOT_EXISTS);
    int totalUpdateCount = 0;
    int totalRemoveCount = 0;
    for (val key : keys) {
        val toUpdate = generateToUpdate.apply(key, createRandomData(MAX_VALUE_LENGTH, context), getKeyVersion.apply(key));
        val updateResult = new AtomicReference<List<Long>>();
        context.ext.put(SEGMENT_NAME, Collections.singletonList(toUpdate), TIMEOUT).thenAccept(updateResult::set).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        Assert.assertEquals("Unexpected result size from update.", 1, updateResult.get().size());
        keyVersions.put(key, updateResult.get().get(0));
        expectedEntries.put(toUpdate.getKey().getKey(), toUpdate.getValue());
        check(expectedEntries, removedKeys, context.ext);
        if (processor != null) {
            processor.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            check(expectedEntries, removedKeys, context.ext);
        }
        totalUpdateCount++;
        checkTableAttributes(totalUpdateCount, totalRemoveCount, totalUpdateCount, context);
    }
    checkIterators(expectedEntries, context.ext);
    // Remove.
    for (val key : keys) {
        val toRemove = generateToRemove.apply(key, getKeyVersion.apply(key));
        context.ext.remove(SEGMENT_NAME, Collections.singleton(toRemove), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        removedKeys.add(key);
        expectedEntries.remove(key);
        keyVersions.remove(key);
        check(expectedEntries, removedKeys, context.ext);
        if (processor != null) {
            processor.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            check(expectedEntries, removedKeys, context.ext);
        }
        totalRemoveCount++;
        checkTableAttributes(totalUpdateCount, totalRemoveCount, totalUpdateCount - totalRemoveCount, context);
    }
    checkIterators(expectedEntries, context.ext);
    deleteSegment(expectedEntries.keySet(), supportsDeleteIfEmpty(), context.ext);
}

Example 49 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableSegmentLayoutTestBase method generateAndPopulateEntriesBatch.

private TestBatchData generateAndPopulateEntriesBatch(int batchSize, TestBatchData previous, TableContext context) {
    val expectedEntries = previous == null ? new HashMap<BufferView, BufferView>() : new HashMap<>(previous.expectedEntries);
    val removalCandidates = previous == null ? new ArrayList<BufferView>() : // Need a list so we can efficiently pick removal candidates.
    new ArrayList<>(expectedEntries.keySet());
    val toUpdate = new HashMap<BufferView, BufferView>();
    val toRemove = new ArrayList<BufferView>();
    for (int i = 0; i < batchSize; i++) {
        // We only generate a remove if we have something to remove.
        boolean remove = removalCandidates.size() > 0 && (context.random.nextDouble() < REMOVE_FRACTION);
        if (remove) {
            val key = removalCandidates.get(context.random.nextInt(removalCandidates.size()));
            toRemove.add(key);
            removalCandidates.remove(key);
        } else {
            // Generate a new Table Entry.
            BufferView key = createRandomKey(context);
            BufferView value = createRandomData(context.random.nextInt(MAX_VALUE_LENGTH), context);
            toUpdate.put(key, value);
            removalCandidates.add(key);
        }
    }
    expectedEntries.putAll(toUpdate);
    expectedEntries.keySet().removeAll(toRemove);
    // Keep track of expected metrics.
    int totalUpdateCount = toUpdate.size();
    int totalRemoveCount = toRemove.size();
    if (previous != null) {
        totalUpdateCount += previous.totalUpdatesCount;
        totalRemoveCount += previous.totalRemoveCount;
    }
    return new TestBatchData(toUpdate, toRemove, expectedEntries, totalUpdateCount, totalRemoveCount);
}

Example 50 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableServiceTests method check.

private void check(HashMap<BufferView, EntryData> keyInfo, TableStore tableStore) throws Exception {
    val bySegment = keyInfo.entrySet().stream().collect(Collectors.groupingBy(e -> e.getValue().segmentName));
    // Check inexistent keys.
    val searchFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    val iteratorFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    // Delta Iteration does not support fixed-key-length TableSegments.
    val unsortedIteratorFutures = new ArrayList<CompletableFuture<List<TableEntry>>>();
    val offsetIteratorFutures = new ArrayList<CompletableFuture<List<IteratorItem<TableEntry>>>>();
    val expectedResult = new ArrayList<Map.Entry<BufferView, EntryData>>();
    for (val e : bySegment.entrySet()) {
        String segmentName = e.getKey();
        boolean fixedKeyLength = isFixedKeyLength(segmentName);
        val info = tableStore.getInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        Assert.assertEquals(segmentName, info.getName());
        AssertExtensions.assertGreaterThan("Unexpected length for " + segmentName, 0, info.getLength());
        val expectedKeyLength = isFixedKeyLength(segmentName) ? getFixedKeyLength(segmentName) : 0;
        Assert.assertEquals("Unexpected key length for " + segmentName, expectedKeyLength, info.getKeyLength());
        Assert.assertEquals(fixedKeyLength, info.getType().isFixedKeyLengthTableSegment());
        val keys = new ArrayList<BufferView>();
        for (val se : e.getValue()) {
            keys.add(se.getKey());
            expectedResult.add(se);
        }
        searchFutures.add(tableStore.get(segmentName, keys, TIMEOUT));
        CompletableFuture<List<TableEntry>> entryIteratorFuture = tableStore.entryIterator(segmentName, IteratorArgs.builder().fetchTimeout(TIMEOUT).build()).thenCompose(ei -> {
            val result = new ArrayList<TableEntry>();
            return ei.forEachRemaining(i -> result.addAll(i.getEntries()), executorService()).thenApply(v -> {
                if (fixedKeyLength) {
                    checkSortedOrder(result);
                }
                return result;
            });
        });
        iteratorFutures.add(entryIteratorFuture);
        if (!fixedKeyLength) {
            unsortedIteratorFutures.add(entryIteratorFuture);
            // For simplicity, always start from beginning of TableSegment.
            offsetIteratorFutures.add(tableStore.entryDeltaIterator(segmentName, 0L, TIMEOUT).thenCompose(ei -> {
                val result = new ArrayList<IteratorItem<TableEntry>>();
                return ei.forEachRemaining(result::add, executorService()).thenApply(v -> result);
            }));
        }
    }
    // Check search results.
    val actualResults = Futures.allOfWithResults(searchFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).collect(Collectors.toList());
    Assert.assertEquals("Unexpected number of search results.", expectedResult.size(), actualResults.size());
    for (int i = 0; i < expectedResult.size(); i++) {
        val expectedKey = expectedResult.get(i).getKey();
        val expectedEntry = expectedResult.get(i).getValue();
        val actual = actualResults.get(i);
        if (expectedEntry.isDeleted()) {
            // Deleted keys will be returned as nulls.
            if (actual != null) {
                val r2 = tableStore.get(expectedEntry.segmentName, Collections.singletonList(expectedKey), TIMEOUT).join();
            }
            Assert.assertNull("Not expecting a value for a deleted Key ", actual);
        } else {
            Assert.assertEquals("Unexpected value for non-deleted Key.", expectedEntry.getValue(), actual.getValue());
            Assert.assertEquals("Unexpected key for non-deleted Key.", expectedKey, actual.getKey().getKey());
            Assert.assertEquals("Unexpected TableKey.Version for non-deleted Key.", expectedEntry.getVersion(), actual.getKey().getVersion());
        }
    }
    // Check iterator results. We sort it (and actualResults) by Version/Offset to ease the comparison.
    val actualIteratorResults = Futures.allOfWithResults(iteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val expectedIteratorResults = actualResults.stream().filter(Objects::nonNull).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    // These lists are used to compare non-delta based iteration with delta based iteration.
    val actualUnsortedIteratorResults = Futures.allOfWithResults(unsortedIteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val expectedUnsortedIteratorResults = actualUnsortedIteratorResults.stream().filter(Objects::nonNull).sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    val actualOffsetIteratorList = Futures.allOfWithResults(offsetIteratorFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream().flatMap(List::stream).collect(Collectors.toList());
    val actualOffsetIteratorResults = processDeltaIteratorItems(actualOffsetIteratorList).stream().sorted(Comparator.comparingLong(e -> e.getKey().getVersion())).collect(Collectors.toList());
    AssertExtensions.assertListEquals("Unexpected result from entryIterator().", expectedIteratorResults, actualIteratorResults, TableEntry::equals);
    for (val entry : expectedUnsortedIteratorResults) {
        Assert.assertNotNull("Missing expected TableEntry from deltaEntryIterator()", actualOffsetIteratorResults.contains(entry));
    }
}