Examples with BufferView io.pravega.common.util.BufferView used on opensource projects

Example 1 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class OperationLogTestBase method performReadIndexChecks.

void performReadIndexChecks(Collection<OperationWithCompletion> operations, ReadIndex readIndex) throws Exception {
    AbstractMap<Long, Integer> expectedLengths = getExpectedLengths(operations);
    AbstractMap<Long, InputStream> expectedData = getExpectedContents(operations);
    for (Map.Entry<Long, InputStream> e : expectedData.entrySet()) {
        int expectedLength = expectedLengths.getOrDefault(e.getKey(), -1);
        @Cleanup ReadResult readResult = readIndex.read(e.getKey(), 0, expectedLength, TIMEOUT);
        int readLength = 0;
        while (readResult.hasNext()) {
            BufferView entry = readResult.next().getContent().join();
            int length = entry.getLength();
            readLength += length;
            int streamSegmentOffset = expectedLengths.getOrDefault(e.getKey(), 0);
            expectedLengths.put(e.getKey(), streamSegmentOffset + length);
            AssertExtensions.assertStreamEquals(String.format("Unexpected data returned from ReadIndex. StreamSegmentId = %d, Offset = %d.", e.getKey(), streamSegmentOffset), e.getValue(), entry.getReader(), length);
        }
        Assert.assertEquals("Not enough bytes were read from the ReadIndex for StreamSegment " + e.getKey(), expectedLength, readLength);
    }
}

Example 2 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableSegmentLayoutTestBase method testBatchUpdates.

@SneakyThrows
protected void testBatchUpdates(int updateCount, int maxBatchSize, KeyHasher keyHasher, EntryGenerator generateToUpdate, KeyGenerator generateToRemove, CheckTable checkTable) {
    @Cleanup val context = new TableContext(keyHasher, executorService());
    // Create the segment and the Table Writer Processor.
    createSegment(context, SEGMENT_NAME);
    context.segment().updateAttributes(Collections.singletonMap(TableAttributes.MIN_UTILIZATION, 99L));
    @Cleanup val processor = createWriterTableProcessor(context);
    // Generate test data (in update & remove batches).
    val data = generateTestData(updateCount, maxBatchSize, context);
    // Process each such batch in turn. Keep track of the removed keys, as well as of existing key versions.
    val removedKeys = new HashSet<BufferView>();
    val keyVersions = new HashMap<BufferView, Long>();
    Function<BufferView, Long> getKeyVersion = k -> keyVersions.getOrDefault(k, TableKey.NOT_EXISTS);
    TestBatchData last = null;
    for (val current : data) {
        // Update entries.
        val toUpdate = current.toUpdate.entrySet().stream().map(e -> generateToUpdate.apply(e.getKey(), e.getValue(), getKeyVersion.apply(e.getKey()))).collect(Collectors.toList());
        context.ext.put(SEGMENT_NAME, toUpdate, TIMEOUT).thenAccept(versions -> {
            // Update key versions.
            Assert.assertEquals(toUpdate.size(), versions.size());
            for (int i = 0; i < versions.size(); i++) {
                keyVersions.put(toUpdate.get(i).getKey().getKey(), versions.get(i));
            }
        }).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        removedKeys.removeAll(current.toUpdate.keySet());
        // Remove entries.
        val toRemove = current.toRemove.stream().map(k -> generateToRemove.apply(k, getKeyVersion.apply(k))).collect(Collectors.toList());
        context.ext.remove(SEGMENT_NAME, toRemove, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        removedKeys.addAll(current.toRemove);
        keyVersions.keySet().removeAll(current.toRemove);
        if (processor != null) {
            // Flush the processor.
            Assert.assertTrue("Unexpected result from WriterTableProcessor.mustFlush().", processor.mustFlush());
            processor.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            Assert.assertFalse("Unexpected result from WriterTableProcessor.mustFlush() after flushing.", processor.mustFlush());
        }
        // Verify result (from cache).
        checkTable.accept(current.expectedEntries, removedKeys, context.ext);
        checkTableAttributes(current.totalUpdatesCount, current.totalRemoveCount, current.expectedEntries.size(), context);
        last = current;
    }
    // At the end, wait for the data to be indexed, clear the cache and verify result.
    // Close the current instance so that we can discard the cache.
    context.ext.close();
    @Cleanup val ext2 = context.createExtension();
    @Cleanup val processor2 = createWriterTableProcessor(ext2, context);
    checkTable.accept(last.expectedEntries, removedKeys, ext2);
    // Finally, remove all data.
    val finalRemoval = last.expectedEntries.keySet().stream().map(k -> toUnconditionalKey(k, 1)).collect(Collectors.toList());
    ext2.remove(SEGMENT_NAME, finalRemoval, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    removedKeys.addAll(last.expectedEntries.keySet());
    if (processor2 != null) {
        processor2.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }
    checkTable.accept(Collections.emptyMap(), removedKeys, ext2);
    deleteSegment(Collections.emptyList(), supportsDeleteIfEmpty(), ext2);
}

Example 3 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableSegmentLayoutTestBase method collectIteratorItems.

private <T> List<T> collectIteratorItems(AsyncIterator<IteratorItem<T>> iterator) throws Exception {
    val result = new ArrayList<T>();
    val hashes = new HashSet<BufferView>();
    iterator.forEachRemaining(item -> {
        Assert.assertTrue("Duplicate IteratorItem.getState().", hashes.add(item.getState()));
        result.addAll(item.getEntries());
    }, executorService()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    return result;
}

Example 4 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableSegmentLayoutTestBase method checkIterators.

@SneakyThrows
protected void checkIterators(Map<BufferView, BufferView> expectedEntries, ContainerTableExtension ext) {
    val emptyIteratorArgs = createEmptyIteratorArgs();
    // Check that invalid serializer state is handled properly.
    val emptyEntryIterator = ext.entryIterator(SEGMENT_NAME, emptyIteratorArgs).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val actualEmptyEntries = collectIteratorItems(emptyEntryIterator);
    Assert.assertEquals("Unexpected entries returned.", 0, actualEmptyEntries.size());
    val iteratorArgs = IteratorArgs.builder().fetchTimeout(TIMEOUT).build();
    // Collect and verify all Table Entries.
    val entryIterator = ext.entryIterator(SEGMENT_NAME, iteratorArgs).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val actualEntries = collectIteratorItems(entryIterator);
    // When we check Entry Iterator, we order by Version and verify that versions match. Entry Iterators also return
    // versions so it's important that we check those.
    actualEntries.sort(Comparator.comparingLong(e -> e.getKey().getVersion()));
    // Get the existing keys. We will use this to check Key Versions.
    val existingEntries = ext.get(SEGMENT_NAME, new ArrayList<>(expectedEntries.keySet()), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    existingEntries.sort(Comparator.comparingLong(e -> e.getKey().getVersion()));
    AssertExtensions.assertListEquals("Unexpected Table Entries from entryIterator().", existingEntries, actualEntries, TableEntry::equals);
    // Collect and verify all Table Keys. We now need to sort by Key, as Key Iterators do not return Version.
    val c = BufferViewComparator.create();
    val existingKeys = existingEntries.stream().sorted((e1, e2) -> c.compare(e1.getKey().getKey(), e2.getKey().getKey())).map(TableEntry::getKey).collect(Collectors.toList());
    val keyIterator = ext.keyIterator(SEGMENT_NAME, iteratorArgs).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val actualKeys = collectIteratorItems(keyIterator);
    actualKeys.sort((e1, e2) -> c.compare(e1.getKey(), e2.getKey()));
    AssertExtensions.assertListEquals("Unexpected Table Keys from keyIterator().", existingKeys, actualKeys, (k1, k2) -> k1.getKey().equals(k2.getKey()));
}

Example 5 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableSegmentLayoutTestBase method testTableSegmentCompacted.

@SneakyThrows
protected void testTableSegmentCompacted(KeyHasher keyHasher, CheckTable checkTable) {
    val config = TableExtensionConfig.builder().with(TableExtensionConfig.MAX_COMPACTION_SIZE, (MAX_KEY_LENGTH + MAX_VALUE_LENGTH) * BATCH_SIZE).with(TableExtensionConfig.COMPACTION_FREQUENCY, 1).build();
    @Cleanup val context = new TableContext(config, keyHasher, executorService());
    // Create the segment and the Table Writer Processor.
    createSegment(context, SEGMENT_NAME);
    context.segment().updateAttributes(Collections.singletonMap(TableAttributes.MIN_UTILIZATION, 99L));
    @Cleanup val processor = createWriterTableProcessor(context);
    // Generate test data (in update & remove batches).
    val data = generateTestData(BATCH_UPDATE_COUNT, BATCH_SIZE, context);
    // Process each such batch in turn. Keep track of the removed keys, as well as of existing key versions.
    val removedKeys = new HashSet<BufferView>();
    val keyVersions = new HashMap<BufferView, Long>();
    Function<BufferView, Long> getKeyVersion = k -> keyVersions.getOrDefault(k, TableKey.NOT_EXISTS);
    TestBatchData last = null;
    for (val current : data) {
        // Update entries.
        val toUpdate = current.toUpdate.entrySet().stream().map(e -> toUnconditionalTableEntry(e.getKey(), e.getValue(), getKeyVersion.apply(e.getKey()))).collect(Collectors.toList());
        context.ext.put(SEGMENT_NAME, toUpdate, TIMEOUT).thenAccept(versions -> {
            // Update key versions.
            Assert.assertEquals(toUpdate.size(), versions.size());
            for (int i = 0; i < versions.size(); i++) {
                keyVersions.put(toUpdate.get(i).getKey().getKey(), versions.get(i));
            }
        }).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        removedKeys.removeAll(current.toUpdate.keySet());
        // Remove entries.
        val toRemove = current.toRemove.stream().map(k -> toUnconditionalKey(k, getKeyVersion.apply(k))).collect(Collectors.toList());
        context.ext.remove(SEGMENT_NAME, toRemove, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        removedKeys.addAll(current.toRemove);
        keyVersions.keySet().removeAll(current.toRemove);
        if (processor != null) {
            // Background indexer Table Segment - flush the processor.
            Assert.assertTrue("Unexpected result from WriterTableProcessor.mustFlush().", processor.mustFlush());
            processor.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        }
        // Verify result (from cache).
        checkTable.accept(current.expectedEntries, removedKeys, context.ext);
        last = current;
    }
    // Verify we have had at least one compaction during this test. This may happen in the background, so give it
    // some time to execute.
    TestUtils.await(() -> 0 < IndexReader.getCompactionOffset(context.segment().getInfo()), 10, TIMEOUT.toMillis());
    AssertExtensions.assertGreaterThan("No truncation occurred", 0, context.segment().getInfo().getStartOffset());
    // Finally, remove all data and delete the segment.
    val finalRemoval = last.expectedEntries.keySet().stream().map(k -> toUnconditionalKey(k, 1)).collect(Collectors.toList());
    context.ext.remove(SEGMENT_NAME, finalRemoval, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    removedKeys.addAll(last.expectedEntries.keySet());
    deleteSegment(Collections.emptyList(), supportsDeleteIfEmpty(), context.ext);
}