Examples with BufferView io.pravega.common.util.BufferView used on opensource projects

Example 11 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class IndexReaderWriterTests method checkIndex.

private void checkIndex(Collection<BufferView> allKeys, Map<Long, BufferView> existingKeysByOffset, IndexWriter w, KeyHasher hasher, SegmentMock segment) {
    val timer = new TimeoutTimer(TIMEOUT);
    // Group all keys by their full hash (each hash should translate to a bucket), and make sure they're ordered by
    // offset (in descending order - so we can verify backpointer ordering).
    val existingKeys = existingKeysByOffset.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
    val keysByHash = allKeys.stream().map(key -> new BucketUpdate.KeyInfo(key, existingKeys.getOrDefault(key, NO_OFFSET), existingKeys.getOrDefault(key, NO_OFFSET))).sorted(// Reverse order.
    (k1, k2) -> Long.compare(k2.getOffset(), k1.getOffset())).collect(Collectors.groupingBy(keyInfo -> hasher.hash(keyInfo.getKey())));
    int existentBucketCount = 0;
    val buckets = w.locateBuckets(segment, keysByHash.keySet(), timer).join();
    for (val e : keysByHash.entrySet()) {
        val hash = e.getKey();
        val keys = e.getValue();
        val bucket = buckets.get(hash);
        Assert.assertNotNull("No bucket found for hash " + hash, bucket);
        boolean allDeleted = keys.stream().allMatch(k -> k.getOffset() == NO_OFFSET);
        Assert.assertNotEquals("Only expecting inexistent bucket when all its keys are deleted " + hash, allDeleted, bucket.exists());
        val bucketOffsets = w.getBucketOffsets(segment, bucket, timer).join();
        // Verify that we didn't return too many or too few keys.
        if (allDeleted) {
            Assert.assertEquals("Not expecting any offsets to be returned for bucket: " + hash, 0, bucketOffsets.size());
        } else {
            AssertExtensions.assertGreaterThan("Expected at least one offset to be returned for bucket: " + hash, 0, bucketOffsets.size());
            existentBucketCount++;
        }
        AssertExtensions.assertLessThanOrEqual("Too many offsets returned for bucket: " + hash, keys.size(), bucketOffsets.size());
        // Verify returned keys are as expected.
        for (int i = 0; i < bucketOffsets.size(); i++) {
            long actualOffset = bucketOffsets.get(i);
            long expectedOffset = keys.get(i).getOffset();
            String id = String.format("{%s[%s]}", hash, i);
            // In this loop, we do not expect to have Deleted Keys. If our Expected Offset indicates this key should
            // have been deleted, then getBucketOffsets() should not have returned this.
            Assert.assertNotEquals("Expecting a deleted key but found existing one: " + id, NO_OFFSET, expectedOffset);
            Assert.assertEquals("Unexpected key offset in bucket " + id, expectedOffset, actualOffset);
        }
        if (bucketOffsets.size() < keys.size()) {
            val prevKeyOffset = keys.get(bucketOffsets.size()).getOffset();
            Assert.assertEquals("Missing key from bucket " + hash, NO_OFFSET, prevKeyOffset);
        }
    }
    checkEntryCount(existingKeysByOffset.size(), segment);
    checkBucketCount(existentBucketCount, segment);
}

Example 12 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableCompactorTestBase method sort.

/**
 * Flattens the given Map of {@link KeyData} and Sorts the result by offset, producing {@link KeyInfo} instances.
 *
 * @param keys    The Keys to flatten and sort.
 * @param context TestContext.
 * @return Result.
 */
private List<KeyInfo> sort(Map<BufferView, KeyData> keys, TestContext context) {
    val result = new ArrayList<KeyInfo>();
    for (val keyData : keys.values()) {
        long lastOffset = keyData.values.lastKey();
        for (val e : keyData.values.entrySet()) {
            // An Entry is active only if it is the last indexed value for that key and it is not a deletion.
            boolean deleted = e.getValue() == null;
            boolean isActive = e.getKey() == lastOffset && !deleted;
            int length = deleted ? context.serializer.getRemovalLength(TableKey.unversioned(keyData.key)) : context.serializer.getUpdateLength(e.getValue());
            result.add(new KeyInfo(keyData.key, e.getKey(), length, isActive));
        }
    }
    result.sort(Comparator.comparingLong(k -> k.offset));
    return result;
}

Example 13 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableCompactorTestBase method populate.

/**
 * Generates a set of Table Entries and serializes them into the segment, then indexes them, using the following strategy:
 * - Keys are identified by their index (0..KEY_COUNT-1)
 * - At each iteration I:
 * -- The first I * ({@link #DELETE_COUNT} + {@link #SKIP_COUNT} keys are ignored.
 * -- The next {@link #DELETE_COUNT} Keys are removed from the index.
 * -- The next {@link #SKIP_COUNT} are also ignored.
 * -- The remaining keys are updated to a new value.
 * - The algorithm ends when there would no longer be any keys to update for a particular iteration.
 *
 * @param context TestContext.
 * @return A Map of Keys to {@link KeyData}.
 */
private Map<BufferView, KeyData> populate(TestContext context) {
    val rnd = new Random(0);
    // Generate keys.
    val keys = new ArrayList<KeyData>();
    for (int i = 0; i < KEY_COUNT; i++) {
        byte[] key = new byte[KEY_LENGTH];
        rnd.nextBytes(key);
        keys.add(new KeyData(new ByteArraySegment(key)));
    }
    // Populate segment.
    int minIndex = 0;
    context.setSegmentState(0, 0, 0, 0, 0);
    while (minIndex < keys.size()) {
        int deleteCount = Math.min(DELETE_COUNT, keys.size() - minIndex);
        for (int i = 0; i < deleteCount; i++) {
            val keyData = keys.get(minIndex);
            if (context.areDeletesSerialized()) {
                // Serialize removal and append it to the segment.
                val key = TableKey.unversioned(keyData.key);
                val serialization = context.serializer.serializeRemoval(Collections.singleton(key));
                val offset = context.segment.append(serialization, null, TIMEOUT).join();
                // Index it.
                val previousOffset = keyData.values.isEmpty() ? -1 : (long) keyData.values.lastKey();
                minIndex++;
                val keyUpdate = new BucketUpdate.KeyUpdate(keyData.key, offset, offset, true);
                context.index(keyUpdate, previousOffset, serialization.getLength());
                // Store it as a deletion.
                keyData.values.put(offset, null);
            } else {
                // If we don't need to serialize the removal, just index it.
                val keyUpdate = new BucketUpdate.KeyUpdate(keyData.key, -1L, -1L, true);
                context.index(keyUpdate, -1L, -1);
            }
        }
        // Update the rest.
        for (int keyIndex = minIndex; keyIndex < keys.size(); keyIndex++) {
            // Generate the value.
            val keyData = keys.get(keyIndex);
            byte[] valueData = new byte[VALUE_LENGTH];
            rnd.nextBytes(valueData);
            val value = new ByteArraySegment(valueData);
            // Serialize and append it to the segment.
            val entry = TableEntry.unversioned(keyData.key, value);
            val serialization = context.serializer.serializeUpdate(Collections.singleton(entry));
            val offset = context.segment.append(serialization, null, TIMEOUT).join();
            // Index it.
            val previousOffset = keyData.values.isEmpty() ? -1 : (long) keyData.values.lastKey();
            val keyUpdate = new BucketUpdate.KeyUpdate(keyData.key, offset, offset, false);
            context.index(keyUpdate, previousOffset, serialization.getLength());
            // Store it, but also encode its version within.
            keyData.values.put(offset, TableEntry.versioned(entry.getKey().getKey(), entry.getValue(), offset));
        }
        // Skip over the next few keys.
        minIndex += Math.min(SKIP_COUNT, keys.size() - minIndex);
    }
    // Sanity checks before we can move on with any test.
    Assert.assertEquals("Expecting the whole segment to have been indexed.", context.segmentMetadata.getLength(), context.getCompactor().getLastIndexedOffset());
    Assert.assertEquals("Not expecting any changes to the COMPACTION_OFFSET attribute.", 0, IndexReader.getCompactionOffset(context.segmentMetadata));
    AssertExtensions.assertLessThan("Expecting fewer active Table Entries than keys.", keys.size(), IndexReader.getEntryCount(context.segmentMetadata));
    AssertExtensions.assertGreaterThan("Expecting more total Table Entries than keys.", keys.size(), IndexReader.getTotalEntryCount(context.segmentMetadata));
    return keys.stream().collect(Collectors.toMap(k -> k.key, k -> k));
}

Example 14 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class AsyncReadResultProcessorTests method testReadFailures.

/**
 * Tests the AsyncReadResultProcessor when it encounters read failures.
 */
@Test
public void testReadFailures() {
    // Pre-generate some entries.
    final int totalLength = 1000;
    final Semaphore barrier = new Semaphore(0);
    // Setup an entry provider supplier that returns Future Reads, which will eventually fail.
    StreamSegmentReadResult.NextEntrySupplier supplier = (offset, length, makeCopy) -> {
        Supplier<BufferView> entryContentsSupplier = () -> {
            barrier.acquireUninterruptibly();
            throw new IntentionalException("Intentional");
        };
        return new TestFutureReadResultEntry(offset, length, entryContentsSupplier, executorService());
    };
    // Start an AsyncReadResultProcessor.
    @Cleanup StreamSegmentReadResult rr = new StreamSegmentReadResult(0, totalLength, supplier, "");
    TestReadResultHandler testReadResultHandler = new TestReadResultHandler(new ArrayList<>());
    try (AsyncReadResultProcessor rp = AsyncReadResultProcessor.process(rr, testReadResultHandler, executorService())) {
        barrier.release();
        // Wait for it to complete, and then verify that no errors have been recorded via the callbacks.
        AssertExtensions.assertThrows("Processor did not complete with the expected failure.", () -> testReadResultHandler.completed.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS), ex -> Exceptions.unwrap(ex) instanceof IntentionalException);
        Assert.assertEquals("Unexpected number of reads processed.", 0, testReadResultHandler.readCount.get());
        Assert.assertNotNull("No read failure encountered.", testReadResultHandler.error.get());
        Assert.assertTrue("Unexpected type of exception was raised: " + testReadResultHandler.error.get(), testReadResultHandler.error.get() instanceof IntentionalException);
    }
    Assert.assertTrue("ReadResult was not closed when the AsyncReadResultProcessor was closed.", rr.isClosed());
}

Example 15 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class ContainerReadIndexTests method testReadDirect.

/**
 * Tests the readDirect() method on the ReadIndex.
 */
@Test
public void testReadDirect() throws Exception {
    final int randomAppendLength = 1024;
    @Cleanup TestContext context = new TestContext();
    ArrayList<Long> segmentIds = new ArrayList<>();
    final long segmentId = createSegment(0, context);
    final UpdateableSegmentMetadata segmentMetadata = context.metadata.getStreamSegmentMetadata(segmentId);
    segmentIds.add(segmentId);
    HashMap<Long, ArrayList<Long>> transactionsBySegment = createTransactions(segmentIds, 1, context);
    final long mergedTxId = transactionsBySegment.get(segmentId).get(0);
    // Add data to all segments.
    HashMap<Long, ByteArrayOutputStream> segmentContents = new HashMap<>();
    transactionsBySegment.values().forEach(segmentIds::addAll);
    appendData(segmentIds, segmentContents, context);
    // Mark everything so far (minus a few bytes) as being written to storage.
    segmentMetadata.setStorageLength(segmentMetadata.getLength() - 100);
    // Now partially merge a second transaction
    final long mergedTxOffset = beginMergeTransaction(mergedTxId, segmentMetadata, segmentContents, context);
    // Add one more append after all of this.
    final long endOfMergedDataOffset = segmentMetadata.getLength();
    byte[] appendData = new byte[randomAppendLength];
    new Random(0).nextBytes(appendData);
    appendSingleWrite(segmentId, new ByteArraySegment(appendData), context);
    recordAppend(segmentId, new ByteArraySegment(appendData), segmentContents);
    // Verify we are not allowed to read from the range which has already been committed to Storage (invalid arguments).
    for (AtomicLong offset = new AtomicLong(0); offset.get() < segmentMetadata.getStorageLength(); offset.incrementAndGet()) {
        AssertExtensions.assertThrows(String.format("readDirect allowed reading from an illegal offset (%s).", offset), () -> context.readIndex.readDirect(segmentId, offset.get(), 1), ex -> ex instanceof IllegalArgumentException);
    }
    // Verify that any reads overlapping a merged transaction return null (that is, we cannot retrieve the requested data).
    for (long offset = mergedTxOffset - 1; offset < endOfMergedDataOffset; offset++) {
        val resultData = context.readIndex.readDirect(segmentId, offset, 2);
        Assert.assertNull("readDirect() returned data overlapping a partially merged transaction", resultData);
    }
    // Verify that we can read from any other offset.
    final byte[] expectedData = segmentContents.get(segmentId).toByteArray();
    BiConsumer<Long, Long> verifyReadResult = (startOffset, endOffset) -> {
        int readLength = (int) (endOffset - startOffset);
        while (readLength > 0) {
            BufferView actualDataBuffer;
            try {
                actualDataBuffer = context.readIndex.readDirect(segmentId, startOffset, readLength);
            } catch (StreamSegmentNotExistsException ex) {
                throw new CompletionException(ex);
            }
            Assert.assertNotNull(String.format("Unexpected result when data is readily available for Offset = %s, Length = %s.", startOffset, readLength), actualDataBuffer);
            byte[] actualData = actualDataBuffer.getCopy();
            AssertExtensions.assertArrayEquals("Unexpected data read from the segment at offset " + startOffset, expectedData, startOffset.intValue(), actualData, 0, actualData.length);
            // Setup the read for the next test (where we read 1 less byte than now).
            readLength--;
            if (readLength % 2 == 0) {
                // For every 2 bytes of decreased read length, increase the start offset by 1. This allows for a greater
                // number of combinations to be tested.
                startOffset++;
            }
        }
    };
    // Verify that we can read the cached data just after the StorageLength but before the merged transaction.
    verifyReadResult.accept(segmentMetadata.getStorageLength(), mergedTxOffset);
    // Verify that we can read the cached data just after the merged transaction but before the end of the segment.
    verifyReadResult.accept(endOfMergedDataOffset, segmentMetadata.getLength());
}