Examples with BufferView io.pravega.common.util.BufferView used on opensource projects

Example 16 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class DirectMemoryCacheTests method testCacheFull.

/**
 * Tests the ability to notify the caller that a cache is full and handle various situations.
 */
@Test
public void testCacheFull() {
    final int cleanAfterInvocations = DirectMemoryCache.MAX_CLEANUP_ATTEMPTS - 1;
    final int maxSize = LAYOUT.bufferSize();
    final int maxStoredSize = maxSize - LAYOUT.blockSize();
    final BufferView toInsert = new ByteArraySegment(new byte[1]);
    @Cleanup val c = new TestCache(maxSize);
    // Fill up the cache.
    val address1 = c.insert(new ByteArraySegment(new byte[maxStoredSize]));
    checkSnapshot(c, (long) maxStoredSize, (long) maxSize, (long) LAYOUT.blockSize(), (long) maxSize, (long) maxSize);
    // Return value from the setCacheFullCallback supplier.
    val reportClean = new AtomicBoolean(false);
    // If true, will actually perform a cleanup.
    val actualClean = new AtomicBoolean(false);
    // If true, will throw an error.
    val throwError = new AtomicBoolean(false);
    val invocationCount = new AtomicInteger(0);
    c.setCacheFullCallback(() -> {
        invocationCount.incrementAndGet();
        if (actualClean.get() && invocationCount.get() >= cleanAfterInvocations) {
            c.delete(address1);
        }
        if (throwError.get()) {
            throw new IntentionalException();
        }
        return reportClean.get();
    }, 1);
    // 1. No cleanup, no error, reporting the truth. We should try as many times as possible.
    reportClean.set(false);
    actualClean.set(false);
    throwError.set(false);
    invocationCount.set(0);
    AssertExtensions.assertThrows("Expected CacheFullException when no cleanup and no error.", () -> c.insert(toInsert), ex -> ex instanceof CacheFullException);
    Assert.assertEquals("Unexpected number of invocations when no cleanup and no error.", DirectMemoryCache.MAX_CLEANUP_ATTEMPTS, invocationCount.get());
    // 2. No cleanup, no error, reporting that we did clean up. We should try as many times as possible.
    reportClean.set(true);
    actualClean.set(false);
    throwError.set(false);
    invocationCount.set(0);
    AssertExtensions.assertThrows("Expected CacheFullException when no cleanup and no error, but reporting clean.", () -> c.insert(toInsert), ex -> ex instanceof CacheFullException);
    Assert.assertEquals("Unexpected number of invocations when no cleanup but reporting clean.", DirectMemoryCache.MAX_CLEANUP_ATTEMPTS, invocationCount.get());
    // 3. No cleanup, throw error.
    reportClean.set(false);
    actualClean.set(false);
    throwError.set(true);
    invocationCount.set(0);
    AssertExtensions.assertThrows("Expected IntentionalException when error.", () -> c.insert(toInsert), ex -> ex instanceof IntentionalException);
    Assert.assertEquals("Unexpected number of invocations when error.", 1, invocationCount.get());
    // 4. Cleanup.
    reportClean.set(true);
    actualClean.set(true);
    throwError.set(false);
    invocationCount.set(0);
    val address2 = c.insert(toInsert);
    Assert.assertEquals("Unexpected number of invocations when successful cleanup.", cleanAfterInvocations, invocationCount.get());
    Assert.assertEquals("Unexpected second entry.", 1, c.get(address2).getLength());
    checkSnapshot(c, 1L, LAYOUT.blockSize() * 2L, (long) LAYOUT.blockSize(), (long) maxSize, (long) maxSize);
}

Example 17 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class TableBasedMetadataStore method read.

/**
 * Reads a metadata record for the given key.
 *
 * @param key Key for the metadata record.
 * @return Associated {@link io.pravega.segmentstore.storage.metadata.BaseMetadataStore.TransactionData}.
 */
@Override
protected CompletableFuture<TransactionData> read(String key) {
    val keys = new ArrayList<BufferView>();
    keys.add(new ByteArraySegment(key.getBytes(Charsets.UTF_8)));
    val t = new Timer();
    return ensureInitialized().thenComposeAsync(v -> this.tableStore.get(tableName, keys, timeout).thenApplyAsync(entries -> {
        try {
            Preconditions.checkState(entries.size() == 1, "Unexpected number of values returned.");
            val entry = entries.get(0);
            if (null != entry) {
                val arr = entry.getValue();
                TransactionData txnData = SERIALIZER.deserialize(arr);
                txnData.setDbObject(entry.getKey().getVersion());
                txnData.setPersisted(true);
                TABLE_GET_LATENCY.reportSuccessEvent(t.getElapsed());
                METADATA_FOUND_IN_STORE.inc();
                return txnData;
            }
        } catch (Exception e) {
            throw new CompletionException(new StorageMetadataException("Error while reading", e));
        }
        TABLE_GET_LATENCY.reportSuccessEvent(t.getElapsed());
        METADATA_NOT_FOUND.inc();
        return TransactionData.builder().key(key).persisted(true).dbObject(TableKey.NOT_EXISTS).build();
    }, getExecutor()).exceptionally(e -> {
        val ex = Exceptions.unwrap(e);
        throw new CompletionException(handleException(ex));
    }), getExecutor());
}

Example 18 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class BenchmarkTests method testRandomOperations.

private RandomResult testRandomOperations(CacheStorage s) {
    val writeBuffer = new ByteArraySegment(new byte[ENTRY_SIZE]);
    this.random.nextBytes(writeBuffer.array());
    val ids = new ArrayList<Integer>();
    val timer = new Timer();
    val threads = new ArrayList<Thread>();
    val iterations = new AtomicInteger(0);
    val insertCount = new AtomicInteger(0);
    val getCount = new AtomicInteger(0);
    val deleteCount = new AtomicInteger(0);
    for (int threadId = 0; threadId < RANDOM_OPERATIONS_THREAD_COUNT; threadId++) {
        val t = new Thread(() -> {
            val readBuffer = new byte[ENTRY_SIZE * 2];
            int i;
            while ((i = iterations.incrementAndGet()) <= ENTRY_COUNT) {
                boolean insert;
                int length;
                synchronized (ids) {
                    insert = (this.random.nextInt(100) < RANDOM_OPERATIONS_INSERT_PERCENTAGE) || ids.isEmpty();
                    length = insert ? this.random.nextInt(writeBuffer.getLength()) : 0;
                }
                if (insert) {
                    int insertedId = s.insert(writeBuffer.slice(0, length));
                    synchronized (ids) {
                        ids.add(insertedId);
                    }
                    insertCount.incrementAndGet();
                } else {
                    // delete
                    int toRemove;
                    synchronized (ids) {
                        toRemove = ids.remove(this.random.nextInt(ids.size()));
                    }
                    s.delete(toRemove);
                    deleteCount.incrementAndGet();
                }
                int toRead = -1;
                synchronized (ids) {
                    if (!ids.isEmpty()) {
                        toRead = ids.get(this.random.nextInt(ids.size()));
                    }
                }
                if (toRead >= 0) {
                    BufferView result = s.get(toRead);
                    if (result != null) {
                        result.copyTo(ByteBuffer.wrap(readBuffer));
                    }
                    getCount.incrementAndGet();
                }
            }
        });
        t.start();
        threads.add(t);
    }
    for (val t : threads) {
        Exceptions.handleInterrupted(t::join);
    }
    Duration elapsed = timer.getElapsed();
    // do not count this.
    ids.forEach(s::delete);
    return new RandomResult(elapsed, insertCount.get(), getCount.get(), deleteCount.get());
}

Example 19 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkReadsWhileTruncating.

private void checkReadsWhileTruncating(HashMap<String, ByteArrayOutputStream> segmentContents, HashMap<String, Long> startOffsets, StreamSegmentStore store) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        long segmentLength = store.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength();
        long expectedCurrentOffset = 0;
        boolean truncate = false;
        while (expectedCurrentOffset < segmentLength) {
            @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset, (int) (segmentLength - expectedCurrentOffset), TIMEOUT).join();
            Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
            // We only test the truncation-related pieces here; other read-related checks are done in checkReads.
            while (readResult.hasNext()) {
                ReadResultEntry readEntry = readResult.next();
                Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
                if (!readEntry.getContent().isDone()) {
                    readEntry.requestContent(TIMEOUT);
                }
                if (readEntry.getType() == ReadResultEntryType.Truncated) {
                    long startOffset = startOffsets.getOrDefault(segmentName, 0L);
                    // Verify that the Segment actually is truncated beyond this offset.
                    AssertExtensions.assertLessThan("Found Truncated ReadResultEntry but current offset not truncated.", startOffset, readEntry.getStreamSegmentOffset());
                    // Verify the ReadResultEntry cannot be used and throws an appropriate exception.
                    AssertExtensions.assertSuppliedFutureThrows("ReadEntry.getContent() did not throw for a Truncated entry.", readEntry::getContent, ex -> ex instanceof StreamSegmentTruncatedException);
                    // Verify ReadResult is done.
                    Assert.assertFalse("Unexpected result from ReadResult.hasNext when encountering truncated entry.", readResult.hasNext());
                    // Verify attempting to read at the current offset will return the appropriate entry (and not throw).
                    @Cleanup ReadResult truncatedResult = store.read(segmentName, readEntry.getStreamSegmentOffset(), 1, TIMEOUT).join();
                    val first = truncatedResult.next();
                    Assert.assertEquals("Read request for a truncated offset did not start with a Truncated ReadResultEntryType.", ReadResultEntryType.Truncated, first.getType());
                    // Skip over until the first non-truncated offset.
                    expectedCurrentOffset = Math.max(expectedCurrentOffset, startOffset);
                    continue;
                }
                // Non-truncated entry; do the usual verifications.
                readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                BufferView readEntryContents = readEntry.getContent().join();
                byte[] actualData = readEntryContents.getCopy();
                AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
                expectedCurrentOffset += readEntryContents.getLength();
                // Every other read, determine if we should truncate or not.
                if (truncate) {
                    long truncateOffset;
                    if (segmentName.hashCode() % 2 == 0) {
                        // Truncate just beyond the current read offset.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset + 1);
                    } else {
                        // Truncate half of what we read so far.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset / 2 + 1);
                    }
                    startOffsets.put(segmentName, truncateOffset);
                    store.truncateStreamSegment(segmentName, truncateOffset, TIMEOUT).join();
                }
                truncate = !truncate;
            }
            Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
        }
    }
}

Example 20 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkSegmentReads.

private void checkSegmentReads(String segmentName, AtomicLong expectedCurrentOffset, long segmentLength, StreamSegmentStore store, byte[] expectedData) throws Exception {
    @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset.get(), (int) (segmentLength - expectedCurrentOffset.get()), TIMEOUT).join();
    Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
    // A more thorough read check is done in StreamSegmentContainerTests; here we just check if the data was merged correctly.
    while (readResult.hasNext()) {
        ReadResultEntry readEntry = readResult.next();
        AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength());
        Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset.get(), readEntry.getStreamSegmentOffset());
        if (!readEntry.getContent().isDone()) {
            readEntry.requestContent(TIMEOUT);
        }
        readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
        BufferView readEntryContents = readEntry.getContent().join();
        byte[] actualData = readEntryContents.getCopy();
        AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset.get(), actualData, 0, readEntryContents.getLength());
        expectedCurrentOffset.addAndGet(readEntryContents.getLength());
    }
    Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
}