Examples with ReusableLatch io.pravega.common.util.ReusableLatch used on opensource projects

Example 11 with ReusableLatch

use of io.pravega.common.util.ReusableLatch in project pravega by pravega.

the class ContainerReadIndexTests method testStorageFailedCacheInsert.

/**
 * Tests the ability to handle Cache/Index Update failures post a successful Storage Read.
 */
@Test
public void testStorageFailedCacheInsert() throws Exception {
    final int segmentLength = 1024;
    // Create a segment and write some data in Storage for it.
    @Cleanup TestContext context = new TestContext();
    ArrayList<Long> segmentIds = createSegments(context);
    createSegmentsInStorage(context);
    val testSegmentId = segmentIds.get(0);
    UpdateableSegmentMetadata sm = context.metadata.getStreamSegmentMetadata(testSegmentId);
    sm.setStorageLength(segmentLength);
    sm.setLength(segmentLength);
    context.storage.openWrite(sm.getName()).thenCompose(handle -> context.storage.write(handle, 0, new ByteArrayInputStream(new byte[segmentLength]), segmentLength, TIMEOUT)).join();
    // Keep track of inserted/deleted calls to the Cache, and "fail" the insert call.
    val inserted = new ReusableLatch();
    val insertedAddress = new AtomicInteger(CacheStorage.NO_ADDRESS);
    val deletedAddress = new AtomicInteger(Integer.MAX_VALUE);
    context.cacheStorage.insertCallback = address -> {
        // Immediately delete this data (prevent leaks).
        context.cacheStorage.delete(address);
        Assert.assertTrue(insertedAddress.compareAndSet(CacheStorage.NO_ADDRESS, address));
        inserted.release();
        throw new IntentionalException();
    };
    context.cacheStorage.deleteCallback = deletedAddress::set;
    // Trigger a read. The first read call will be served with data directly from Storage, so we expect it to be successful.
    @Cleanup ReadResult readResult = context.readIndex.read(testSegmentId, 0, segmentLength, TIMEOUT);
    ReadResultEntry entry = readResult.next();
    Assert.assertEquals("Unexpected ReadResultEntryType.", ReadResultEntryType.Storage, entry.getType());
    entry.requestContent(TIMEOUT);
    // This should complete without issues.
    entry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Verify that the cache insert attempt has been made
    inserted.await();
    Assert.assertNotEquals("Expected an insert attempt to have been made.", CacheStorage.NO_ADDRESS, insertedAddress.get());
    AssertExtensions.assertEventuallyEquals(CacheStorage.NO_ADDRESS, deletedAddress::get, TIMEOUT.toMillis());
}

Example 12 with ReusableLatch

use of io.pravega.common.util.ReusableLatch in project pravega by pravega.

the class AsyncStorageWrapperTests method testConcurrencyDifferentSegment.

/**
 * Tests the fact that different segment do not interfere (block) with each other for concurrent operations.
 */
@Test
public void testConcurrencyDifferentSegment() throws Exception {
    final String segment1 = "Segment1";
    final String segment2 = "Segment2";
    // Create a set of latches that can be used to detect when an operation was invoked and when to release it.
    val invoked = new HashMap<String, ReusableLatch>();
    val waitOn = new HashMap<String, ReusableLatch>();
    invoked.put(segment1, new ReusableLatch());
    invoked.put(segment2, new ReusableLatch());
    waitOn.put(segment1, new ReusableLatch());
    waitOn.put(segment2, new ReusableLatch());
    val innerStorage = new TestStorage((operation, segment) -> {
        invoked.get(segment).release();
        Exceptions.handleInterrupted(() -> waitOn.get(segment).await());
        return null;
    });
    @Cleanup val s = new AsyncStorageWrapper(innerStorage, executorService());
    // Begin executing one create.
    val futures = new ArrayList<CompletableFuture<?>>();
    futures.add(s.create(segment1, TIMEOUT));
    invoked.get(segment1).await(LOCK_TIMEOUT_MILLIS);
    Assert.assertEquals("Unexpected number of active segments.", 1, s.getSegmentWithOngoingOperationsCount());
    // Begin executing the second create and verify it is not blocked by the first one.
    futures.add(s.create(segment2, TIMEOUT));
    invoked.get(segment2).await(LOCK_TIMEOUT_MILLIS);
    Assert.assertEquals("Unexpected number of active segments.", 2, s.getSegmentWithOngoingOperationsCount());
    // Complete the first operation and await the second operation to begin executing, then release it too.
    waitOn.get(segment1).release();
    waitOn.get(segment2).release();
    // Wait for both operations to complete. This will re-throw any exceptions that may have occurred.
    allOf(futures).get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected final number of active segments.", 0, s.getSegmentWithOngoingOperationsCount());
}

Example 13 with ReusableLatch

use of io.pravega.common.util.ReusableLatch in project pravega by pravega.

the class AsyncStorageWrapperTests method testConcurrencyConcat.

/**
 * Tests the segment-based concurrency when concat is involved. In particular, that a concat() will wait for any pending
 * operations on each involved segment and that any subsequent operation on any of those segments will be queued up.
 */
@Test
public void testConcurrencyConcat() throws Exception {
    final String segment1 = "Segment1";
    final String segment2 = "Segment2";
    final BiFunction<String, String, String> joiner = (op, segment) -> op + "|" + segment;
    final String createSegment1Key = joiner.apply(TestStorage.CREATE, segment1);
    final String createSegment2Key = joiner.apply(TestStorage.CREATE, segment2);
    final String concatKey = joiner.apply(TestStorage.CONCAT, segment1 + "|" + segment2);
    final String writeSegment1Key = joiner.apply(TestStorage.WRITE, segment1);
    final String writeSegment2Key = joiner.apply(TestStorage.WRITE, segment2);
    // Create a set of latches that can be used to detect when an operation was invoked and when to release it.
    val invoked = new HashMap<String, ReusableLatch>();
    val waitOn = new HashMap<String, ReusableLatch>();
    invoked.put(createSegment1Key, new ReusableLatch());
    invoked.put(createSegment2Key, new ReusableLatch());
    invoked.put(concatKey, new ReusableLatch());
    invoked.put(writeSegment1Key, new ReusableLatch());
    invoked.put(writeSegment2Key, new ReusableLatch());
    waitOn.put(createSegment1Key, new ReusableLatch());
    waitOn.put(createSegment2Key, new ReusableLatch());
    waitOn.put(concatKey, new ReusableLatch());
    waitOn.put(writeSegment1Key, new ReusableLatch());
    waitOn.put(writeSegment2Key, new ReusableLatch());
    val innerStorage = new TestStorage((operation, segment) -> {
        invoked.get(joiner.apply(operation, segment)).release();
        Exceptions.handleInterrupted(() -> waitOn.get(joiner.apply(operation, segment)).await());
        return null;
    });
    @Cleanup val s = new AsyncStorageWrapper(innerStorage, executorService());
    // Issue two Create operations with the two segments and wait for both of them to be running.
    val futures = new ArrayList<CompletableFuture<?>>();
    futures.add(s.create(segment1, TIMEOUT));
    futures.add(s.create(segment2, TIMEOUT));
    invoked.get(createSegment1Key).await(LOCK_TIMEOUT_MILLIS);
    invoked.get(createSegment2Key).await(LOCK_TIMEOUT_MILLIS);
    Assert.assertEquals("Unexpected number of active segments.", 2, s.getSegmentWithOngoingOperationsCount());
    // Initiate the concat and complete one of the original operations, and verify the concat did not start.
    futures.add(s.concat(InMemoryStorage.newHandle(segment1, false), 0, segment2, TIMEOUT));
    waitOn.get(createSegment1Key).release();
    AssertExtensions.assertThrows("Concat was invoked while the at least one of the creates was running.", () -> invoked.get(concatKey).await(LOCK_TIMEOUT_MILLIS), ex -> ex instanceof TimeoutException);
    // Finish up the "source" create and verify the concat is released.
    waitOn.get(createSegment2Key).release();
    invoked.get(concatKey).await(TIMEOUT_MILLIS);
    // Add more operations after the concat and verify they are queued up (that they haven't started).
    futures.add(s.write(InMemoryStorage.newHandle(segment1, false), 0, new ByteArrayInputStream(new byte[0]), 0, TIMEOUT));
    futures.add(s.write(InMemoryStorage.newHandle(segment2, false), 0, new ByteArrayInputStream(new byte[0]), 0, TIMEOUT));
    AssertExtensions.assertThrows("Write(target) was invoked while concat was running", () -> invoked.get(writeSegment1Key).await(LOCK_TIMEOUT_MILLIS), ex -> ex instanceof TimeoutException);
    AssertExtensions.assertThrows("Write(source) was invoked while concat was running", () -> invoked.get(writeSegment2Key).await(LOCK_TIMEOUT_MILLIS), ex -> ex instanceof TimeoutException);
    Assert.assertEquals("Unexpected number of active segments.", 2, s.getSegmentWithOngoingOperationsCount());
    // Finish up the concat and verify the two writes are released.
    waitOn.get(concatKey).release();
    invoked.get(writeSegment1Key).await(LOCK_TIMEOUT_MILLIS);
    invoked.get(writeSegment2Key).await(LOCK_TIMEOUT_MILLIS);
    waitOn.get(writeSegment1Key).release();
    waitOn.get(writeSegment2Key).release();
    allOf(futures).get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected number of active segments.", 0, s.getSegmentWithOngoingOperationsCount());
}

Example 14 with ReusableLatch

use of io.pravega.common.util.ReusableLatch in project pravega by pravega.

the class ContainerReadIndexTests method testConcurrentReadTransactionStorageReadCacheFull.

/**
 * Tests the following scenario:
 * 1. Segment B has been merged into A
 * 2. We are executing a read on Segment A over a portion where B was merged into A.
 * 3. Concurrently with 2, a read on Segment B that went to LTS (possibly from the same result as before) wants to
 * insert into the Cache, but the cache is full. The Cache Manager would want to clean up the cache.
 * <p>
 * We want to ensure that there is no deadlock for this scenario.
 */
@Test
public void testConcurrentReadTransactionStorageReadCacheFull() throws Exception {
    // Must equal Cache Block size for easy eviction.
    val appendLength = 4 * 1024;
    val maxCacheSize = 2 * 1024 * 1024;
    // We set the policy's max size to a much higher value to avoid entering "essential-only" state.
    CachePolicy cachePolicy = new CachePolicy(2 * maxCacheSize, Duration.ZERO, Duration.ofMillis(1));
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG, cachePolicy, maxCacheSize);
    val rnd = new Random(0);
    // Create parent segment and one transaction
    long targetId = createSegment(0, context);
    long sourceId = createTransaction(1, context);
    val targetMetadata = context.metadata.getStreamSegmentMetadata(targetId);
    val sourceMetadata = context.metadata.getStreamSegmentMetadata(sourceId);
    createSegmentsInStorage(context);
    // Write something to the transaction; and immediately evict it.
    val append1 = new byte[appendLength];
    val append2 = new byte[appendLength];
    rnd.nextBytes(append1);
    rnd.nextBytes(append2);
    val allData = BufferView.builder().add(new ByteArraySegment(append1)).add(new ByteArraySegment(append2)).build();
    appendSingleWrite(sourceId, new ByteArraySegment(append1), context);
    sourceMetadata.setStorageLength(sourceMetadata.getLength());
    // Increment the generation.
    context.cacheManager.applyCachePolicy();
    // Write a second thing to the transaction, and do not evict it.
    appendSingleWrite(sourceId, new ByteArraySegment(append2), context);
    context.storage.openWrite(sourceMetadata.getName()).thenCompose(handle -> context.storage.write(handle, 0, allData.getReader(), allData.getLength(), TIMEOUT)).join();
    // Seal & Begin-merge the transaction (do not seal in storage).
    sourceMetadata.markSealed();
    targetMetadata.setLength(sourceMetadata.getLength());
    context.readIndex.beginMerge(targetId, 0L, sourceId);
    sourceMetadata.markMerged();
    sourceMetadata.markDeleted();
    // At this point, the first append in the transaction should be evicted, while the second one should still be there.
    @Cleanup val rr = context.readIndex.read(targetId, 0, (int) targetMetadata.getLength(), TIMEOUT);
    @Cleanup val cacheCleanup = new AutoCloseObject();
    @Cleanup("release") val insertingInCache = new ReusableLatch();
    @Cleanup("release") val finishInsertingInCache = new ReusableLatch();
    context.cacheStorage.beforeInsert = () -> {
        // Prevent a stack overflow.
        context.cacheStorage.beforeInsert = null;
        // Fill up the cache with garbage - this will cause an unrecoverable Cache Full event (which is what we want).
        int toFill = (int) (context.cacheStorage.getState().getMaxBytes() - context.cacheStorage.getState().getUsedBytes());
        int address = context.cacheStorage.insert(new ByteArraySegment(new byte[toFill]));
        cacheCleanup.onClose = () -> context.cacheStorage.delete(address);
        // Notify that we have inserted.
        insertingInCache.release();
        // Block (while holding locks) until notified.
        Exceptions.handleInterrupted(finishInsertingInCache::await);
    };
    // Begin a read process.
    // First read must be a storage read.
    val storageRead = rr.next();
    Assert.assertEquals(ReadResultEntryType.Storage, storageRead.getType());
    storageRead.requestContent(TIMEOUT);
    // Copy contents out; this is not affected by our cache insert block.
    byte[] readData1 = storageRead.getContent().join().slice(0, appendLength).getCopy();
    // Wait for the insert callback to be blocked on our latch.
    insertingInCache.await();
    // Continue with the read. We are now expecting a Cache Read. Do it asynchronously (new thread).
    val cacheReadFuture = CompletableFuture.supplyAsync(rr::next, executorService());
    // Notify the cache insert that it's time to release now.
    finishInsertingInCache.release();
    // Wait for the async read to finish and grab its contents.
    val cacheRead = cacheReadFuture.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals(ReadResultEntryType.Cache, cacheRead.getType());
    byte[] readData2 = cacheRead.getContent().join().slice(0, appendLength).getCopy();
    // Validate data was read correctly.
    val readData = BufferView.builder().add(new ByteArraySegment(readData1)).add(new ByteArraySegment(readData2)).build();
    Assert.assertEquals("Unexpected data written.", allData, readData);
}

Example 15 with ReusableLatch

use of io.pravega.common.util.ReusableLatch in project pravega by pravega.

the class ContainerReadIndexTests method testStorageReadsConcurrent.

private void testStorageReadsConcurrent(int offsetDeltaBetweenReads, int extraAllowedStorageReads, BiConsumerWithException<TestContext, UpdateableSegmentMetadata> executeBetweenReads, BiConsumerWithException<TestContext, UpdateableSegmentMetadata> finalCheck) throws Exception {
    val maxAllowedStorageReads = 2 + extraAllowedStorageReads;
    // Set a cache size big enough to prevent the Cache Manager from enabling "essential-only" mode due to over-utilization.
    val cachePolicy = new CachePolicy(10000, 0.01, 1.0, Duration.ofMillis(10), Duration.ofMillis(10));
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG, cachePolicy);
    // Create the segment
    val segmentId = createSegment(0, context);
    val metadata = context.metadata.getStreamSegmentMetadata(segmentId);
    context.storage.create(metadata.getName(), TIMEOUT).join();
    // Append some data to the Read Index.
    val dataInStorage = getAppendData(metadata.getName(), segmentId, 0, 0);
    metadata.setLength(dataInStorage.getLength());
    context.readIndex.append(segmentId, 0, dataInStorage);
    // Then write to Storage.
    context.storage.openWrite(metadata.getName()).thenCompose(handle -> context.storage.write(handle, 0, dataInStorage.getReader(), dataInStorage.getLength(), TIMEOUT)).join();
    metadata.setStorageLength(dataInStorage.getLength());
    // Then evict it from the cache.
    boolean evicted = context.cacheManager.applyCachePolicy();
    Assert.assertTrue("Expected an eviction.", evicted);
    @Cleanup("release") val firstReadBlocker = new ReusableLatch();
    @Cleanup("release") val firstRead = new ReusableLatch();
    @Cleanup("release") val secondReadBlocker = new ReusableLatch();
    @Cleanup("release") val secondRead = new ReusableLatch();
    val cacheInsertCount = new AtomicInteger();
    context.cacheStorage.insertCallback = address -> {
        if (cacheInsertCount.incrementAndGet() > 1) {
            Assert.fail("Too many cache inserts.");
        }
    };
    val storageReadCount = new AtomicInteger();
    context.storage.setReadInterceptor((segment, wrappedStorage) -> {
        int readCount = storageReadCount.incrementAndGet();
        if (readCount == 1) {
            firstRead.release();
            Exceptions.handleInterrupted(firstReadBlocker::await);
        } else if (readCount == 2) {
            secondRead.release();
            Exceptions.handleInterrupted(secondReadBlocker::await);
        } else if (readCount > maxAllowedStorageReads) {
            Assert.fail("Too many storage reads. Max allowed = " + maxAllowedStorageReads);
        }
    });
    // Initiate the first Storage Read.
    val read1Result = context.readIndex.read(segmentId, 0, dataInStorage.getLength(), TIMEOUT);
    val read1Data = new byte[dataInStorage.getLength()];
    val read1Future = CompletableFuture.runAsync(() -> read1Result.readRemaining(read1Data, TIMEOUT), executorService());
    // Wait for it to process.
    firstRead.await();
    // Initiate the second storage read.
    val read2Length = dataInStorage.getLength() - offsetDeltaBetweenReads;
    val read2Result = context.readIndex.read(segmentId, offsetDeltaBetweenReads, read2Length, TIMEOUT);
    val read2Data = new byte[read2Length];
    val read2Future = CompletableFuture.runAsync(() -> read2Result.readRemaining(read2Data, TIMEOUT), executorService());
    secondRead.await();
    // Unblock the first Storage Read and wait for it to complete.
    firstReadBlocker.release();
    read1Future.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Wait for the data from the first read to be fully added to the cache. Without this the subsequent append will not write to this entry.
    TestUtils.await(() -> {
        try {
            return context.readIndex.read(0, 0, dataInStorage.getLength(), TIMEOUT).next().getType() == ReadResultEntryType.Cache;
        } catch (StreamSegmentNotExistsException ex) {
            throw new CompletionException(ex);
        }
    }, 10, TIMEOUT.toMillis());
    // If there's anything to do between the two reads, do it now.
    executeBetweenReads.accept(context, metadata);
    // Unblock second Storage Read.
    secondReadBlocker.release();
    read2Future.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Perform final check.
    finalCheck.accept(context, metadata);
    Assert.assertEquals("Unexpected number of storage reads.", maxAllowedStorageReads, storageReadCount.get());
    Assert.assertEquals("Unexpected number of cache inserts.", 1, cacheInsertCount.get());
}