Search in sources :

Example 46 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testMetadataException.

/**
 * Test for metadata exception.
 */
@Test
public void testMetadataException() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    insertChunk(chunkStorage, "deletedChunk", dataSize);
    insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    metadataStore.markFenced();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("deletedChunk")).join();
    // Validate state before
    assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
    // The chunk is deleted, but we have failure while deleting metadata.
    Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) lombok.val(lombok.val) CompletableFuture(java.util.concurrent.CompletableFuture) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Duration(java.time.Duration) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Example 47 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method assertQueueEquals.

private void assertQueueEquals(String queueName, InMemoryTaskQueueManager queueManager, String[] expected) {
    HashSet<String> visited = new HashSet<>();
    val queue = queueManager.getTaskQueueMap().get(queueName).stream().peek(info -> visited.add(info.getName())).collect(Collectors.counting());
    Assert.assertEquals(expected.length, visited.size());
    for (String chunk : expected) {
        Assert.assertTrue(visited.contains(chunk));
    }
}
Also used : lombok.val(lombok.val) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) HashSet(java.util.HashSet)

Example 48 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testNewChunkOnFailure.

/**
 * Test for chunk that does not exist in metadata but added as garbage.
 */
@Test
public void testNewChunkOnFailure() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    int dataSize = 1;
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    insertChunk(chunkStorage, "newChunk", dataSize);
    garbageCollector.trackNewChunk(TXN_ID, "newChunk").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("newChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    Assert.assertNull(getChunkMetadata(metadataStore, "newChunk"));
    Assert.assertTrue(chunkStorage.exists("newChunk").get());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertFalse(chunkStorage.exists("newChunk").get());
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) lombok.val(lombok.val) CompletableFuture(java.util.concurrent.CompletableFuture) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Duration(java.time.Duration) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Example 49 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedRollingStorageTests method createStorage.

/**
 * Creates a new instance of the Storage implementation to be tested. This will be cleaned up (via close()) upon
 * test termination.
 */
@Override
protected Storage createStorage() throws Exception {
    useOldLayout = false;
    ScheduledExecutorService executor = executorService();
    // Initialize
    synchronized (ChunkedRollingStorageTests.class) {
        if (null == chunkStorage) {
            chunkMetadataStore = getMetadataStore();
            chunkStorage = getChunkStorage();
        }
    }
    val ret = new ChunkedSegmentStorage(CONTAINER_ID, chunkStorage, chunkMetadataStore, executor, getDefaultConfig());
    ret.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    return ret;
}
Also used : lombok.val(lombok.val) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager)

Aggregations

InMemoryTaskQueueManager (io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager)49 lombok.val (lombok.val)46 Cleanup (lombok.Cleanup)44 Test (org.junit.Test)40 ChunkMetadataStore (io.pravega.segmentstore.storage.metadata.ChunkMetadataStore)36 ByteArrayInputStream (java.io.ByteArrayInputStream)36 InMemoryChunkStorage (io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage)35 InMemoryMetadataStore (io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore)27 StorageMetadataException (io.pravega.segmentstore.storage.metadata.StorageMetadataException)25 CompletableFuture (java.util.concurrent.CompletableFuture)22 Preconditions (com.google.common.base.Preconditions)19 ChunkMetadata (io.pravega.segmentstore.storage.metadata.ChunkMetadata)19 ReadIndexBlockMetadata (io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata)19 SegmentMetadata (io.pravega.segmentstore.storage.metadata.SegmentMetadata)19 StatusFlags (io.pravega.segmentstore.storage.metadata.StatusFlags)19 NameUtils (io.pravega.shared.NameUtils)19 AssertExtensions (io.pravega.test.common.AssertExtensions)19 ThreadPooledTestSuite (io.pravega.test.common.ThreadPooledTestSuite)19 Duration (java.time.Duration)19 Arrays (java.util.Arrays)19