Search in sources :

Example 21 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testMaxAttempts.

/**
 * Test for Max Attempts.
 */
@Test
public void testMaxAttempts() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    insertChunk(chunkStorage, "deletedChunk", dataSize);
    insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    chunkStorage.setReadOnly(chunkStorage.openWrite("deletedChunk").get(), true).join();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).garbageCollectionMaxAttempts(3).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("deletedChunk")).join();
    // Validate state before
    assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
    for (int i = 0; i < 3; i++) {
        val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
        Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
        Assert.assertEquals(1, garbageCollector.getQueueSize().get());
        garbageCollector.processBatch(list).join();
        // Validate state after
        assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
    }
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).size());
    Assert.assertEquals("deletedChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).peek().getName());
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) lombok.val(lombok.val) CompletableFuture(java.util.concurrent.CompletableFuture) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Duration(java.time.Duration) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Example 22 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testMetadataExceptionForSegmentPartialMetadataUpdate.

/**
 * Test for segment that for which metadata is partially updated already in previous attempt.
 */
@Test
public void testMetadataExceptionForSegmentPartialMetadataUpdate() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = spy(getMetadataStore());
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 0);
    val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
    // Simulate partial update of metadata.
    @Cleanup val txn = metadataStore.beginTransaction(false, "testSegment");
    val metadata = (ChunkMetadata) txn.get(chunkNames.stream().findFirst().get()).join();
    metadata.setActive(false);
    txn.update(metadata);
    txn.commit().join();
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(4, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(4, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 4)).join();
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    chunkNames.stream().forEach(chunkName -> Assert.assertFalse(chunkName + " should not exist", chunkStorage.exists(chunkName).join()));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) lombok.val(lombok.val) Duration(java.time.Duration) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) CompletableFuture(java.util.concurrent.CompletableFuture) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Example 23 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testStorageFullDuringWrite.

@Test
public void testStorageFullDuringWrite() throws Exception {
    String testSegmentName = "test";
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageFullException("Test", new IntentionalException());
    val clazz = StorageFullException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
}
Also used : lombok.val(lombok.val) BaseMetadataStore(io.pravega.segmentstore.storage.metadata.BaseMetadataStore) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) NoOpChunkStorage(io.pravega.segmentstore.storage.noop.NoOpChunkStorage) StorageNotPrimaryException(io.pravega.segmentstore.storage.StorageNotPrimaryException) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) StorageMetadataWritesFencedOutException(io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException) IntentionalException(io.pravega.test.common.IntentionalException) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StorageMetadataVersionMismatchException(io.pravega.segmentstore.storage.metadata.StorageMetadataVersionMismatchException) IntentionalException(io.pravega.test.common.IntentionalException) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) ByteArrayInputStream(java.io.ByteArrayInputStream) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) Test(org.junit.Test)

Example 24 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testUpdateStorageStatsException.

@Test
public void testUpdateStorageStatsException() throws Exception {
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup val spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    Exception exceptionToThrow = new ChunkStorageException("Intentional", "Intentional");
    doReturn(CompletableFuture.failedFuture(exceptionToThrow)).when(spyChunkStorage).doGetUsedSpaceAsync(any());
    // Should not throw an exception
    chunkedSegmentStorage.updateStorageStats().get();
}
Also used : lombok.val(lombok.val) BaseMetadataStore(io.pravega.segmentstore.storage.metadata.BaseMetadataStore) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) Cleanup(lombok.Cleanup) NoOpChunkStorage(io.pravega.segmentstore.storage.noop.NoOpChunkStorage) StorageNotPrimaryException(io.pravega.segmentstore.storage.StorageNotPrimaryException) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) StorageMetadataWritesFencedOutException(io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException) IntentionalException(io.pravega.test.common.IntentionalException) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StorageMetadataVersionMismatchException(io.pravega.segmentstore.storage.metadata.StorageMetadataVersionMismatchException) Test(org.junit.Test)

Example 25 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testIOExceptionDuringWrite.

@Test
public void testIOExceptionDuringWrite() throws Exception {
    String testSegmentName = "test";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageException("test", "Test Exception", new IOException("Test Exception"));
    val clazz = ChunkStorageException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
}
Also used : lombok.val(lombok.val) BaseMetadataStore(io.pravega.segmentstore.storage.metadata.BaseMetadataStore) SegmentRollingPolicy(io.pravega.segmentstore.storage.SegmentRollingPolicy) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) IOException(java.io.IOException) Cleanup(lombok.Cleanup) NoOpChunkStorage(io.pravega.segmentstore.storage.noop.NoOpChunkStorage) StorageNotPrimaryException(io.pravega.segmentstore.storage.StorageNotPrimaryException) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) StorageMetadataWritesFencedOutException(io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException) IntentionalException(io.pravega.test.common.IntentionalException) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StorageMetadataVersionMismatchException(io.pravega.segmentstore.storage.metadata.StorageMetadataVersionMismatchException) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) ByteArrayInputStream(java.io.ByteArrayInputStream) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) Test(org.junit.Test)

Aggregations

InMemoryTaskQueueManager (io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager)49 lombok.val (lombok.val)46 Cleanup (lombok.Cleanup)44 Test (org.junit.Test)40 ChunkMetadataStore (io.pravega.segmentstore.storage.metadata.ChunkMetadataStore)36 ByteArrayInputStream (java.io.ByteArrayInputStream)36 InMemoryChunkStorage (io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage)35 InMemoryMetadataStore (io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore)27 StorageMetadataException (io.pravega.segmentstore.storage.metadata.StorageMetadataException)25 CompletableFuture (java.util.concurrent.CompletableFuture)22 Preconditions (com.google.common.base.Preconditions)19 ChunkMetadata (io.pravega.segmentstore.storage.metadata.ChunkMetadata)19 ReadIndexBlockMetadata (io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata)19 SegmentMetadata (io.pravega.segmentstore.storage.metadata.SegmentMetadata)19 StatusFlags (io.pravega.segmentstore.storage.metadata.StatusFlags)19 NameUtils (io.pravega.shared.NameUtils)19 AssertExtensions (io.pravega.test.common.AssertExtensions)19 ThreadPooledTestSuite (io.pravega.test.common.ThreadPooledTestSuite)19 Duration (java.time.Duration)19 Arrays (java.util.Arrays)19