Examples with InMemoryTaskQueueManager io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager used on opensource projects

Example 31 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testMetadataExceptionWithSegment.

/**
 * Test for segment that is marked active and added as garbage.
 */
@Test
public void testMetadataExceptionWithSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = spy(getMetadataStore());
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 0);
    val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    // Step 2: Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(new StorageMetadataException("Test Exception"));
    doReturn(f).when(metadataStore).commit(any());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertNotNull(getSegmentMetadata(metadataStore, "testSegment"));
    chunkNames.stream().forEach(chunkName -> Assert.assertTrue(chunkStorage.exists(chunkName).join()));
}

Example 32 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testNonExistentSegment.

/**
 * Test for segment that does not exist and added as garbage.
 */
@Test
public void testNonExistentSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
}

Example 33 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testInitializationInvalidArgs.

/**
 * Test Initialization
 */
@Test
public void testInitializationInvalidArgs() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
}

Example 34 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testIOExceptionDuringTruncate.

@Test
public void testIOExceptionDuringTruncate() throws Exception {
    String testSegmentName = "test";
    SegmentRollingPolicy policy = new SegmentRollingPolicy(100);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().relocateOnTruncateEnabled(true).minSizeForTruncateRelocationInbytes(1).minPercentForTruncateRelocation(50).storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(config, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    val taskQueueManager = new InMemoryTaskQueueManager();
    chunkedSegmentStorage.getGarbageCollector().initialize(taskQueueManager).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    val chunksListBefore = TestUtils.getChunkNameList(spyMetadataStore, testSegmentName);
    Assert.assertEquals(1, chunksListBefore.size());
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageException("test", "Test Exception", new IOException("Test Exception"));
    val clazz = ChunkStorageException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("truncate succeeded when exception was expected.", chunkedSegmentStorage.truncate(h1, 8, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.checkSegmentLayout(spyMetadataStore, testSegmentName, new long[] { 10 });
    val chunksListAfter = TestUtils.getChunkNameList(spyMetadataStore, testSegmentName);
    Assert.assertEquals(1, chunksListAfter.size());
    Assert.assertTrue(chunksListAfter.containsAll(chunksListBefore));
    Assert.assertEquals(2, chunkedSegmentStorage.getGarbageCollector().getQueueSize().get());
    val list = taskQueueManager.drain(chunkedSegmentStorage.getGarbageCollector().getTaskQueueName(), 2);
    Assert.assertTrue(chunksListBefore.contains(list.get(0).getName()));
    val nameTemplate = String.format("%s.E-%d-O-%d.", testSegmentName, chunkedSegmentStorage.getEpoch(), 8);
    Assert.assertTrue("New first chunk should be added to GC queue.", list.get(1).getName().startsWith(nameTemplate));
}

Example 35 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testExceptionDuringMetadataRead.

public void testExceptionDuringMetadataRead(Exception exceptionToThrow, Class clazz) throws Exception {
    String testSegmentName = "test";
    String concatSegmentName = "concat";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    spyMetadataStore.setMaxEntriesInTxnBuffer(0);
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Step 2: Increase epoch.
    chunkedSegmentStorage.initialize(2);
    val h2 = chunkedSegmentStorage.create(concatSegmentName, policy, null).get();
    chunkedSegmentStorage.write(h2, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.seal(h2, null).get();
    // Step 3: Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(exceptionToThrow);
    doReturn(f).when(spyMetadataStore).get(any(), anyString());
    // These calls are all read calls they can potentially run in parallel,
    // therefore we must force them to be synchronous to avoid org.mockito.exceptions.misusing.UnfinishedStubbingException
    AssertExtensions.assertThrows("write succeeded when exception was expected.", () -> chunkedSegmentStorage.write(h2, 10, new ByteArrayInputStream(new byte[10]), 10, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("Seal succeeded when exception was expected.", () -> chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("openWrite succeeded when exception was expected.", () -> chunkedSegmentStorage.openWrite(testSegmentName).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("delete succeeded when exception was expected.", () -> chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("truncate succeeded when exception was expected.", () -> chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 2, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("create succeeded when exception was expected.", () -> chunkedSegmentStorage.create("foo", policy, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("concat succeeded when exception was expected.", () -> chunkedSegmentStorage.concat(h1, 10, h2.getSegmentName(), null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("getStreamSegmentInfo succeeded  when exception was expected.", () -> chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("read  succeeded when exception was expected.", () -> chunkedSegmentStorage.read(SegmentStorageHandle.readHandle(testSegmentName), 0, new byte[1], 0, 1, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("openRead  succeeded when exception was expected.", () -> chunkedSegmentStorage.openRead(testSegmentName).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("exists  succeeded when exception was expected.", () -> chunkedSegmentStorage.exists(testSegmentName, null).get(), ex -> clazz.equals(ex.getClass()));
}