use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testMaxAttempts.
/**
* Test for Max Attempts.
*/
@Test
public void testMaxAttempts() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
insertChunk(chunkStorage, "deletedChunk", dataSize);
insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
chunkStorage.setReadOnly(chunkStorage.openWrite("deletedChunk").get(), true).join();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).garbageCollectionMaxAttempts(3).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Add some garbage
garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("deletedChunk")).join();
// Validate state before
assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
for (int i = 0; i < 3; i++) {
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
}
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).size());
Assert.assertEquals("deletedChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).peek().getName());
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testMetadataExceptionForSegmentPartialMetadataUpdate.
/**
* Test for segment that for which metadata is partially updated already in previous attempt.
*/
@Test
public void testMetadataExceptionForSegmentPartialMetadataUpdate() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = spy(getMetadataStore());
int containerId = CONTAINER_ID;
int dataSize = 1;
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 0);
val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
// Simulate partial update of metadata.
@Cleanup val txn = metadataStore.beginTransaction(false, "testSegment");
val metadata = (ChunkMetadata) txn.get(chunkNames.stream().findFirst().get()).join();
metadata.setActive(false);
txn.update(metadata);
txn.commit().join();
// Add some garbage
garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
// Validate state before
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(4, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(4, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 4)).join();
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
chunkNames.stream().forEach(chunkName -> Assert.assertFalse(chunkName + " should not exist", chunkStorage.exists(chunkName).join()));
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class ChunkedSegmentStorageMockTests method testStorageFullDuringWrite.
@Test
public void testStorageFullDuringWrite() throws Exception {
String testSegmentName = "test";
@Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
@Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
@Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
chunkedSegmentStorage.initialize(1);
chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Step 1: Create segment and write some data.
val h1 = chunkedSegmentStorage.create(testSegmentName, null).get();
Assert.assertEquals(h1.getSegmentName(), testSegmentName);
Assert.assertFalse(h1.isReadOnly());
chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
// Step 2: Inject fault.
Exception exceptionToThrow = new ChunkStorageFullException("Test", new IntentionalException());
val clazz = StorageFullException.class;
doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class ChunkedSegmentStorageMockTests method testUpdateStorageStatsException.
@Test
public void testUpdateStorageStatsException() throws Exception {
@Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
@Cleanup val spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
@Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
chunkedSegmentStorage.initialize(1);
chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
Exception exceptionToThrow = new ChunkStorageException("Intentional", "Intentional");
doReturn(CompletableFuture.failedFuture(exceptionToThrow)).when(spyChunkStorage).doGetUsedSpaceAsync(any());
// Should not throw an exception
chunkedSegmentStorage.updateStorageStats().get();
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class ChunkedSegmentStorageMockTests method testIOExceptionDuringWrite.
@Test
public void testIOExceptionDuringWrite() throws Exception {
String testSegmentName = "test";
// Force rollover after every 2 byte.
SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
@Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
@Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
@Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
chunkedSegmentStorage.initialize(1);
chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Step 1: Create segment and write some data.
val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
Assert.assertEquals(h1.getSegmentName(), testSegmentName);
Assert.assertFalse(h1.isReadOnly());
chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
// Step 2: Inject fault.
Exception exceptionToThrow = new ChunkStorageException("test", "Test Exception", new IOException("Test Exception"));
val clazz = ChunkStorageException.class;
doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
}
Aggregations