use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testNewChunkOnSuccessful.
/**
* Test for chunk that is marked active but added as garbage.
*/
@Test
public void testNewChunkOnSuccessful() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Add some garbage
insertChunk(chunkStorage, "newChunk", dataSize);
garbageCollector.trackNewChunk(TXN_ID, "newChunk").join();
insertChunkMetadata(metadataStore, "newChunk", dataSize, 1);
// Validate state before
Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals("newChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertTrue(chunkStorage.exists("newChunk").get());
Assert.assertNotNull(getChunkMetadata(metadataStore, "newChunk"));
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testSegmentDelete.
private void testSegmentDelete(int numChunks, int chunkSize, int maxBatchSize, int indexBlockSize) throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).indexBlockSize(// Create huge number of block index entries
indexBlockSize).garbageCollectionTransactionBatchSize(// Keep batch size very low.
maxBatchSize).build();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Large number of chunks with large number of index entries
long[] chunks = new long[numChunks];
Arrays.fill(chunks, chunkSize);
insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, chunks, false, 0);
val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
// Add some garbage
garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
// Validate state before
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
chunkNames.stream().forEach(chunkName -> Assert.assertTrue(chunkStorage.exists(chunkName).join()));
garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1)).join();
// Validate state after
Assert.assertEquals(numChunks, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(numChunks, garbageCollector.getQueueSize().get());
Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, chunkNames.toArray(new String[numChunks]));
garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), numChunks)).join();
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
chunkNames.stream().forEach(chunkName -> Assert.assertFalse(chunkName + " should not exist", chunkStorage.exists(chunkName).join()));
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testInitialization.
/**
* Test Initialization
*/
@Test
public void testInitialization() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
AssertExtensions.assertThrows("Should not allow null chunkStorage", () -> {
@Cleanup val x = new GarbageCollector(containerId, null, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
}, ex -> ex instanceof NullPointerException);
AssertExtensions.assertThrows("Should not allow null metadataStore", () -> {
@Cleanup val x = new GarbageCollector(containerId, chunkStorage, null, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
}, ex -> ex instanceof NullPointerException);
AssertExtensions.assertThrows("Should not allow null config", () -> {
@Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, null, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
}, ex -> ex instanceof NullPointerException);
AssertExtensions.assertThrows("Should not allow null executorService", () -> {
@Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, null, System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
}, ex -> ex instanceof NullPointerException);
AssertExtensions.assertThrows("Should not allow null currentTimeSupplier", () -> {
@Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), null, d -> CompletableFuture.completedFuture(null));
}, ex -> ex instanceof NullPointerException);
AssertExtensions.assertThrows("Should not allow null delaySupplier", () -> {
@Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, null);
}, ex -> ex instanceof NullPointerException);
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testDeletedSegment.
/**
* Test for segment that is marked inactive and added as garbage.
*/
@Test
public void testDeletedSegment() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 0);
val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
// Add some garbage
garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
// Validate state before
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1)).join();
// Validate state after
Assert.assertEquals(4, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(4, garbageCollector.getQueueSize().get());
Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 10)).join();
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
chunkNames.stream().forEach(chunkName -> Assert.assertFalse(chunkName + " should not exist", chunkStorage.exists(chunkName).join()));
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class ChunkedSegmentStorageMockTests method testExceptionDuringListSegments.
@Test
public void testExceptionDuringListSegments() throws Exception {
Exception exceptionToThrow = new CompletionException(new StorageMetadataException("Test Exception"));
val clazz = StorageMetadataException.class;
val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG;
@Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
spyMetadataStore.setMaxEntriesInTxnBuffer(0);
@Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
@Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
chunkedSegmentStorage.initialize(1);
chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Inject fault.
CompletableFuture f = new CompletableFuture();
f.completeExceptionally(exceptionToThrow);
doReturn(f).when(spyMetadataStore).getAllEntries();
doReturn(f).when(spyMetadataStore).getAllKeys();
AssertExtensions.assertThrows("listSegments succeeded when exception was expected.", () -> chunkedSegmentStorage.listSegments().get(), ex -> clazz.equals(ex.getClass()));
}
Aggregations