use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testMetadataException.
/**
* Test for metadata exception.
*/
@Test
public void testMetadataException() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
insertChunk(chunkStorage, "deletedChunk", dataSize);
insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
metadataStore.markFenced();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Add some garbage
garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("deletedChunk")).join();
// Validate state before
assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
// The chunk is deleted, but we have failure while deleting metadata.
Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method assertQueueEquals.
private void assertQueueEquals(String queueName, InMemoryTaskQueueManager queueManager, String[] expected) {
HashSet<String> visited = new HashSet<>();
val queue = queueManager.getTaskQueueMap().get(queueName).stream().peek(info -> visited.add(info.getName())).collect(Collectors.counting());
Assert.assertEquals(expected.length, visited.size());
for (String chunk : expected) {
Assert.assertTrue(visited.contains(chunk));
}
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testNewChunkOnFailure.
/**
* Test for chunk that does not exist in metadata but added as garbage.
*/
@Test
public void testNewChunkOnFailure() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
int dataSize = 1;
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Add some garbage
insertChunk(chunkStorage, "newChunk", dataSize);
garbageCollector.trackNewChunk(TXN_ID, "newChunk").join();
// Validate state before
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals("newChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
Assert.assertNull(getChunkMetadata(metadataStore, "newChunk"));
Assert.assertTrue(chunkStorage.exists("newChunk").get());
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertFalse(chunkStorage.exists("newChunk").get());
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class ChunkedRollingStorageTests method createStorage.
/**
* Creates a new instance of the Storage implementation to be tested. This will be cleaned up (via close()) upon
* test termination.
*/
@Override
protected Storage createStorage() throws Exception {
useOldLayout = false;
ScheduledExecutorService executor = executorService();
// Initialize
synchronized (ChunkedRollingStorageTests.class) {
if (null == chunkStorage) {
chunkMetadataStore = getMetadataStore();
chunkStorage = getChunkStorage();
}
}
val ret = new ChunkedSegmentStorage(CONTAINER_ID, chunkStorage, chunkMetadataStore, executor, getDefaultConfig());
ret.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
return ret;
}
Aggregations