Examples with ChunkMetadataStore io.pravega.segmentstore.storage.metadata.ChunkMetadataStore used on opensource projects

Example 21 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testMaxAttemptsWithSegment.

/**
 * Test for Max Attempts.
 */
@Test
public void testMaxAttemptsWithSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = spy(getMetadataStore());
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] {}, false, 0);
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    for (int i = 0; i < 3; i++) {
        val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
        Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
        Assert.assertEquals(1, garbageCollector.getQueueSize().get());
        // Step 2: Inject fault.
        CompletableFuture f = new CompletableFuture();
        f.completeExceptionally(new StorageMetadataException("Test Exception"));
        doReturn(f).when(metadataStore).commit(any());
        garbageCollector.processBatch(list).join();
        // Validate state after
        Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
        Assert.assertEquals(1, garbageCollector.getQueueSize().get());
        Assert.assertNotNull(getSegmentMetadata(metadataStore, "testSegment"));
    }
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).size());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).peek().getName());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertNotNull(getSegmentMetadata(metadataStore, "testSegment"));
}

Example 22 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testNonExistentChunk.

/**
 * Test for chunk that does not exist in metadata but added as garbage.
 */
@Test
public void testNonExistentChunk() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertFalse(chunkStorage.exists("nonExistingChunk").join());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Collections.singleton("nonExistingChunk")).join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("nonExistingChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
}

Example 23 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testMetadataExceptionWithSegment.

/**
 * Test for segment that is marked active and added as garbage.
 */
@Test
public void testMetadataExceptionWithSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = spy(getMetadataStore());
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 0);
    val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    // Step 2: Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(new StorageMetadataException("Test Exception"));
    doReturn(f).when(metadataStore).commit(any());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertNotNull(getSegmentMetadata(metadataStore, "testSegment"));
    chunkNames.stream().forEach(chunkName -> Assert.assertTrue(chunkStorage.exists(chunkName).join()));
}

Example 24 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testNonExistentSegment.

/**
 * Test for segment that does not exist and added as garbage.
 */
@Test
public void testNonExistentSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
}

Example 25 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testInitializationInvalidArgs.

/**
 * Test Initialization
 */
@Test
public void testInitializationInvalidArgs() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
}