Search in sources :

Example 1 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testNewChunkOnSuccessful.

/**
 * Test for chunk that is marked active but added as garbage.
 */
@Test
public void testNewChunkOnSuccessful() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    insertChunk(chunkStorage, "newChunk", dataSize);
    garbageCollector.trackNewChunk(TXN_ID, "newChunk").join();
    insertChunkMetadata(metadataStore, "newChunk", dataSize, 1);
    // Validate state before
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("newChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertTrue(chunkStorage.exists("newChunk").get());
    Assert.assertNotNull(getChunkMetadata(metadataStore, "newChunk"));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) lombok.val(lombok.val) CompletableFuture(java.util.concurrent.CompletableFuture) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Duration(java.time.Duration) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Example 2 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testSegmentDelete.

private void testSegmentDelete(int numChunks, int chunkSize, int maxBatchSize, int indexBlockSize) throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).indexBlockSize(// Create huge number of block index entries
    indexBlockSize).garbageCollectionTransactionBatchSize(// Keep batch size very low.
    maxBatchSize).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Large number of chunks with large number of index entries
    long[] chunks = new long[numChunks];
    Arrays.fill(chunks, chunkSize);
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, chunks, false, 0);
    val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    chunkNames.stream().forEach(chunkName -> Assert.assertTrue(chunkStorage.exists(chunkName).join()));
    garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1)).join();
    // Validate state after
    Assert.assertEquals(numChunks, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(numChunks, garbageCollector.getQueueSize().get());
    Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
    assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, chunkNames.toArray(new String[numChunks]));
    garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), numChunks)).join();
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    chunkNames.stream().forEach(chunkName -> Assert.assertFalse(chunkName + " should not exist", chunkStorage.exists(chunkName).join()));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) lombok.val(lombok.val) Duration(java.time.Duration) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore)

Example 3 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testInitialization.

/**
 * Test Initialization
 */
@Test
public void testInitialization() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    AssertExtensions.assertThrows("Should not allow null chunkStorage", () -> {
        @Cleanup val x = new GarbageCollector(containerId, null, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
    }, ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null metadataStore", () -> {
        @Cleanup val x = new GarbageCollector(containerId, chunkStorage, null, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
    }, ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null config", () -> {
        @Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, null, executorService(), System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
    }, ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null executorService", () -> {
        @Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, null, System::currentTimeMillis, d -> CompletableFuture.completedFuture(null));
    }, ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null currentTimeSupplier", () -> {
        @Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), null, d -> CompletableFuture.completedFuture(null));
    }, ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null delaySupplier", () -> {
        @Cleanup val x = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService(), System::currentTimeMillis, null);
    }, ex -> ex instanceof NullPointerException);
}
Also used : lombok.val(lombok.val) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Example 4 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class GarbageCollectorTests method testDeletedSegment.

/**
 * Test for segment that is marked inactive and added as garbage.
 */
@Test
public void testDeletedSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 0);
    val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1)).join();
    // Validate state after
    Assert.assertEquals(4, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(4, garbageCollector.getQueueSize().get());
    Assert.assertNull(getSegmentMetadata(metadataStore, "testSegment"));
    garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 10)).join();
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    chunkNames.stream().forEach(chunkName -> Assert.assertFalse(chunkName + " should not exist", chunkStorage.exists(chunkName).join()));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) AssertExtensions(io.pravega.test.common.AssertExtensions) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Duration(java.time.Duration) Timeout(org.junit.rules.Timeout) Mockito.doReturn(org.mockito.Mockito.doReturn) Before(org.junit.Before) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) NameUtils(io.pravega.shared.NameUtils) lombok.val(lombok.val) Test(org.junit.Test) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) TreeMap(java.util.TreeMap) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) lombok.val(lombok.val) CompletableFuture(java.util.concurrent.CompletableFuture) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) Duration(java.time.Duration) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Example 5 with ChunkMetadataStore

use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.

the class SystemJournalTests method testIsSystemSegment.

@Test
public void testIsSystemSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    int maxLength = 8;
    long epoch = 1;
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    val journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
    Assert.assertFalse(journal.isStorageSystemSegment("foo"));
    Assert.assertFalse(journal.isStorageSystemSegment("_system/foo"));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getStorageMetadataSegmentName(containerId)));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getAttributeSegmentName(NameUtils.getStorageMetadataSegmentName(containerId))));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getMetadataSegmentName(containerId)));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getAttributeSegmentName(NameUtils.getMetadataSegmentName(containerId))));
}
Also used : lombok.val(lombok.val) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) SegmentRollingPolicy(io.pravega.segmentstore.storage.SegmentRollingPolicy) InMemoryChunkStorage(io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage) Cleanup(lombok.Cleanup) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) Test(org.junit.Test)

Aggregations

ChunkMetadataStore (io.pravega.segmentstore.storage.metadata.ChunkMetadataStore)36 Cleanup (lombok.Cleanup)36 InMemoryChunkStorage (io.pravega.segmentstore.storage.mocks.InMemoryChunkStorage)35 InMemoryTaskQueueManager (io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager)35 lombok.val (lombok.val)35 Test (org.junit.Test)34 ByteArrayInputStream (java.io.ByteArrayInputStream)29 Preconditions (com.google.common.base.Preconditions)18 ChunkMetadata (io.pravega.segmentstore.storage.metadata.ChunkMetadata)18 ReadIndexBlockMetadata (io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata)18 SegmentMetadata (io.pravega.segmentstore.storage.metadata.SegmentMetadata)18 StatusFlags (io.pravega.segmentstore.storage.metadata.StatusFlags)18 StorageMetadataException (io.pravega.segmentstore.storage.metadata.StorageMetadataException)18 InMemoryMetadataStore (io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore)18 NameUtils (io.pravega.shared.NameUtils)18 AssertExtensions (io.pravega.test.common.AssertExtensions)18 ThreadPooledTestSuite (io.pravega.test.common.ThreadPooledTestSuite)18 Duration (java.time.Duration)18 Arrays (java.util.Arrays)18 Collections (java.util.Collections)18