Examples with InMemoryTaskQueueManager io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager used on opensource projects

Example 6 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testExceptionDuringCommit.

public void testExceptionDuringCommit(Exception exceptionToThrow, Class clazz, boolean skipCreate, boolean skipConcat) throws Exception {
    String testSegmentName = "test";
    String concatSegmentName = "concat";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Capture segment layout information, so that we can check that after aborted operation it is still unchanged.
    val expectedSegmentMetadata = TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName);
    val expectedChunkMetadataList = TestUtils.getChunkList(spyMetadataStore, testSegmentName);
    // Make sure mock is working
    Assert.assertEquals(10, expectedSegmentMetadata.getLength());
    Assert.assertEquals(5, expectedChunkMetadataList.size());
    Assert.assertEquals(expectedChunkMetadataList.get(0).getName(), expectedSegmentMetadata.getFirstChunk());
    Assert.assertEquals(expectedChunkMetadataList.get(4).getName(), expectedSegmentMetadata.getLastChunk());
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // Step 2: Increase epoch.
    chunkedSegmentStorage.initialize(2);
    val h2 = chunkedSegmentStorage.create(concatSegmentName, policy, null).get();
    chunkedSegmentStorage.write(h2, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.seal(h2, null).get();
    // Step 3: Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(exceptionToThrow);
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // Make sure 15 chunks in total were created and then 5 of them garbage collected later.
    verify(spyChunkStorage, times(15)).doCreate(anyString());
    // verify(spyChunkStorage, times(5)).doDelete(any());
    // seal.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("Seal succeeded when exception was expected.", chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // openWrite.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("openWrite succeeded when exception was expected.", chunkedSegmentStorage.openWrite(testSegmentName), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    // delete.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("delete succeeded when exception was expected.", chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // truncate.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("truncate succeeded when exception was expected.", chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 2, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    if (!skipCreate) {
        // create.
        doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
        AssertExtensions.assertFutureThrows("create succeeded when exception was expected.", chunkedSegmentStorage.create("foo", policy, null), ex -> clazz.equals(ex.getClass()));
    }
    if (!skipConcat) {
        // concat.
        doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
        AssertExtensions.assertFutureThrows("concat succeeded when exception was expected.", chunkedSegmentStorage.concat(h1, 10, h2.getSegmentName(), null), ex -> clazz.equals(ex.getClass()));
        TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
        TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    }
}

Example 7 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testStorageFullDuringConcat.

@Test
public void testStorageFullDuringConcat() throws Exception {
    String testSegmentName = "test";
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, null).get();
    val h2 = chunkedSegmentStorage.create("source", null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.write(h2, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.seal(h2, null).get();
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageFullException("Test");
    val clazz = StorageFullException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.concat(h1, 10, "source", null), ex -> clazz.equals(ex.getClass()));
}

Example 8 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SimpleStorageTests method createStorage.

/**
 * Creates a new instance of the Storage implementation to be tested. This will be cleaned up (via close()) upon
 * test termination.
 */
@Override
protected Storage createStorage() throws Exception {
    ScheduledExecutorService executor = executorService();
    synchronized (SimpleStorageTests.class) {
        if (null == chunkStorage) {
            chunkMetadataStore = getMetadataStore();
            chunkStorage = getChunkStorage();
        }
    }
    ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, chunkStorage, chunkMetadataStore, executor, getDefaultConfig());
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    return chunkedSegmentStorage;
}

Example 9 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testIsSystemSegment.

@Test
public void testIsSystemSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    int maxLength = 8;
    long epoch = 1;
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    val journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
    Assert.assertFalse(journal.isStorageSystemSegment("foo"));
    Assert.assertFalse(journal.isStorageSystemSegment("_system/foo"));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getStorageMetadataSegmentName(containerId)));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getAttributeSegmentName(NameUtils.getStorageMetadataSegmentName(containerId))));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getMetadataSegmentName(containerId)));
    Assert.assertTrue(journal.isStorageSystemSegment(NameUtils.getAttributeSegmentName(NameUtils.getMetadataSegmentName(containerId))));
}

Example 10 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testSimpleTruncation.

/**
 * Test simple chunk truncation.
 * We failover two times to test correct interaction between snapshot and system logs.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleTruncation() throws Exception {
    @Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
    int containerId = 42;
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    @Cleanup val garbageCollector1 = new GarbageCollector(containerId, chunkStorage, metadataStoreBeforeCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector1.initialize(new InMemoryTaskQueueManager()).join();
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 1;
    val policy = new SegmentRollingPolicy(2);
    val config = getDefaultConfigBuilder(policy).build();
    // Step 1: Initial set of additions
    SystemJournal systemJournalBefore = new SystemJournal(containerId, chunkStorage, metadataStoreBeforeCrash, garbageCollector1, config, executorService());
    systemJournalBefore.bootstrap(epoch, snapshotInfoStore).join();
    String lastChunk = null;
    long totalBytesWritten = 0;
    for (int i = 0; i < 10; i++) {
        String newChunk = "chunk" + i;
        val h = chunkStorage.createWithContent(newChunk, Math.toIntExact(policy.getMaxLength()), new ByteArrayInputStream(new byte[Math.toIntExact(policy.getMaxLength())])).get();
        totalBytesWritten += policy.getMaxLength();
        systemJournalBefore.commitRecord(SystemJournal.ChunkAddedRecord.builder().segmentName(systemSegmentName).offset(policy.getMaxLength() * i).newChunkName(newChunk).oldChunkName(lastChunk).build()).join();
        lastChunk = newChunk;
    }
    Assert.assertEquals(policy.getMaxLength() * 10, totalBytesWritten);
    // Step 2: First failover, truncate first 5 chunks.
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    @Cleanup val garbageCollector2 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector2.initialize(new InMemoryTaskQueueManager()).join();
    SystemJournal systemJournalAfter = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash, garbageCollector2, config, executorService());
    systemJournalAfter.bootstrap(2, snapshotInfoStore).join();
    TestUtils.checkSegmentLayout(metadataStoreAfterCrash, systemSegmentName, policy.getMaxLength(), 10);
    TestUtils.checkSegmentBounds(metadataStoreAfterCrash, systemSegmentName, 0, totalBytesWritten);
    // Truncate first five chunks
    for (int i = 0; i <= 10; i++) {
        val firstChunkIndex = i / policy.getMaxLength();
        systemJournalAfter.commitRecord(SystemJournal.TruncationRecord.builder().segmentName(systemSegmentName).offset(i).firstChunkName("chunk" + firstChunkIndex).startOffset(policy.getMaxLength() * firstChunkIndex).build()).join();
    }
    // Step 3: Second failover, truncate last 5 chunks.
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash2 = getMetadataStore();
    @Cleanup val garbageCollector3 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector3.initialize(new InMemoryTaskQueueManager()).join();
    SystemJournal systemJournalAfter2 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash2, garbageCollector3, config, executorService());
    systemJournalAfter2.bootstrap(3, snapshotInfoStore).join();
    TestUtils.checkSegmentLayout(metadataStoreAfterCrash2, systemSegmentName, policy.getMaxLength(), 5);
    TestUtils.checkSegmentBounds(metadataStoreAfterCrash2, systemSegmentName, 10, 20);
    // Truncate last five chunks
    for (int i = 10; i <= 20; i++) {
        val firstChunkIndex = i / policy.getMaxLength();
        systemJournalAfter2.commitRecord(SystemJournal.TruncationRecord.builder().segmentName(systemSegmentName).offset(i).firstChunkName("chunk" + firstChunkIndex).startOffset(policy.getMaxLength() * firstChunkIndex).build()).join();
    }
    // Step 4: third failover validate.
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash3 = getMetadataStore();
    @Cleanup val garbageCollector4 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector4.initialize(new InMemoryTaskQueueManager()).join();
    SystemJournal systemJournalAfter3 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash3, garbageCollector4, config, executorService());
    systemJournalAfter3.bootstrap(4, snapshotInfoStore).join();
    TestUtils.checkSegmentBounds(metadataStoreAfterCrash3, systemSegmentName, 20, 20);
}