Search in sources :

Example 6 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testExceptionDuringListSegments.

@Test
public void testExceptionDuringListSegments() throws Exception {
    Exception exceptionToThrow = new CompletionException(new StorageMetadataException("Test Exception"));
    val clazz = StorageMetadataException.class;
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG;
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    spyMetadataStore.setMaxEntriesInTxnBuffer(0);
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(exceptionToThrow);
    doReturn(f).when(spyMetadataStore).getAllEntries();
    doReturn(f).when(spyMetadataStore).getAllKeys();
    AssertExtensions.assertThrows("listSegments succeeded when exception was expected.", () -> chunkedSegmentStorage.listSegments().get(), ex -> clazz.equals(ex.getClass()));
}
Also used : lombok.val(lombok.val) BaseMetadataStore(io.pravega.segmentstore.storage.metadata.BaseMetadataStore) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) CompletableFuture(java.util.concurrent.CompletableFuture) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) CompletionException(java.util.concurrent.CompletionException) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) Cleanup(lombok.Cleanup) StorageNotPrimaryException(io.pravega.segmentstore.storage.StorageNotPrimaryException) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) StorageMetadataWritesFencedOutException(io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException) IntentionalException(io.pravega.test.common.IntentionalException) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StorageMetadataVersionMismatchException(io.pravega.segmentstore.storage.metadata.StorageMetadataVersionMismatchException) NoOpChunkStorage(io.pravega.segmentstore.storage.noop.NoOpChunkStorage) Test(org.junit.Test)

Example 7 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testExceptionDuringCommit.

public void testExceptionDuringCommit(Exception exceptionToThrow, Class clazz, boolean skipCreate, boolean skipConcat) throws Exception {
    String testSegmentName = "test";
    String concatSegmentName = "concat";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Capture segment layout information, so that we can check that after aborted operation it is still unchanged.
    val expectedSegmentMetadata = TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName);
    val expectedChunkMetadataList = TestUtils.getChunkList(spyMetadataStore, testSegmentName);
    // Make sure mock is working
    Assert.assertEquals(10, expectedSegmentMetadata.getLength());
    Assert.assertEquals(5, expectedChunkMetadataList.size());
    Assert.assertEquals(expectedChunkMetadataList.get(0).getName(), expectedSegmentMetadata.getFirstChunk());
    Assert.assertEquals(expectedChunkMetadataList.get(4).getName(), expectedSegmentMetadata.getLastChunk());
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // Step 2: Increase epoch.
    chunkedSegmentStorage.initialize(2);
    val h2 = chunkedSegmentStorage.create(concatSegmentName, policy, null).get();
    chunkedSegmentStorage.write(h2, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.seal(h2, null).get();
    // Step 3: Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(exceptionToThrow);
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // Make sure 15 chunks in total were created and then 5 of them garbage collected later.
    verify(spyChunkStorage, times(15)).doCreate(anyString());
    // verify(spyChunkStorage, times(5)).doDelete(any());
    // seal.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("Seal succeeded when exception was expected.", chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // openWrite.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("openWrite succeeded when exception was expected.", chunkedSegmentStorage.openWrite(testSegmentName), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    // delete.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("delete succeeded when exception was expected.", chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // truncate.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("truncate succeeded when exception was expected.", chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 2, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    if (!skipCreate) {
        // create.
        doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
        AssertExtensions.assertFutureThrows("create succeeded when exception was expected.", chunkedSegmentStorage.create("foo", policy, null), ex -> clazz.equals(ex.getClass()));
    }
    if (!skipConcat) {
        // concat.
        doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
        AssertExtensions.assertFutureThrows("concat succeeded when exception was expected.", chunkedSegmentStorage.concat(h1, 10, h2.getSegmentName(), null), ex -> clazz.equals(ex.getClass()));
        TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
        TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    }
}
Also used : lombok.val(lombok.val) BaseMetadataStore(io.pravega.segmentstore.storage.metadata.BaseMetadataStore) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) CompletableFuture(java.util.concurrent.CompletableFuture) SegmentRollingPolicy(io.pravega.segmentstore.storage.SegmentRollingPolicy) ByteArrayInputStream(java.io.ByteArrayInputStream) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) NoOpChunkStorage(io.pravega.segmentstore.storage.noop.NoOpChunkStorage)

Example 8 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testStorageFullDuringConcat.

@Test
public void testStorageFullDuringConcat() throws Exception {
    String testSegmentName = "test";
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, null).get();
    val h2 = chunkedSegmentStorage.create("source", null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.write(h2, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.seal(h2, null).get();
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageFullException("Test");
    val clazz = StorageFullException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.concat(h1, 10, "source", null), ex -> clazz.equals(ex.getClass()));
}
Also used : lombok.val(lombok.val) BaseMetadataStore(io.pravega.segmentstore.storage.metadata.BaseMetadataStore) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) NoOpChunkStorage(io.pravega.segmentstore.storage.noop.NoOpChunkStorage) StorageNotPrimaryException(io.pravega.segmentstore.storage.StorageNotPrimaryException) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) StorageMetadataWritesFencedOutException(io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException) IntentionalException(io.pravega.test.common.IntentionalException) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StorageMetadataVersionMismatchException(io.pravega.segmentstore.storage.metadata.StorageMetadataVersionMismatchException) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) ByteArrayInputStream(java.io.ByteArrayInputStream) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) Test(org.junit.Test)

Example 9 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class SystemJournalTests method getMetadataStore.

protected ChunkMetadataStore getMetadataStore() throws Exception {
    val metadataStore = new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    metadataStore.setReadCallback(transactionData -> {
        Assert.assertFalse("Attempt to read pinned metadata from store", transactionData.getKey().contains("_system/containers"));
        return CompletableFuture.completedFuture(null);
    });
    return metadataStore;
}
Also used : lombok.val(lombok.val) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore)

Example 10 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testStorageFullDuringWrite.

@Test
public void testStorageFullDuringWrite() throws Exception {
    String testSegmentName = "test";
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageFullException("Test", new IntentionalException());
    val clazz = StorageFullException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
}
Also used : lombok.val(lombok.val) BaseMetadataStore(io.pravega.segmentstore.storage.metadata.BaseMetadataStore) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) NoOpChunkStorage(io.pravega.segmentstore.storage.noop.NoOpChunkStorage) StorageNotPrimaryException(io.pravega.segmentstore.storage.StorageNotPrimaryException) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) StorageMetadataWritesFencedOutException(io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException) IntentionalException(io.pravega.test.common.IntentionalException) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) StorageMetadataException(io.pravega.segmentstore.storage.metadata.StorageMetadataException) StorageMetadataVersionMismatchException(io.pravega.segmentstore.storage.metadata.StorageMetadataVersionMismatchException) IntentionalException(io.pravega.test.common.IntentionalException) InMemoryTaskQueueManager(io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager) ByteArrayInputStream(java.io.ByteArrayInputStream) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) Test(org.junit.Test)

Aggregations

InMemoryMetadataStore (io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore)33 lombok.val (lombok.val)30 Test (org.junit.Test)25 Cleanup (lombok.Cleanup)20 BaseMetadataStore (io.pravega.segmentstore.storage.metadata.BaseMetadataStore)11 NoOpChunkStorage (io.pravega.segmentstore.storage.noop.NoOpChunkStorage)11 IntentionalException (io.pravega.test.common.IntentionalException)11 ByteArrayInputStream (java.io.ByteArrayInputStream)11 InMemoryTaskQueueManager (io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager)10 StorageFullException (io.pravega.segmentstore.storage.StorageFullException)9 StorageNotPrimaryException (io.pravega.segmentstore.storage.StorageNotPrimaryException)9 MockStorageMetadata (io.pravega.segmentstore.storage.mocks.MockStorageMetadata)9 CompletableFuture (java.util.concurrent.CompletableFuture)8 CompletionException (java.util.concurrent.CompletionException)8 SegmentRollingPolicy (io.pravega.segmentstore.storage.SegmentRollingPolicy)6 StorageMetadataException (io.pravega.segmentstore.storage.metadata.StorageMetadataException)6 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)6 StorageMetadataVersionMismatchException (io.pravega.segmentstore.storage.metadata.StorageMetadataVersionMismatchException)5 StorageMetadataWritesFencedOutException (io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException)5 IOException (java.io.IOException)5