Examples with SegmentRollingPolicy io.pravega.segmentstore.storage.SegmentRollingPolicy used on opensource projects

Example 16 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testWriteSequential.

/**
 * Test Write for sequential scheduling.
 *
 * @throws Exception Exception if any.
 */
@Test
public void testWriteSequential() throws Exception {
    String testSegmentName = "foo";
    @Cleanup TestContext testContext = getTestContext();
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    // Create
    val hWrite = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    HashSet<String> chunksBefore = new HashSet<>();
    chunksBefore.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    // Write some data sequentially.
    val bytes = populate(100);
    ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
    for (int i = 0; i < bytes.length; i++) {
        futures.add(testContext.chunkedSegmentStorage.write(hWrite, i, new ByteArrayInputStream(bytes, i, 1), 1, null));
    }
    Futures.allOf(futures).join();
    HashSet<String> chunksAfter = new HashSet<>();
    chunksAfter.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    TestUtils.checkGarbageCollectionQueue(testContext.chunkedSegmentStorage, chunksBefore, chunksAfter);
    checkDataRead(testSegmentName, testContext, 0, bytes.length, bytes);
}

Example 17 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testSegmentNotExistsExceptionForDeletedSegment.

@Test
public void testSegmentNotExistsExceptionForDeletedSegment() throws Exception {
    String testSegmentName = "foo";
    // Force rollover after each byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(1);
    @Cleanup TestContext testContext = getTestContext();
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, null).get();
    Assert.assertTrue(testContext.chunkedSegmentStorage.exists(testSegmentName, null).get());
    // Delete
    testContext.chunkedSegmentStorage.delete(h, null).get();
    Assert.assertFalse(testContext.chunkedSegmentStorage.exists(testSegmentName, null).get());
    AssertExtensions.assertFutureThrows("getStreamSegmentInfo succeeded on missing segment.", testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Seal succeeded on missing segment.", testContext.chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("openWrite succeeded on missing segment.", testContext.chunkedSegmentStorage.openWrite(testSegmentName), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("openRead succeeded on missing segment.", testContext.chunkedSegmentStorage.openRead(testSegmentName), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("write succeeded on missing segment.", testContext.chunkedSegmentStorage.write(SegmentStorageHandle.writeHandle(testSegmentName), 0, new ByteArrayInputStream(new byte[1]), 1, null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("read succeeded on missing segment.", testContext.chunkedSegmentStorage.read(SegmentStorageHandle.readHandle(testSegmentName), 0, new byte[1], 0, 1, null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.concat(SegmentStorageHandle.writeHandle(testSegmentName), 0, "inexistent", null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 0, null), ex -> ex instanceof StreamSegmentNotExistsException);
}

Example 18 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testSimpleScenarioAsync.

private CompletableFuture<Void> testSimpleScenarioAsync(String testSegmentName, SegmentRollingPolicy policy, TestContext testContext, Executor executor) {
    // Step 1: Create segment.
    return testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).thenComposeAsync(h -> {
        Assert.assertEquals(h.getSegmentName(), testSegmentName);
        Assert.assertFalse(h.isReadOnly());
        // Check exists
        return testContext.chunkedSegmentStorage.exists(testSegmentName, null).thenApplyAsync(exists -> {
            Assert.assertTrue(exists);
            return null;
        }, executor).thenComposeAsync(v -> {
            // Check getStreamSegmentInfo.
            return testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).thenComposeAsync(info -> {
                Assert.assertFalse(info.isDeleted());
                Assert.assertEquals(info.getName(), testSegmentName);
                Assert.assertEquals(info.getLength(), 0);
                Assert.assertEquals(info.getStartOffset(), 0);
                return testContext.chunkedSegmentStorage.write(h, 0, new ByteArrayInputStream(new byte[10]), 10, null).thenComposeAsync(x -> checkDataReadAsync(testSegmentName, testContext, 0, 10, executor), executor).thenComposeAsync(x -> testContext.chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null), executor);
            }, executor);
        }, executor);
    }, executor);
}

Example 19 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testExceptionDuringCommit.

public void testExceptionDuringCommit(Exception exceptionToThrow, Class clazz, boolean skipCreate, boolean skipConcat) throws Exception {
    String testSegmentName = "test";
    String concatSegmentName = "concat";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Capture segment layout information, so that we can check that after aborted operation it is still unchanged.
    val expectedSegmentMetadata = TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName);
    val expectedChunkMetadataList = TestUtils.getChunkList(spyMetadataStore, testSegmentName);
    // Make sure mock is working
    Assert.assertEquals(10, expectedSegmentMetadata.getLength());
    Assert.assertEquals(5, expectedChunkMetadataList.size());
    Assert.assertEquals(expectedChunkMetadataList.get(0).getName(), expectedSegmentMetadata.getFirstChunk());
    Assert.assertEquals(expectedChunkMetadataList.get(4).getName(), expectedSegmentMetadata.getLastChunk());
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // Step 2: Increase epoch.
    chunkedSegmentStorage.initialize(2);
    val h2 = chunkedSegmentStorage.create(concatSegmentName, policy, null).get();
    chunkedSegmentStorage.write(h2, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.seal(h2, null).get();
    // Step 3: Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(exceptionToThrow);
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // Make sure 15 chunks in total were created and then 5 of them garbage collected later.
    verify(spyChunkStorage, times(15)).doCreate(anyString());
    // verify(spyChunkStorage, times(5)).doDelete(any());
    // seal.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("Seal succeeded when exception was expected.", chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // openWrite.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("openWrite succeeded when exception was expected.", chunkedSegmentStorage.openWrite(testSegmentName), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    // delete.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("delete succeeded when exception was expected.", chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    // truncate.
    doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
    AssertExtensions.assertFutureThrows("truncate succeeded when exception was expected.", chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 2, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
    TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    if (!skipCreate) {
        // create.
        doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
        AssertExtensions.assertFutureThrows("create succeeded when exception was expected.", chunkedSegmentStorage.create("foo", policy, null), ex -> clazz.equals(ex.getClass()));
    }
    if (!skipConcat) {
        // concat.
        doReturn(f).when(spyMetadataStore).commit(any(), anyBoolean(), anyBoolean());
        AssertExtensions.assertFutureThrows("concat succeeded when exception was expected.", chunkedSegmentStorage.concat(h1, 10, h2.getSegmentName(), null), ex -> clazz.equals(ex.getClass()));
        TestUtils.assertEquals(expectedSegmentMetadata, expectedChunkMetadataList, TestUtils.getSegmentMetadata(spyMetadataStore, testSegmentName), TestUtils.getChunkList(spyMetadataStore, testSegmentName));
        TestUtils.checkChunksExistInStorage(spyChunkStorage, spyMetadataStore, testSegmentName);
    }
}

Example 20 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithMultipleFailovers.

/**
 * Tests a scenario when there are multiple fail overs overs.
 * The test adds a few chunks to the system segments and then fails over.
 * After fail over the zombie instances continue to write junk data to both system segment and journal file.
 * The new instance should read the journal log file and recreate the layout of system segments.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithMultipleFailovers() throws Exception {
    val containerId = 42;
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    val policy = new SegmentRollingPolicy(100);
    val config = getDefaultConfigBuilder(policy).selfCheckEnabled(true).build();
    testSimpleBootstrapWithMultipleFailovers(containerId, chunkStorage, config, null);
}