Examples with SegmentRollingPolicy io.pravega.segmentstore.storage.SegmentRollingPolicy used on opensource projects

Example 36 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testReadWriteWithFixedSize.

private void testReadWriteWithFixedSize(String testSegmentName, TestContext testContext) throws Exception {
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val total = 28;
    val numberOfWrites = 7;
    val bytesToWrite = populate(total);
    // Write some data.
    int writeAt = 0;
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h.getSegmentName(), testSegmentName);
    Assert.assertFalse(h.isReadOnly());
    for (int i = 1; i <= numberOfWrites; i++) {
        testContext.chunkedSegmentStorage.write(h, writeAt, new ByteArrayInputStream(bytesToWrite, writeAt, i), i, null).join();
        writeAt += i;
    }
    TestUtils.checkSegmentLayout(testContext.metadataStore, testSegmentName, 2, 14);
    TestUtils.checkSegmentBounds(testContext.metadataStore, testSegmentName, 0, 28);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, testSegmentName);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, testSegmentName, 0, 28, true);
    checkDataRead(testSegmentName, testContext, 0, 28, bytesToWrite);
    checkDataReadPermutations(testContext, testSegmentName, total, numberOfWrites, bytesToWrite);
}

Example 37 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testIOExceptionDuringTruncate.

@Test
public void testIOExceptionDuringTruncate() throws Exception {
    String testSegmentName = "test";
    SegmentRollingPolicy policy = new SegmentRollingPolicy(100);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().relocateOnTruncateEnabled(true).minSizeForTruncateRelocationInbytes(1).minPercentForTruncateRelocation(50).storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(config, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    val taskQueueManager = new InMemoryTaskQueueManager();
    chunkedSegmentStorage.getGarbageCollector().initialize(taskQueueManager).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    val chunksListBefore = TestUtils.getChunkNameList(spyMetadataStore, testSegmentName);
    Assert.assertEquals(1, chunksListBefore.size());
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageException("test", "Test Exception", new IOException("Test Exception"));
    val clazz = ChunkStorageException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("truncate succeeded when exception was expected.", chunkedSegmentStorage.truncate(h1, 8, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.checkSegmentLayout(spyMetadataStore, testSegmentName, new long[] { 10 });
    val chunksListAfter = TestUtils.getChunkNameList(spyMetadataStore, testSegmentName);
    Assert.assertEquals(1, chunksListAfter.size());
    Assert.assertTrue(chunksListAfter.containsAll(chunksListBefore));
    Assert.assertEquals(2, chunkedSegmentStorage.getGarbageCollector().getQueueSize().get());
    val list = taskQueueManager.drain(chunkedSegmentStorage.getGarbageCollector().getTaskQueueName(), 2);
    Assert.assertTrue(chunksListBefore.contains(list.get(0).getName()));
    val nameTemplate = String.format("%s.E-%d-O-%d.", testSegmentName, chunkedSegmentStorage.getEpoch(), 8);
    Assert.assertTrue("New first chunk should be added to GC queue.", list.get(1).getName().startsWith(nameTemplate));
}

Example 38 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testExceptionDuringMetadataRead.

public void testExceptionDuringMetadataRead(Exception exceptionToThrow, Class clazz) throws Exception {
    String testSegmentName = "test";
    String concatSegmentName = "concat";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    spyMetadataStore.setMaxEntriesInTxnBuffer(0);
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Step 2: Increase epoch.
    chunkedSegmentStorage.initialize(2);
    val h2 = chunkedSegmentStorage.create(concatSegmentName, policy, null).get();
    chunkedSegmentStorage.write(h2, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    chunkedSegmentStorage.seal(h2, null).get();
    // Step 3: Inject fault.
    CompletableFuture f = new CompletableFuture();
    f.completeExceptionally(exceptionToThrow);
    doReturn(f).when(spyMetadataStore).get(any(), anyString());
    // These calls are all read calls they can potentially run in parallel,
    // therefore we must force them to be synchronous to avoid org.mockito.exceptions.misusing.UnfinishedStubbingException
    AssertExtensions.assertThrows("write succeeded when exception was expected.", () -> chunkedSegmentStorage.write(h2, 10, new ByteArrayInputStream(new byte[10]), 10, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("Seal succeeded when exception was expected.", () -> chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("openWrite succeeded when exception was expected.", () -> chunkedSegmentStorage.openWrite(testSegmentName).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("delete succeeded when exception was expected.", () -> chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("truncate succeeded when exception was expected.", () -> chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 2, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("create succeeded when exception was expected.", () -> chunkedSegmentStorage.create("foo", policy, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("concat succeeded when exception was expected.", () -> chunkedSegmentStorage.concat(h1, 10, h2.getSegmentName(), null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("getStreamSegmentInfo succeeded  when exception was expected.", () -> chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("read  succeeded when exception was expected.", () -> chunkedSegmentStorage.read(SegmentStorageHandle.readHandle(testSegmentName), 0, new byte[1], 0, 1, null).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("openRead  succeeded when exception was expected.", () -> chunkedSegmentStorage.openRead(testSegmentName).get(), ex -> clazz.equals(ex.getClass()));
    AssertExtensions.assertThrows("exists  succeeded when exception was expected.", () -> chunkedSegmentStorage.exists(testSegmentName, null).get(), ex -> clazz.equals(ex.getClass()));
}

Example 39 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSystemSegmentNoConcatAllowed.

@Test
public void testSystemSegmentNoConcatAllowed() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    int maxLength = 8;
    long epoch = 1;
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    val journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
    val systemSegmentName = NameUtils.getAttributeSegmentName(NameUtils.getMetadataSegmentName(containerId));
    Assert.assertTrue(journal.isStorageSystemSegment(systemSegmentName));
    // Init
    long offset = 0;
    // Start container with epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStore, executorService(), config);
    segmentStorage.initialize(epoch);
    segmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorage.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage);
    segmentStorage.create("test", null).get();
    AssertExtensions.assertFutureThrows("concat() should throw", segmentStorage.concat(SegmentStorageHandle.writeHandle(systemSegmentName), 0, "test", null), ex -> ex instanceof IllegalStateException);
    AssertExtensions.assertFutureThrows("concat() should throw", segmentStorage.concat(SegmentStorageHandle.writeHandle("test"), 0, systemSegmentName, null), ex -> ex instanceof IllegalStateException);
}

Example 40 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testInitialization.

@Test
public void testInitialization() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    int maxLength = 8;
    long epoch = 1;
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    // Init
    SystemJournal journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
    // Assert.assertEquals(epoch, journal.getEpoch());
    Assert.assertEquals(containerId, journal.getContainerId());
    Assert.assertEquals(policy.getMaxLength(), journal.getConfig().getStorageMetadataRollingPolicy().getMaxLength());
    // Assert.assertEquals(epoch, journal.getEpoch());
    Assert.assertEquals(0, journal.getCurrentFileIndex().get());
    Assert.assertEquals(NameUtils.INTERNAL_CONTAINER_PREFIX, journal.getSystemSegmentsPrefix());
    Assert.assertArrayEquals(SystemJournal.getChunkStorageSystemSegments(containerId), journal.getSystemSegments());
    journal.initialize();
}