Examples with SegmentRollingPolicy io.pravega.segmentstore.storage.SegmentRollingPolicy used on opensource projects

Example 11 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testParallelSegmentOperations.

public void testParallelSegmentOperations(int numberOfRequests, int threadPoolSize) throws Exception {
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext();
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
    CompletableFuture[] futures = new CompletableFuture[numberOfRequests];
    for (int i = 0; i < numberOfRequests; i++) {
        String testSegmentName = "test" + i;
        val f = testSimpleScenarioAsync(testSegmentName, policy, testContext, executor);
        futures[i] = f;
    }
    CompletableFuture.allOf(futures).join();
}

Example 12 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testDeleteInvalidParameters.

@Test
public void testDeleteInvalidParameters() throws Exception {
    String testSegmentName = "foo";
    // Force rollover after each byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(1);
    @Cleanup TestContext testContext = getTestContext();
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.delete(null, null), ex -> ex instanceof IllegalArgumentException);
}

Example 13 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testSegmentNotExistsExceptionForDeleted.

/**
 * Test various operations on deleted segment.
 *
 * @throws Exception
 */
@Test
public void testSegmentNotExistsExceptionForDeleted() throws Exception {
    String testSegmentName = "foo";
    // Force rollover after each byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(1);
    @Cleanup TestContext testContext = getTestContext();
    Assert.assertFalse(testContext.chunkedSegmentStorage.exists(testSegmentName, null).get());
    // Step 1: Create segment.
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h.getSegmentName(), testSegmentName);
    Assert.assertFalse(h.isReadOnly());
    val segmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
    Assert.assertNotNull(segmentMetadata);
    Assert.assertEquals(segmentMetadata.getName(), testSegmentName);
    Assert.assertEquals(segmentMetadata.getKey(), testSegmentName);
    Assert.assertTrue(testContext.chunkedSegmentStorage.exists(testSegmentName, null).get());
    testContext.chunkedSegmentStorage.delete(h, null).join();
    Assert.assertFalse(testContext.chunkedSegmentStorage.exists(testSegmentName, null).get());
    val segmentMetadataAfterDelete = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
    Assert.assertFalse(segmentMetadataAfterDelete.isActive());
    AssertExtensions.assertFutureThrows("getStreamSegmentInfo succeeded on missing segment.", testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Seal succeeded on missing segment.", testContext.chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Seal succeeded on missing segment.", testContext.chunkedSegmentStorage.openWrite(testSegmentName), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Seal succeeded on missing segment.", testContext.chunkedSegmentStorage.openRead(testSegmentName), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 0, null), ex -> ex instanceof StreamSegmentNotExistsException);
}

Example 14 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testConcatUsingAppendsAfterWriteFailure.

private void testConcatUsingAppendsAfterWriteFailure(long maxRollingSize, long[] targetLayoutBefore, long[] sourceLayout, int[] chunksWithGarbageIndex, long[] targetLayoutAfter, long expectedLength) throws Exception {
    String targetSegmentName = "target";
    String sourceSegmentName = "source";
    // Force rollover after every 20 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(maxRollingSize);
    ChunkedSegmentStorageConfig config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().maxSizeLimitForConcat(100).minSizeLimitForConcat(100).indexBlockSize(3).build();
    @Cleanup TestContext testContext = getTestContext(config);
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setShouldSupportConcat(true);
    // Create target
    testContext.insertMetadata(targetSegmentName, maxRollingSize, 1, targetLayoutBefore);
    // Create source
    testContext.insertMetadata(sourceSegmentName, maxRollingSize, 1, sourceLayout);
    val hSource = testContext.chunkedSegmentStorage.openWrite(sourceSegmentName).get();
    testContext.chunkedSegmentStorage.seal(hSource, null).get();
    // Add some garbage data at the end of last chunk
    val lastChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, TestUtils.getSegmentMetadata(testContext.metadataStore, targetSegmentName).getLastChunk());
    testContext.chunkStorage.write(ChunkHandle.writeHandle(lastChunkMetadata.getName()), lastChunkMetadata.getLength(), 1, new ByteArrayInputStream(new byte[1])).join();
    // Write some garbage at the end.
    val sourceList = TestUtils.getChunkList(testContext.metadataStore, sourceSegmentName);
    for (int i : chunksWithGarbageIndex) {
        // Append some data to the last chunk to simulate partial write during failure
        val chunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, sourceList.get(i).getName());
        testContext.chunkStorage.write(ChunkHandle.writeHandle(chunkMetadata.getName()), chunkMetadata.getLength(), 1, new ByteArrayInputStream(new byte[1])).join();
    }
    val hTarget = testContext.chunkedSegmentStorage.openWrite(targetSegmentName).get();
    val concatAt = Arrays.stream(targetLayoutBefore).sum();
    testContext.chunkedSegmentStorage.concat(hTarget, concatAt, sourceSegmentName, null).join();
    val list = TestUtils.getChunkList(testContext.metadataStore, targetSegmentName);
    checkDataRead(targetSegmentName, testContext, 0, expectedLength);
    TestUtils.checkSegmentLayout(testContext.metadataStore, targetSegmentName, targetLayoutAfter);
    TestUtils.checkSegmentBounds(testContext.metadataStore, targetSegmentName, 0, expectedLength);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, targetSegmentName, 0, expectedLength, true);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, targetSegmentName);
}

Example 15 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testSegmentNotExistsExceptionForNonExistent.

/**
 * Test exceptions for opertions on non-existent chunk.
 */
@Test
public void testSegmentNotExistsExceptionForNonExistent() throws Exception {
    String testSegmentName = "foo";
    // Force rollover after each byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(1);
    @Cleanup TestContext testContext = getTestContext();
    Assert.assertFalse(testContext.chunkedSegmentStorage.exists(testSegmentName, null).get());
    AssertExtensions.assertFutureThrows("getStreamSegmentInfo succeeded on missing segment.", testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Seal succeeded on missing segment.", testContext.chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("openWrite succeeded on missing segment.", testContext.chunkedSegmentStorage.openWrite(testSegmentName), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("openRead succeeded on missing segment.", testContext.chunkedSegmentStorage.openRead(testSegmentName), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("write succeeded on missing segment.", testContext.chunkedSegmentStorage.write(SegmentStorageHandle.writeHandle(testSegmentName), 0, new ByteArrayInputStream(new byte[1]), 1, null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("read succeeded on missing segment.", testContext.chunkedSegmentStorage.read(SegmentStorageHandle.readHandle(testSegmentName), 0, new byte[1], 0, 1, null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.concat(SegmentStorageHandle.writeHandle(testSegmentName), 0, "inexistent", null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.delete(SegmentStorageHandle.writeHandle(testSegmentName), null), ex -> ex instanceof StreamSegmentNotExistsException);
    AssertExtensions.assertFutureThrows("Concat succeeded on missing segment.", testContext.chunkedSegmentStorage.truncate(SegmentStorageHandle.writeHandle(testSegmentName), 0, null), ex -> ex instanceof StreamSegmentNotExistsException);
}