Examples with SegmentRollingPolicy io.pravega.segmentstore.storage.SegmentRollingPolicy used on opensource projects

Example 6 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testParallelReadRequestsOnSingleSegmentWithReentry.

private void testParallelReadRequestsOnSingleSegmentWithReentry(int numberOfRequests, int threadPoolSize, boolean shouldBlock) throws Exception {
    String testSegmentName = "testSegment";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().indexBlockSize(3).build());
    if (!(testContext.metadataStore instanceof InMemoryMetadataStore)) {
        return;
    }
    CompletableFuture<Void> futureToWaitOn = shouldBlock ? new CompletableFuture<Void>() : CompletableFuture.completedFuture(null);
    // Step 1: Populate dummy system segment segment.
    // Write some data to system segment so that we can read it back in call back.
    val systemSegment = "SystemSegment";
    val hSystem = testContext.chunkedSegmentStorage.create(systemSegment, policy, null).get();
    testContext.chunkedSegmentStorage.write(hSystem, 0, new ByteArrayInputStream(new byte[1]), 1, null).join();
    // Step 2: Create test segment.
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h.getSegmentName(), testSegmentName);
    Assert.assertFalse(h.isReadOnly());
    // Step 3: Write some data to test segment.
    long writeAt = 0;
    for (int i = 1; i < 5; i++) {
        testContext.chunkedSegmentStorage.write(h, writeAt, new ByteArrayInputStream(new byte[i]), i, null).join();
        writeAt += i;
    }
    TestUtils.checkSegmentLayout(testContext.metadataStore, testSegmentName, 2, 5);
    TestUtils.checkSegmentBounds(testContext.metadataStore, testSegmentName, 0, 10);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, testSegmentName, 0, 10, true);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, testSegmentName);
    // Step 4: Setup call backs that read system segment on each read.
    // Set up a call back which will be invoked during get call.
    ((InMemoryMetadataStore) testContext.metadataStore).setReadCallback(transactionData -> {
        // Make sure we don't invoke read for system segment itself.
        if (!transactionData.getKey().equals(systemSegment)) {
            return futureToWaitOn.thenComposeAsync(v -> checkDataReadAsync(systemSegment, testContext, 0, 1, executorService()), executorService()).thenApplyAsync(v -> null, executorService());
        }
        return CompletableFuture.completedFuture(null);
    });
    // Step 5: Read back data concurrently.
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
    CompletableFuture[] futures = new CompletableFuture[numberOfRequests];
    for (int i = 0; i < numberOfRequests; i++) {
        CompletableFuture<Void> f = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).thenComposeAsync(info -> {
            Assert.assertFalse(info.isSealed());
            Assert.assertFalse(info.isDeleted());
            Assert.assertEquals(info.getName(), testSegmentName);
            Assert.assertEquals(info.getLength(), 10);
            Assert.assertEquals(info.getStartOffset(), 0);
            return checkDataReadAsync(testSegmentName, testContext, 0, 10, executor);
        }, executor);
        futures[i] = f;
    }
    if (shouldBlock) {
        futureToWaitOn.complete(null);
    }
    CompletableFuture.allOf(futures).join();
}

Example 7 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testWrite.

/**
 * Test Write.
 *
 * @throws Exception Exception if any.
 */
@Test
public void testWrite() throws Exception {
    String testSegmentName = "foo";
    @Cleanup TestContext testContext = getTestContext();
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    // Create
    val hWrite = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    HashSet<String> chunksBefore = new HashSet<>();
    chunksBefore.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    // Write some data.
    long writeAt = 0;
    for (int i = 1; i < 5; i++) {
        testContext.chunkedSegmentStorage.write(hWrite, writeAt, new ByteArrayInputStream(new byte[i]), i, null).join();
        writeAt += i;
    }
    HashSet<String> chunksAfter = new HashSet<>();
    chunksAfter.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    TestUtils.checkGarbageCollectionQueue(testContext.chunkedSegmentStorage, chunksBefore, chunksAfter);
    int total = 10;
    checkDataRead(testSegmentName, testContext, 0, total);
}

Example 8 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testSimpleScenario.

/**
 * Test simple scenario.
 *
 * @throws Exception Exception if any.
 */
@Test
public void testSimpleScenario() throws Exception {
    String testSegmentName = "foo";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext();
    testSimpleScenario(testSegmentName, policy, testContext);
}

Example 9 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testParallelSegmentOperationsWithReentry.

private void testParallelSegmentOperationsWithReentry(int numberOfRequests, int threadPoolSize, boolean shouldBlock) throws Exception {
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext();
    if (!(testContext.metadataStore instanceof InMemoryMetadataStore)) {
        return;
    }
    CompletableFuture<Void> futureToWaitOn = shouldBlock ? new CompletableFuture<Void>() : CompletableFuture.completedFuture(null);
    // Step 1: Populate system segment.
    // Write some data to system segment so that we can read it back in call back.
    val systemSegment = "SystemSegment";
    val h = testContext.chunkedSegmentStorage.create(systemSegment, policy, null).get();
    testContext.chunkedSegmentStorage.write(h, 0, new ByteArrayInputStream(new byte[1]), 1, null).join();
    // Step 2: Setup call backs.
    // Set up a call back which will be invoked during get call.
    ((InMemoryMetadataStore) testContext.metadataStore).setReadCallback(transactionData -> {
        // Make sure we don't invoke read for system segment itself.
        if (!transactionData.getKey().equals(systemSegment)) {
            return futureToWaitOn.thenComposeAsync(v -> checkDataReadAsync(systemSegment, testContext, 0, 1, executorService()), executorService()).thenApplyAsync(v -> null, executorService());
        }
        return CompletableFuture.completedFuture(null);
    });
    // Set up a call back which will be invoked during writeAll call.
    ((InMemoryMetadataStore) testContext.metadataStore).setWriteCallback(transactionDataList -> {
        // Make sure we don't invoke read for system segment itself.
        if (transactionDataList.stream().filter(t -> !t.getKey().equals(systemSegment)).findAny().isPresent()) {
            return futureToWaitOn.thenComposeAsync(v -> checkDataReadAsync(systemSegment, testContext, 0, 1, executorService()), executorService()).thenApplyAsync(v -> null, executorService());
        }
        return CompletableFuture.completedFuture(null);
    });
    // Step 3: Perform operations on multiple segments.
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
    CompletableFuture[] futures = new CompletableFuture[numberOfRequests];
    for (int i = 0; i < numberOfRequests; i++) {
        String testSegmentName = "test" + i;
        val f = testSimpleScenarioAsync(testSegmentName, policy, testContext, executor);
        futures[i] = f;
    }
    if (shouldBlock) {
        futureToWaitOn.complete(null);
    }
    CompletableFuture.allOf(futures).join();
}

Example 10 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testRepeatedTruncatesAtFullLength.

@Test
public void testRepeatedTruncatesAtFullLength() throws Exception {
    @Cleanup TestContext testContext = getTestContext();
    String testSegmentName = "testSegmentName";
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    // create.
    val h1 = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    long expectedLength = 0;
    for (int i = 1; i < 5; i++) {
        val info1 = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
        Assert.assertEquals(expectedLength, info1.getLength());
        Assert.assertEquals(info1.getLength(), info1.getStartOffset());
        // Write some data.
        byte[] buffer = new byte[i];
        testContext.chunkedSegmentStorage.write(h1, info1.getStartOffset(), new ByteArrayInputStream(buffer), buffer.length, null).join();
        val info2 = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
        expectedLength += i;
        Assert.assertEquals(expectedLength, info2.getLength());
        HashSet<String> chunksBefore = new HashSet<>();
        chunksBefore.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
        // Now truncate
        testContext.chunkedSegmentStorage.truncate(h1, info2.getLength(), null).join();
        // Validate info
        val metadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
        Assert.assertEquals(expectedLength, metadata.getLength());
        Assert.assertEquals(expectedLength, metadata.getStartOffset());
        Assert.assertEquals(expectedLength, metadata.getFirstChunkStartOffset());
        Assert.assertEquals(expectedLength, metadata.getLastChunkStartOffset());
        Assert.assertEquals(null, metadata.getLastChunk());
        Assert.assertEquals(null, metadata.getFirstChunk());
        TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, testSegmentName);
        HashSet<String> chunksAfter = new HashSet<>();
        chunksAfter.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
        TestUtils.checkGarbageCollectionQueue(testContext.chunkedSegmentStorage, chunksBefore, chunksAfter);
        // Validate Exceptions.
        val expectedLength2 = expectedLength;
        val h = testContext.chunkedSegmentStorage.openRead(testSegmentName).get();
        AssertExtensions.assertFutureThrows("read succeeded on invalid offset.", testContext.chunkedSegmentStorage.read(h, expectedLength - 1, new byte[1], 0, 1, null), ex -> ex instanceof StreamSegmentTruncatedException && ((StreamSegmentTruncatedException) ex).getStartOffset() == expectedLength2);
        AssertExtensions.assertFutureThrows("read succeeded on invalid offset.", testContext.chunkedSegmentStorage.read(h, expectedLength, new byte[1], 0, 1, null), ex -> ex instanceof IllegalArgumentException);
    }
}