Examples with AbstractInMemoryChunkStorage io.pravega.segmentstore.storage.mocks.AbstractInMemoryChunkStorage used on opensource projects

Example 1 with AbstractInMemoryChunkStorage

use of io.pravega.segmentstore.storage.mocks.AbstractInMemoryChunkStorage in project pravega by pravega.

the class ChunkedSegmentStorageTests method testBaseConcatWithDefragWithMinMaxLimits.

@Test
public void testBaseConcatWithDefragWithMinMaxLimits() throws Exception {
    // Set limits.
    val maxRollingSize = 30;
    ChunkedSegmentStorageConfig config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().maxSizeLimitForConcat(20).minSizeLimitForConcat(10).build();
    @Cleanup TestContext testContext = getTestContext(config);
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setShouldSupportConcat(true);
    // no-op.
    testBaseConcat(testContext, maxRollingSize, new long[] { 1 }, new long[] { 21, 21, 21 }, new long[] { 1, 21, 21, 21 });
    // no-op - max rollover size.
    testBaseConcat(testContext, maxRollingSize, new long[] { 30 }, new long[] { 29, 2 }, new long[] { 30, 29, 2 });
    // no-op - max rollover size.
    testBaseConcat(testContext, maxRollingSize, new long[] { 30 }, new long[] { 1, 2, 3, 4 }, new long[] { 30, 10 });
    // small chunks followed by normal chunks.
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 1, 1, 1, 3, 1, 1, 3, 1, 3 }, new long[] { 25 });
    // normal chunks followed by small chunks.
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 3, 1, 1, 1, 3, 1, 1, 3, 1 }, new long[] { 25 });
    // consecutive normal.
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 1, 3, 3, 3, 1, 2, 2 }, new long[] { 25 });
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 5, 5, 5 }, new long[] { 25 });
    // all small chunks.
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 2, 2, 2, 2, 2, 2, 2, 1 }, new long[] { 25 });
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 12, 3 }, new long[] { 25 });
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 13, 2 }, new long[] { 25 });
    // First chunk is greater than max concat size
    testBaseConcat(testContext, maxRollingSize, new long[] { 13 }, new long[] { 11, 1 }, new long[] { 25 });
    // First chunk is greater than max concat size
    testBaseConcat(testContext, maxRollingSize, new long[] { 13 }, new long[] { 10, 2 }, new long[] { 25 });
}

Example 2 with AbstractInMemoryChunkStorage

use of io.pravega.segmentstore.storage.mocks.AbstractInMemoryChunkStorage in project pravega by pravega.

the class ChunkedSegmentStorageTests method testBasicConcatWithDefrag.

@Test
public void testBasicConcatWithDefrag() throws Exception {
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().indexBlockSize(3).build());
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setShouldSupportAppend(true);
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setShouldSupportConcat(true);
    // Populate segments
    val sourceLayout = new long[] { 1, 2, 3, 4, 5 };
    val targetLayout = new long[] { 10 };
    val resultLayout = new long[] { 25 };
    int maxRollingLength = 1024;
    testBaseConcat(testContext, maxRollingLength, targetLayout, sourceLayout, resultLayout);
    return;
}

Example 3 with AbstractInMemoryChunkStorage

use of io.pravega.segmentstore.storage.mocks.AbstractInMemoryChunkStorage in project pravega by pravega.

the class ChunkedSegmentStorageTests method testConcatUsingAppendsAfterWriteFailure.

private void testConcatUsingAppendsAfterWriteFailure(long maxRollingSize, long[] targetLayoutBefore, long[] sourceLayout, int[] chunksWithGarbageIndex, long[] targetLayoutAfter, long expectedLength) throws Exception {
    String targetSegmentName = "target";
    String sourceSegmentName = "source";
    // Force rollover after every 20 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(maxRollingSize);
    ChunkedSegmentStorageConfig config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().maxSizeLimitForConcat(100).minSizeLimitForConcat(100).indexBlockSize(3).build();
    @Cleanup TestContext testContext = getTestContext(config);
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setShouldSupportConcat(true);
    // Create target
    testContext.insertMetadata(targetSegmentName, maxRollingSize, 1, targetLayoutBefore);
    // Create source
    testContext.insertMetadata(sourceSegmentName, maxRollingSize, 1, sourceLayout);
    val hSource = testContext.chunkedSegmentStorage.openWrite(sourceSegmentName).get();
    testContext.chunkedSegmentStorage.seal(hSource, null).get();
    // Add some garbage data at the end of last chunk
    val lastChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, TestUtils.getSegmentMetadata(testContext.metadataStore, targetSegmentName).getLastChunk());
    testContext.chunkStorage.write(ChunkHandle.writeHandle(lastChunkMetadata.getName()), lastChunkMetadata.getLength(), 1, new ByteArrayInputStream(new byte[1])).join();
    // Write some garbage at the end.
    val sourceList = TestUtils.getChunkList(testContext.metadataStore, sourceSegmentName);
    for (int i : chunksWithGarbageIndex) {
        // Append some data to the last chunk to simulate partial write during failure
        val chunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, sourceList.get(i).getName());
        testContext.chunkStorage.write(ChunkHandle.writeHandle(chunkMetadata.getName()), chunkMetadata.getLength(), 1, new ByteArrayInputStream(new byte[1])).join();
    }
    val hTarget = testContext.chunkedSegmentStorage.openWrite(targetSegmentName).get();
    val concatAt = Arrays.stream(targetLayoutBefore).sum();
    testContext.chunkedSegmentStorage.concat(hTarget, concatAt, sourceSegmentName, null).join();
    val list = TestUtils.getChunkList(testContext.metadataStore, targetSegmentName);
    checkDataRead(targetSegmentName, testContext, 0, expectedLength);
    TestUtils.checkSegmentLayout(testContext.metadataStore, targetSegmentName, targetLayoutAfter);
    TestUtils.checkSegmentBounds(testContext.metadataStore, targetSegmentName, 0, expectedLength);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, targetSegmentName, 0, expectedLength, true);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, targetSegmentName);
}

Example 4 with AbstractInMemoryChunkStorage

use of io.pravega.segmentstore.storage.mocks.AbstractInMemoryChunkStorage in project pravega by pravega.

the class ChunkedSegmentStorageTests method testFullStorage.

@Test
public void testFullStorage() throws Exception {
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().maxSafeStorageSize(1000).build());
    Assert.assertFalse(testContext.chunkedSegmentStorage.isSafeMode());
    val h = testContext.chunkedSegmentStorage.create("test", TIMEOUT).get();
    testContext.chunkedSegmentStorage.write(h, 0, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT).get();
    testContext.chunkedSegmentStorage.create("segment", TIMEOUT).get();
    testContext.chunkedSegmentStorage.create("_system/something", TIMEOUT).get();
    // Simulate storage full.
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setUsedSizeToReturn(1000);
    testContext.chunkedSegmentStorage.updateStorageStats().join();
    Assert.assertTrue(testContext.chunkedSegmentStorage.isSafeMode());
    // These operations should pass
    Assert.assertEquals(10, testContext.chunkedSegmentStorage.getStreamSegmentInfo("test", TIMEOUT).get().getLength());
    checkDataRead("test", testContext, 0, 10);
    val h3 = testContext.chunkedSegmentStorage.create("A", TIMEOUT).get();
    testContext.chunkedSegmentStorage.seal(h3, TIMEOUT).get();
    testContext.chunkedSegmentStorage.delete(h3, TIMEOUT).get();
    testContext.chunkedSegmentStorage.write(SegmentStorageHandle.writeHandle("_system/something"), 0, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT).get();
    // These operations should fail
    AssertExtensions.assertFutureThrows("write() should throw an exception", testContext.chunkedSegmentStorage.write(h, 10, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT), ex -> ex instanceof StorageFullException);
    AssertExtensions.assertFutureThrows("conact() should throw an exception", testContext.chunkedSegmentStorage.concat(h, 10, "A", TIMEOUT), ex -> ex instanceof StorageFullException);
    // Remove storage full
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setUsedSizeToReturn(50);
    testContext.chunkedSegmentStorage.updateStorageStats().join();
    Assert.assertFalse(testContext.chunkedSegmentStorage.isSafeMode());
    testContext.chunkedSegmentStorage.write(SegmentStorageHandle.writeHandle("test"), 10, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT).get();
    testContext.chunkedSegmentStorage.write(SegmentStorageHandle.writeHandle("segment"), 0, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT).get();
    testContext.chunkedSegmentStorage.write(SegmentStorageHandle.writeHandle("_system/something"), 10, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT).get();
    Assert.assertEquals(20, testContext.chunkedSegmentStorage.getStreamSegmentInfo("test", TIMEOUT).get().getLength());
    Assert.assertEquals(10, testContext.chunkedSegmentStorage.getStreamSegmentInfo("segment", TIMEOUT).get().getLength());
    Assert.assertEquals(20, testContext.chunkedSegmentStorage.getStreamSegmentInfo("_system/something", TIMEOUT).get().getLength());
    checkDataRead("test", testContext, 0, 20);
    checkDataRead("segment", testContext, 0, 10);
    val h4 = testContext.chunkedSegmentStorage.create("B", TIMEOUT).get();
    testContext.chunkedSegmentStorage.delete(h4, TIMEOUT).get();
    testContext.chunkedSegmentStorage.seal(SegmentStorageHandle.writeHandle("segment"), TIMEOUT).get();
    testContext.chunkedSegmentStorage.concat(SegmentStorageHandle.writeHandle("test"), 20, "segment", TIMEOUT).get();
}

Example 5 with AbstractInMemoryChunkStorage

use of io.pravega.segmentstore.storage.mocks.AbstractInMemoryChunkStorage in project pravega by pravega.

the class ChunkedSegmentStorageTests method testBaseConcatWithDefragWithMinMaxLimitsNoAppends.

@Test
public void testBaseConcatWithDefragWithMinMaxLimitsNoAppends() throws Exception {
    // Set limits.
    val maxRollingSize = 30;
    ChunkedSegmentStorageConfig config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().maxSizeLimitForConcat(20).minSizeLimitForConcat(10).appendEnabled(false).build();
    @Cleanup TestContext testContext = getTestContext(config);
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setShouldSupportConcat(true);
    // Normal case.
    testBaseConcat(testContext, maxRollingSize, new long[] { 11 }, new long[] { 12 }, new long[] { 23 });
    // Bigger than max allowed.
    testBaseConcat(testContext, maxRollingSize, new long[] { 10 }, new long[] { 20 }, new long[] { 10, 20 });
    // Target is bigger than max allowed after first concat.
    testBaseConcat(testContext, maxRollingSize, new long[] { 11 }, new long[] { 12, 13 }, new long[] { 23, 13 });
    // One of the chunks in the middle is smaller than min size allowed.
    testBaseConcat(testContext, maxRollingSize, new long[] { 11 }, new long[] { 12, 5, 13 }, new long[] { 23, 5, 13 });
    // All chunks are smaller, resultant chunk gets bigger than max size allowed.
    testBaseConcat(testContext, maxRollingSize, new long[] { 11 }, new long[] { 2, 2, 2, 2, 2, 2 }, new long[] { 21, 2 });
    // Chunks are already at max rolling size.
    testBaseConcat(testContext, maxRollingSize, new long[] { 30 }, new long[] { 2, 30, 2, 30, 2, 30 }, new long[] { 30, 2, 30, 2, 30, 2, 30 });
    // Test max rollover size.
    testBaseConcat(testContext, maxRollingSize, new long[] { 11 }, new long[] { 9, 10 }, new long[] { 30 });
    // Test max rollover size.
    testBaseConcat(testContext, maxRollingSize, new long[] { 20 }, new long[] { 10, 10 }, new long[] { 30, 10 });
}