Examples with SegmentRollingPolicy io.pravega.segmentstore.storage.SegmentRollingPolicy used on opensource projects

Example 31 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithTwoFailovers.

/**
 * Tests a scenario when there are two fail overs.
 * The test adds a few chunks to the system segments and then fails over.
 * After fail over the zombie instance continues to write junk data to both system segment and journal file.
 * The new instance should read the journal log file and recreate the layout of system segments.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithTwoFailovers() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 1;
    val policy = new SegmentRollingPolicy(8);
    val config = getDefaultConfigBuilder(policy).build();
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    // Epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
    segmentStorage1.initialize(epoch);
    segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage1.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage1);
    checkSystemSegmentsLayout(segmentStorage1);
    // Simulate some writes to system segment, this should cause some new chunks being added.
    val h = segmentStorage1.openWrite(systemSegmentName).join();
    val b1 = "Hello".getBytes();
    segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
    offset += b1.length;
    checkSystemSegmentsLayout(segmentStorage1);
    // Epoch 2
    epoch++;
    @Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
    segmentStorage2.initialize(epoch);
    segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage2.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage2);
    checkSystemSegmentsLayout(segmentStorage2);
    val h2 = segmentStorage2.openWrite(systemSegmentName).join();
    // Write Junk Data to from first instance.
    segmentStorage1.write(h, offset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
    val b2 = " World".getBytes();
    segmentStorage2.write(h2, offset, new ByteArrayInputStream(b2), b2.length, null).join();
    offset += b2.length;
    checkSystemSegmentsLayout(segmentStorage2);
    val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(b1.length + b2.length, info.getLength());
    byte[] out = new byte[b1.length + b2.length];
    val hr = segmentStorage2.openRead(systemSegmentName).join();
    segmentStorage2.read(hr, 0, out, 0, b1.length + b2.length, null).join();
    Assert.assertEquals("Hello World", new String(out));
}

Example 32 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithMissingSnapshot.

@Test
public void testSimpleBootstrapWithMissingSnapshot() throws Exception {
    val containerId = 42;
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    val policy = new SegmentRollingPolicy(100);
    val config = getDefaultConfigBuilder(policy).selfCheckEnabled(true).build();
    try {
        testSimpleBootstrapWithMultipleFailovers(containerId, chunkStorage, config, epoch -> {
            val snapShotFile = NameUtils.getSystemJournalSnapshotFileName(containerId, epoch, 1);
            chunkStorage.delete(ChunkHandle.writeHandle(snapShotFile)).join();
        });
    } catch (Exception e) {
        val ex = Exceptions.unwrap(e);
        Assert.assertTrue(Exceptions.unwrap(e) instanceof IllegalStateException && ex.getMessage().contains("Chunk pointed by SnapshotInfo must exist"));
    }
}

Example 33 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class RollingStorageTestBase method testWriteOnRollOverBoundary.

@Test
public void testWriteOnRollOverBoundary() throws Exception {
    final String segmentName = "Segment";
    // Really small rolling length.
    final int maxLength = 3;
    val seq1 = "01234";
    val seq2 = "56789";
    val totalWriteLength = seq1.length() + seq2.length();
    @Cleanup val s = createStorage();
    s.initialize(1);
    val writeHandle = s.create(segmentName, new SegmentRollingPolicy(maxLength), TIMEOUT).thenCompose(v -> s.openWrite(segmentName)).join();
    val byteInputStream1 = new ByteArrayInputStream(seq1.getBytes());
    val byteInputStream2 = new ByteArrayInputStream(seq2.getBytes());
    val sequenceInputStream = new SequenceInputStream(byteInputStream1, byteInputStream2);
    // This write should cause 3 rollovers.
    s.write(writeHandle, 0, sequenceInputStream, totalWriteLength, TIMEOUT).join();
    // Check rollover actually happened as expected.
    if (useOldLayout) {
        RollingSegmentHandle checkHandle = (RollingSegmentHandle) s.openWrite(segmentName).join();
        val chunks = checkHandle.chunks();
        int numberOfRollovers = totalWriteLength / maxLength;
        Assert.assertEquals(numberOfRollovers + 1, chunks.size());
        for (int i = 0; i < numberOfRollovers; i++) {
            Assert.assertEquals(maxLength * i, chunks.get(i).getStartOffset());
            Assert.assertEquals(maxLength, chunks.get(i).getLength());
        }
        // Last chunk has index == numberOfRollovers, as list is 0 based.
        Assert.assertEquals(numberOfRollovers * maxLength, chunks.get(numberOfRollovers).getStartOffset());
        Assert.assertEquals(1, chunks.get(numberOfRollovers).getLength());
        // Now validate the contents written.
        val readHandle = s.openRead(segmentName).join();
        byte[] output = new byte[totalWriteLength];
        s.read(readHandle, 0, output, 0, totalWriteLength, TIMEOUT).join();
        Assert.assertEquals(seq1 + seq2, new String(output));
    }
}

Example 34 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testSimpleScenarioWithNonAppendProvider.

/**
 * Test simple scenario for storage that does not support any appends.
 *
 * @throws Exception
 */
@Test
public void testSimpleScenarioWithNonAppendProvider() throws Exception {
    String testSegmentName = "foo";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().indexBlockSize(3).build());
    ((AbstractInMemoryChunkStorage) testContext.chunkStorage).setShouldSupportAppend(false);
    // Step 1: Create segment.
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h.getSegmentName(), testSegmentName);
    Assert.assertFalse(h.isReadOnly());
    HashSet<String> chunksBefore = new HashSet<>();
    chunksBefore.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    // Check metadata is stored.
    val segmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
    Assert.assertNotNull(segmentMetadata);
    Assert.assertEquals(segmentMetadata.getName(), testSegmentName);
    Assert.assertEquals(segmentMetadata.getKey(), testSegmentName);
    // Check exists
    Assert.assertTrue(testContext.chunkedSegmentStorage.exists(testSegmentName, null).get());
    // Check getStreamSegmentInfo.
    SegmentProperties info = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
    Assert.assertFalse(info.isSealed());
    Assert.assertFalse(info.isDeleted());
    Assert.assertEquals(info.getName(), testSegmentName);
    Assert.assertEquals(info.getLength(), 0);
    Assert.assertEquals(info.getStartOffset(), 0);
    // Write some data.
    long writeAt = 0;
    for (int i = 1; i < 5; i++) {
        testContext.chunkedSegmentStorage.write(h, writeAt, new ByteArrayInputStream(new byte[i]), i, null).join();
        writeAt += i;
    }
    TestUtils.checkSegmentLayout(testContext.metadataStore, testSegmentName, new long[] { // First write
    1, // Second write
    2, // Third write
    2, // Third write
    1, // Fourth write
    2, // Fourth write
    2 });
    TestUtils.checkSegmentBounds(testContext.metadataStore, testSegmentName, 0, 10);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, testSegmentName, 0, 10, true);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, testSegmentName);
    // Check getStreamSegmentInfo.
    info = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
    Assert.assertFalse(info.isSealed());
    Assert.assertFalse(info.isDeleted());
    Assert.assertEquals(info.getName(), testSegmentName);
    Assert.assertEquals(info.getLength(), 10);
    Assert.assertEquals(info.getStartOffset(), 0);
    // Open write handle.
    val hWrite = testContext.chunkedSegmentStorage.openWrite(testSegmentName).get();
    Assert.assertEquals(hWrite.getSegmentName(), testSegmentName);
    Assert.assertFalse(hWrite.isReadOnly());
    testContext.chunkedSegmentStorage.write(hWrite, 10, new ByteArrayInputStream(new byte[4]), 4, null).join();
    TestUtils.checkSegmentLayout(testContext.metadataStore, testSegmentName, new long[] { // First write
    1, // Second write
    2, // Third write
    2, // Third write
    1, // Fourth write
    2, // Fourth write
    2, // Recent write
    2, // Recent write
    2 });
    TestUtils.checkSegmentBounds(testContext.metadataStore, testSegmentName, 0, 14);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, testSegmentName, 0, 14, true);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, testSegmentName);
    HashSet<String> chunksAfter = new HashSet<>();
    chunksAfter.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    TestUtils.checkGarbageCollectionQueue(testContext.chunkedSegmentStorage, chunksBefore, chunksAfter);
    info = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
    Assert.assertFalse(info.isSealed());
    Assert.assertFalse(info.isDeleted());
    Assert.assertEquals(info.getName(), testSegmentName);
    Assert.assertEquals(info.getLength(), 14);
    Assert.assertEquals(info.getStartOffset(), 0);
    // Make sure calling create again does not succeed
    AssertExtensions.assertFutureThrows("Create succeeded on missing segment.", testContext.chunkedSegmentStorage.create(testSegmentName, policy, null), ex -> ex instanceof StreamSegmentExistsException);
    testContext.chunkedSegmentStorage.delete(hWrite, null);
}

Example 35 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testSystemSegmentConcurrency.

/**
 * Test concurrent writes to storage system segments by simulating concurrent writes.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSystemSegmentConcurrency() throws Exception {
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().indexBlockSize(3).build());
    // Force parallel writes irrespective of thread pool size for tests themselves.
    val writeSize = 10;
    val numWrites = 10;
    val numOfStorageSystemSegments = SystemJournal.getChunkStorageSystemSegments(CONTAINER_ID).length;
    val data = new byte[numOfStorageSystemSegments][writeSize * numWrites];
    var futures = new ArrayList<CompletableFuture<Void>>();
    // To make sure all write operations are concurrent.
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(numOfStorageSystemSegments);
    for (int i = 0; i < numOfStorageSystemSegments; i++) {
        final int k = i;
        futures.add(CompletableFuture.runAsync(() -> {
            populate(data[k]);
            String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(CONTAINER_ID)[k];
            val h = testContext.chunkedSegmentStorage.create(systemSegmentName, null).join();
            // Init
            long offset = 0;
            for (int j = 0; j < numWrites; j++) {
                testContext.chunkedSegmentStorage.write(h, offset, new ByteArrayInputStream(data[k], writeSize * j, writeSize), writeSize, null).join();
                offset += writeSize;
            }
            val info = testContext.chunkedSegmentStorage.getStreamSegmentInfo(systemSegmentName, null).join();
            Assert.assertEquals(writeSize * numWrites, info.getLength());
            byte[] out = new byte[writeSize * numWrites];
            val hr = testContext.chunkedSegmentStorage.openRead(systemSegmentName).join();
            testContext.chunkedSegmentStorage.read(hr, 0, out, 0, writeSize * numWrites, null).join();
            Assert.assertArrayEquals(data[k], out);
        }, executor));
    }
    Futures.allOf(futures).join();
}