Examples with SegmentRollingPolicy io.pravega.segmentstore.storage.SegmentRollingPolicy used on opensource projects

Example 41 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithPartialDataWrite.

/**
 * Tests a scenario when journal chunks may be partially written during failure.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithPartialDataWrite() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    if (!chunkStorage.supportsAppend()) {
        return;
    }
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 1;
    val policy = new SegmentRollingPolicy(4);
    val config = getDefaultConfigBuilder(policy).build();
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    // Epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStore, executorService(), config);
    segmentStorage1.initialize(epoch);
    segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage1.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage1);
    checkSystemSegmentsLayout(segmentStorage1);
    // Simulate some writes to system segment, this should cause some new chunks being added.
    val h = segmentStorage1.openWrite(systemSegmentName).join();
    val b1 = "Hello".getBytes();
    segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
    offset += b1.length;
    // Inject a fault by adding some garbage at the end
    checkSystemSegmentsLayout(segmentStorage1);
    val chunkFileName = NameUtils.getSystemJournalFileName(segmentStorage1.getSystemJournal().getContainerId(), segmentStorage1.getSystemJournal().getEpoch(), segmentStorage1.getSystemJournal().getCurrentFileIndex().get());
    val chunkInfo = chunkStorage.getInfo(chunkFileName);
    chunkStorage.write(ChunkHandle.writeHandle(chunkFileName), chunkInfo.get().getLength(), 1, new ByteArrayInputStream(new byte[1])).get();
    // This next write will encounter partially written chunk and is expected to start a new chunk.
    val b2 = " World".getBytes();
    segmentStorage1.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
    offset += b2.length;
    checkSystemSegmentsLayout(segmentStorage1);
    // Step 2
    // Start container with epoch 2
    epoch++;
    @Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
    segmentStorage2.initialize(epoch);
    segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage2.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage2);
    checkSystemSegmentsLayout(segmentStorage2);
    // Validate
    val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(b1.length + b2.length, info.getLength());
    byte[] out = new byte[b1.length + b2.length];
    val hr = segmentStorage2.openRead(systemSegmentName).join();
    segmentStorage2.read(hr, 0, out, 0, b1.length + b2.length, null).join();
    Assert.assertEquals("Hello World", new String(out));
}

Example 42 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSystemSegmentConcurrency.

/**
 * Test concurrent writes to storage system segments by simulating concurrent writes.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSystemSegmentConcurrency() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    int containerId = 42;
    int maxLength = 8;
    long epoch = 1;
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    val snapshotData = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> snapshotData.setSnapshotId(containerId, snapshotId), () -> snapshotData.getSnapshotId(containerId));
    // Start container with epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
    segmentStorage1.initialize(epoch);
    // Bootstrap
    segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorage1.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage1);
    checkSystemSegmentsLayout(segmentStorage1);
    // Simulate some writes to system segment, this should cause some new chunks being added.
    val writeSize = 10;
    val numWrites = 10;
    val numOfStorageSystemSegments = SystemJournal.getChunkStorageSystemSegments(containerId).length;
    val data = new byte[numOfStorageSystemSegments][writeSize * numWrites];
    var futures = new ArrayList<CompletableFuture<Void>>();
    val rnd = new Random(0);
    for (int i = 0; i < numOfStorageSystemSegments; i++) {
        final int k = i;
        futures.add(CompletableFuture.runAsync(() -> {
            rnd.nextBytes(data[k]);
            String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[k];
            val h = segmentStorage1.openWrite(systemSegmentName).join();
            // Init
            long offset = 0;
            for (int j = 0; j < numWrites; j++) {
                segmentStorage1.write(h, offset, new ByteArrayInputStream(data[k], writeSize * j, writeSize), writeSize, null).join();
                offset += writeSize;
            }
            val info = segmentStorage1.getStreamSegmentInfo(systemSegmentName, null).join();
            Assert.assertEquals(writeSize * numWrites, info.getLength());
            byte[] out = new byte[writeSize * numWrites];
            val hr = segmentStorage1.openRead(systemSegmentName).join();
            segmentStorage1.read(hr, 0, out, 0, writeSize * numWrites, null).join();
            Assert.assertArrayEquals(data[k], out);
        }, executorService()));
    }
    Futures.allOf(futures).join();
    // Step 2
    // Start container with epoch 2
    epoch++;
    @Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
    segmentStorage2.initialize(epoch);
    // Bootstrap
    segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorage2.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage2);
    checkSystemSegmentsLayout(segmentStorage2);
    // Validate
    for (int i = 0; i < numOfStorageSystemSegments; i++) {
        String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[i];
        val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
        Assert.assertEquals(writeSize * numWrites, info.getLength());
        byte[] out = new byte[writeSize * numWrites];
        val hr = segmentStorage2.openRead(systemSegmentName).join();
        segmentStorage2.read(hr, 0, out, 0, writeSize * numWrites, null).join();
        Assert.assertArrayEquals(data[i], out);
    }
}

Example 43 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testCommitInvalidArgs.

@Test
public void testCommitInvalidArgs() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    int maxLength = 8;
    long epoch = 1;
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    val journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
    AssertExtensions.assertThrows("commitRecords() should throw", () -> journal.commitRecord(null), ex -> ex instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("commitRecords() should throw", () -> journal.commitRecords(null), ex -> ex instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("commitRecords() should throw", () -> journal.commitRecords(new ArrayList<>()), ex -> ex instanceof IllegalArgumentException);
}

Example 44 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithTwoTruncates.

/**
 * Tests a scenario when there are two fail overs.
 * The test adds a few chunks to the system segments and then fails over.
 * It also truncates system segments.
 * The new instance should read the journal log file and recreate the layout of system segments.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithTwoTruncates() throws Exception {
    @Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 1;
    val policy = new SegmentRollingPolicy(8);
    val config = getDefaultConfigBuilder(policy).build();
    long offset = 0;
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    // Epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
    segmentStorage1.initialize(epoch);
    segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage1.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage1);
    checkSystemSegmentsLayout(segmentStorage1);
    // Simulate some writes to system segment, this should cause some new chunks being added.
    val h = segmentStorage1.openWrite(systemSegmentName).join();
    segmentStorage1.write(h, offset, new ByteArrayInputStream("JUNKJUNKJUNK".getBytes()), 12, null).join();
    offset += 12;
    val b1 = "Hello".getBytes();
    segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
    offset += b1.length;
    segmentStorage1.truncate(h, 6, null).join();
    // Epoch 2
    epoch++;
    @Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
    segmentStorage2.initialize(epoch);
    segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorage2.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage2);
    checkSystemSegmentsLayout(segmentStorage2);
    val h2 = segmentStorage2.openWrite(systemSegmentName).join();
    // Write Junk data to both system segment and journal file.
    segmentStorage1.write(h, offset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
    val b2 = " World".getBytes();
    segmentStorage2.write(h2, offset, new ByteArrayInputStream(b2), b2.length, null).join();
    offset += b2.length;
    segmentStorage2.truncate(h, 12, null).join();
    val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(b1.length + b2.length + 12, info.getLength());
    Assert.assertEquals(12, info.getStartOffset());
    byte[] out = new byte[b1.length + b2.length];
    val hr = segmentStorage2.openRead(systemSegmentName).join();
    segmentStorage2.read(hr, 12, out, 0, b1.length + b2.length, null).join();
    Assert.assertEquals("Hello World", new String(out));
}

Example 45 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageTests method testWriteAfterWriteFailure.

/**
 * Test Write after repeated failure.
 *
 * @throws Exception Exception if any.
 */
@Test
public void testWriteAfterWriteFailure() throws Exception {
    String testSegmentName = "foo";
    @Cleanup TestContext testContext = getTestContext();
    // Force rollover after every 20 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(20);
    // Create
    val hWrite = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    HashSet<String> chunksBefore = new HashSet<>();
    chunksBefore.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    // Write some data.
    long writeAt = 0;
    for (int i = 1; i < 5; i++) {
        testContext.chunkedSegmentStorage.write(hWrite, writeAt, new ByteArrayInputStream(new byte[i]), i, null).join();
        // Append some data to the last chunk to simulate partial write during failure
        val lastChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName).getLastChunk());
        testContext.chunkStorage.write(ChunkHandle.writeHandle(lastChunkMetadata.getName()), lastChunkMetadata.getLength(), 1, new ByteArrayInputStream(new byte[1])).join();
        writeAt += i;
    }
    checkDataRead(testSegmentName, testContext, 0, 10);
    TestUtils.checkSegmentLayout(testContext.metadataStore, testSegmentName, new long[] { 1, 2, 3, 4 });
    TestUtils.checkSegmentBounds(testContext.metadataStore, testSegmentName, 0, 10);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, testSegmentName, 0, 10, true);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, testSegmentName);
    HashSet<String> chunksAfter = new HashSet<>();
    chunksAfter.addAll(TestUtils.getChunkNameList(testContext.metadataStore, testSegmentName));
    TestUtils.checkGarbageCollectionQueue(testContext.chunkedSegmentStorage, chunksBefore, chunksAfter);
}