Examples with SegmentRollingPolicy io.pravega.segmentstore.storage.SegmentRollingPolicy used on opensource projects

Example 26 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testInitializationInvalidArgs.

@Test
public void testInitializationInvalidArgs() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    int maxLength = 8;
    long epoch = 1;
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    AssertExtensions.assertThrows("Should not allow null chunkStorage", () -> new SystemJournal(containerId, null, metadataStore, garbageCollector, config, executorService()), ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null metadataStore", () -> new SystemJournal(containerId, chunkStorage, null, garbageCollector, config, executorService()), ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null policy", () -> new SystemJournal(containerId, chunkStorage, metadataStore, null, config, executorService()), ex -> ex instanceof NullPointerException);
    AssertExtensions.assertThrows("Should not allow null config", () -> new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, null, executorService()), ex -> ex instanceof NullPointerException);
}

Example 27 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSimpleOperationSequence.

/**
 * Test simple series of operations.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleOperationSequence() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
    int containerId = 42;
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    @Cleanup val garbageCollector1 = new GarbageCollector(containerId, chunkStorage, metadataStoreBeforeCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector1.initialize(new InMemoryTaskQueueManager()).join();
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 1;
    val policy = new SegmentRollingPolicy(2);
    val config = getDefaultConfigBuilder(policy).build();
    // Inital set of additions
    SystemJournal systemJournalBefore = new SystemJournal(containerId, chunkStorage, metadataStoreBeforeCrash, garbageCollector1, config, executorService());
    systemJournalBefore.bootstrap(epoch, snapshotInfoStore).join();
    String lastChunk = null;
    long totalBytesWritten = 0;
    for (int i = 0; i < 10; i++) {
        String newChunk = "chunk" + i;
        val h = chunkStorage.createWithContent(newChunk, Math.toIntExact(policy.getMaxLength()), new ByteArrayInputStream(new byte[Math.toIntExact(policy.getMaxLength())])).get();
        totalBytesWritten += policy.getMaxLength();
        systemJournalBefore.commitRecord(SystemJournal.ChunkAddedRecord.builder().segmentName(systemSegmentName).offset(policy.getMaxLength() * i).newChunkName(newChunk).oldChunkName(lastChunk).build()).join();
        lastChunk = newChunk;
    }
    Assert.assertEquals(policy.getMaxLength() * 10, totalBytesWritten);
    // Failover
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    @Cleanup val garbageCollector2 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector2.initialize(new InMemoryTaskQueueManager()).join();
    SystemJournal systemJournalAfter = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash, garbageCollector2, config, executorService());
    systemJournalAfter.bootstrap(epoch + 1, snapshotInfoStore).join();
    TestUtils.checkSegmentLayout(metadataStoreAfterCrash, systemSegmentName, policy.getMaxLength(), 10);
    TestUtils.checkSegmentBounds(metadataStoreAfterCrash, systemSegmentName, 0, totalBytesWritten);
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash2 = getMetadataStore();
    @Cleanup val garbageCollector3 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector3.initialize(new InMemoryTaskQueueManager()).join();
    SystemJournal systemJournalAfter2 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash2, garbageCollector3, config, executorService());
    systemJournalAfter2.bootstrap(epoch + 2, snapshotInfoStore).join();
    TestUtils.checkSegmentLayout(metadataStoreAfterCrash2, systemSegmentName, policy.getMaxLength(), 10);
}

Example 28 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithMultipleFailoversWithTruncate.

/**
 * Tests a scenario when there are multiple fail overs overs.
 * The test adds a few chunks to the system segments and then fails over.
 * After fail over the zombie instances continue to write junk data to both system segment and journal file.
 * The new instance should read the journal log file and recreate the layout of system segments.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithMultipleFailoversWithTruncate() throws Exception {
    @Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 0;
    val policy = new SegmentRollingPolicy(1024);
    val config = getDefaultConfigBuilder(policy).build();
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    long offsetToTruncateAt = 0;
    ChunkedSegmentStorage oldChunkedSegmentStorage = null;
    SegmentHandle oldHandle = null;
    long oldOffset = 0;
    for (int i = 1; i < 10; i++) {
        // Epoch 2
        epoch++;
        ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
        cleanupHelper.add(metadataStoreAfterCrash);
        ChunkedSegmentStorage segmentStorageInLoop = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
        cleanupHelper.add(segmentStorageInLoop);
        segmentStorageInLoop.initialize(epoch);
        segmentStorageInLoop.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
        segmentStorageInLoop.bootstrap(snapshotInfoStore, null).join();
        deleteGarbage(segmentStorageInLoop);
        checkSystemSegmentsLayout(segmentStorageInLoop);
        val h = segmentStorageInLoop.openWrite(systemSegmentName).join();
        if (null != oldChunkedSegmentStorage) {
            // Add some junk to previous instance after failover
            oldChunkedSegmentStorage.write(oldHandle, oldOffset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
        } else {
            // Only first time.
            for (int j = 1; j < 10; j++) {
                val b0 = "JUNK".getBytes();
                segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b0), b0.length, null).join();
                offset += b0.length;
            }
        }
        offsetToTruncateAt += 4;
        val b1 = "Test".getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
        offset += b1.length;
        val b2 = Integer.toString(i).getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
        offset += b2.length;
        segmentStorageInLoop.truncate(h, offsetToTruncateAt, null).join();
        TestUtils.checkSegmentBounds(metadataStoreAfterCrash, h.getSegmentName(), offsetToTruncateAt, offset);
        val length = Math.toIntExact(offset - offsetToTruncateAt);
        byte[] readBytes = new byte[length];
        val bytesRead = segmentStorageInLoop.read(h, offsetToTruncateAt, readBytes, 0, length, null).get();
        Assert.assertEquals(length, readBytes.length);
        String s = new String(readBytes);
        // Add some garbage
        if (null != oldChunkedSegmentStorage) {
            oldChunkedSegmentStorage.write(oldHandle, oldOffset + 4, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
        }
        // Save these instances so that you can write some junk after bootstrap.
        oldChunkedSegmentStorage = segmentStorageInLoop;
        oldHandle = h;
        oldOffset = offset;
    }
    epoch++;
    @Cleanup ChunkMetadataStore metadataStoreFinal = getMetadataStore();
    ChunkedSegmentStorage segmentStorageFinal = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreFinal, executorService(), config);
    cleanupHelper.add(segmentStorageFinal);
    segmentStorageFinal.initialize(epoch);
    segmentStorageFinal.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorageFinal.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorageFinal);
    checkSystemSegmentsLayout(segmentStorageFinal);
    val info = segmentStorageFinal.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(offset, info.getLength());
    Assert.assertEquals(offsetToTruncateAt, info.getStartOffset());
    byte[] out = new byte[Math.toIntExact(offset - offsetToTruncateAt)];
    val hr = segmentStorageFinal.openRead(systemSegmentName).join();
    segmentStorageFinal.read(hr, offsetToTruncateAt, out, 0, Math.toIntExact(offset - offsetToTruncateAt), null).join();
    val expected = "Test1Test2Test3Test4Test5Test6Test7Test8Test9";
    val actual = new String(out);
    Assert.assertEquals(expected, actual);
}

Example 29 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testIOExceptionDuringWrite.

@Test
public void testIOExceptionDuringWrite() throws Exception {
    String testSegmentName = "test";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageException("test", "Test Exception", new IOException("Test Exception"));
    val clazz = ChunkStorageException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
}

Example 30 with SegmentRollingPolicy

use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.

the class ChunkedSegmentStorage method create.

@Override
public CompletableFuture<SegmentHandle> create(String streamSegmentName, SegmentRollingPolicy rollingPolicy, Duration timeout) {
    checkInitialized();
    return executeSerialized(() -> {
        val traceId = LoggerHelpers.traceEnter(log, "create", streamSegmentName, rollingPolicy);
        val timer = new Timer();
        log.debug("{} create - started segment={}, rollingPolicy={}.", logPrefix, streamSegmentName, rollingPolicy);
        return tryWith(metadataStore.beginTransaction(false, streamSegmentName), txn -> {
            // Retrieve metadata and make sure it does not exist.
            return txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
                val oldSegmentMetadata = (SegmentMetadata) storageMetadata;
                if (null != oldSegmentMetadata) {
                    throw new CompletionException(new StreamSegmentExistsException(streamSegmentName));
                }
                // Create a new record.
                val newSegmentMetadata = SegmentMetadata.builder().name(streamSegmentName).maxRollinglength(rollingPolicy.getMaxLength() == 0 ? SegmentRollingPolicy.NO_ROLLING.getMaxLength() : rollingPolicy.getMaxLength()).ownerEpoch(this.epoch).build();
                newSegmentMetadata.setActive(true);
                txn.create(newSegmentMetadata);
                // commit.
                return txn.commit().thenApplyAsync(v -> {
                    val retValue = SegmentStorageHandle.writeHandle(streamSegmentName);
                    Duration elapsed = timer.getElapsed();
                    SLTS_CREATE_LATENCY.reportSuccessEvent(elapsed);
                    SLTS_CREATE_COUNT.inc();
                    log.debug("{} create - finished segment={}, rollingPolicy={}, latency={}.", logPrefix, streamSegmentName, rollingPolicy, elapsed.toMillis());
                    LoggerHelpers.traceLeave(log, "create", traceId, retValue);
                    return retValue;
                }, executor);
            }, executor);
        }, executor).handleAsync((v, e) -> {
            if (null != e) {
                log.debug("{} create - exception segment={}, rollingPolicy={}, latency={}.", logPrefix, streamSegmentName, rollingPolicy, timer.getElapsedMillis(), e);
                handleException(streamSegmentName, e);
            }
            return v;
        }, executor);
    }, streamSegmentName);
}