Examples with InMemoryTaskQueueManager io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager used on opensource projects

Example 36 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SimpleStorageTests method forkStorage.

/**
 * Creates a new instance of Storage with forked metadata state.
 *
 * Any changes made inside forked instance are not visible in other instances. Useful for testing zombie scenarios.
 *
 * @param storage Storage to fork.
 * @return New forked storage.
 * @throws Exception Exceptions in case of any errors.
 */
protected Storage forkStorage(ChunkedSegmentStorage storage) throws Exception {
    ScheduledExecutorService executor = executorService();
    ChunkedSegmentStorage forkedChunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, storage.getChunkStorage(), getCloneMetadataStore(storage.getMetadataStore()), executor, storage.getConfig());
    forkedChunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    return forkedChunkedSegmentStorage;
}

Example 37 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testSystemSegmentNoConcatAllowed.

@Test
public void testSystemSegmentNoConcatAllowed() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    int maxLength = 8;
    long epoch = 1;
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    val journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
    val systemSegmentName = NameUtils.getAttributeSegmentName(NameUtils.getMetadataSegmentName(containerId));
    Assert.assertTrue(journal.isStorageSystemSegment(systemSegmentName));
    // Init
    long offset = 0;
    // Start container with epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStore, executorService(), config);
    segmentStorage.initialize(epoch);
    segmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorage.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage);
    segmentStorage.create("test", null).get();
    AssertExtensions.assertFutureThrows("concat() should throw", segmentStorage.concat(SegmentStorageHandle.writeHandle(systemSegmentName), 0, "test", null), ex -> ex instanceof IllegalStateException);
    AssertExtensions.assertFutureThrows("concat() should throw", segmentStorage.concat(SegmentStorageHandle.writeHandle("test"), 0, systemSegmentName, null), ex -> ex instanceof IllegalStateException);
}

Example 38 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testInitialization.

@Test
public void testInitialization() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = 42;
    @Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
    garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
    int maxLength = 8;
    long epoch = 1;
    val policy = new SegmentRollingPolicy(maxLength);
    val config = getDefaultConfigBuilder(policy).build();
    // Init
    SystemJournal journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
    // Assert.assertEquals(epoch, journal.getEpoch());
    Assert.assertEquals(containerId, journal.getContainerId());
    Assert.assertEquals(policy.getMaxLength(), journal.getConfig().getStorageMetadataRollingPolicy().getMaxLength());
    // Assert.assertEquals(epoch, journal.getEpoch());
    Assert.assertEquals(0, journal.getCurrentFileIndex().get());
    Assert.assertEquals(NameUtils.INTERNAL_CONTAINER_PREFIX, journal.getSystemSegmentsPrefix());
    Assert.assertArrayEquals(SystemJournal.getChunkStorageSystemSegments(containerId), journal.getSystemSegments());
    journal.initialize();
}

Example 39 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithMultipleFailovers.

private void testSimpleBootstrapWithMultipleFailovers(int containerId, ChunkStorage chunkStorage, ChunkedSegmentStorageConfig config, Consumer<Long> faultInjection) throws Exception {
    @Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 0;
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    ChunkedSegmentStorage oldChunkedSegmentStorage = null;
    SegmentHandle oldHandle = null;
    for (int i = 1; i < 10; i++) {
        // Epoch 2
        epoch++;
        ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
        cleanupHelper.add(metadataStoreAfterCrash);
        ChunkedSegmentStorage segmentStorageInLoop = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
        cleanupHelper.add(segmentStorageInLoop);
        segmentStorageInLoop.initialize(epoch);
        segmentStorageInLoop.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
        segmentStorageInLoop.bootstrap(snapshotInfoStore, null).join();
        deleteGarbage(segmentStorageInLoop);
        checkSystemSegmentsLayout(segmentStorageInLoop);
        val h = segmentStorageInLoop.openWrite(systemSegmentName).join();
        if (null != oldChunkedSegmentStorage) {
            oldChunkedSegmentStorage.write(oldHandle, offset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
        }
        val b1 = "Test".getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
        offset += b1.length;
        val b2 = Integer.toString(i).getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
        offset += b2.length;
        oldChunkedSegmentStorage = segmentStorageInLoop;
        oldHandle = h;
    }
    if (null != faultInjection) {
        faultInjection.accept(epoch);
    }
    epoch++;
    @Cleanup ChunkMetadataStore metadataStoreFinal = getMetadataStore();
    @Cleanup ChunkedSegmentStorage segmentStorageFinal = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreFinal, executorService(), config);
    segmentStorageFinal.initialize(epoch);
    segmentStorageFinal.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorageFinal.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorageFinal);
    checkSystemSegmentsLayout(segmentStorageFinal);
    val info = segmentStorageFinal.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(offset, info.getLength());
    byte[] out = new byte[Math.toIntExact(offset)];
    val hr = segmentStorageFinal.openRead(systemSegmentName).join();
    segmentStorageFinal.read(hr, 0, out, 0, Math.toIntExact(offset), null).join();
    val expected = "Test1Test2Test3Test4Test5Test6Test7Test8Test9";
    val actual = new String(out);
    Assert.assertEquals(expected, actual);
}

Example 40 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithPartialDataWrite.

/**
 * Tests a scenario when journal chunks may be partially written during failure.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithPartialDataWrite() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    if (!chunkStorage.supportsAppend()) {
        return;
    }
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 1;
    val policy = new SegmentRollingPolicy(4);
    val config = getDefaultConfigBuilder(policy).build();
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    // Epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStore, executorService(), config);
    segmentStorage1.initialize(epoch);
    segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage1.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage1);
    checkSystemSegmentsLayout(segmentStorage1);
    // Simulate some writes to system segment, this should cause some new chunks being added.
    val h = segmentStorage1.openWrite(systemSegmentName).join();
    val b1 = "Hello".getBytes();
    segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
    offset += b1.length;
    // Inject a fault by adding some garbage at the end
    checkSystemSegmentsLayout(segmentStorage1);
    val chunkFileName = NameUtils.getSystemJournalFileName(segmentStorage1.getSystemJournal().getContainerId(), segmentStorage1.getSystemJournal().getEpoch(), segmentStorage1.getSystemJournal().getCurrentFileIndex().get());
    val chunkInfo = chunkStorage.getInfo(chunkFileName);
    chunkStorage.write(ChunkHandle.writeHandle(chunkFileName), chunkInfo.get().getLength(), 1, new ByteArrayInputStream(new byte[1])).get();
    // This next write will encounter partially written chunk and is expected to start a new chunk.
    val b2 = " World".getBytes();
    segmentStorage1.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
    offset += b2.length;
    checkSystemSegmentsLayout(segmentStorage1);
    // Step 2
    // Start container with epoch 2
    epoch++;
    @Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
    segmentStorage2.initialize(epoch);
    segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage2.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage2);
    checkSystemSegmentsLayout(segmentStorage2);
    // Validate
    val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(b1.length + b2.length, info.getLength());
    byte[] out = new byte[b1.length + b2.length];
    val hr = segmentStorage2.openRead(systemSegmentName).join();
    segmentStorage2.read(hr, 0, out, 0, b1.length + b2.length, null).join();
    Assert.assertEquals("Hello World", new String(out));
}