Examples with InMemoryTaskQueueManager io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager used on opensource projects

Example 16 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithMultipleFailoversWithTruncate.

/**
 * Tests a scenario when there are multiple fail overs overs.
 * The test adds a few chunks to the system segments and then fails over.
 * After fail over the zombie instances continue to write junk data to both system segment and journal file.
 * The new instance should read the journal log file and recreate the layout of system segments.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithMultipleFailoversWithTruncate() throws Exception {
    @Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 0;
    val policy = new SegmentRollingPolicy(1024);
    val config = getDefaultConfigBuilder(policy).build();
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    long offsetToTruncateAt = 0;
    ChunkedSegmentStorage oldChunkedSegmentStorage = null;
    SegmentHandle oldHandle = null;
    long oldOffset = 0;
    for (int i = 1; i < 10; i++) {
        // Epoch 2
        epoch++;
        ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
        cleanupHelper.add(metadataStoreAfterCrash);
        ChunkedSegmentStorage segmentStorageInLoop = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
        cleanupHelper.add(segmentStorageInLoop);
        segmentStorageInLoop.initialize(epoch);
        segmentStorageInLoop.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
        segmentStorageInLoop.bootstrap(snapshotInfoStore, null).join();
        deleteGarbage(segmentStorageInLoop);
        checkSystemSegmentsLayout(segmentStorageInLoop);
        val h = segmentStorageInLoop.openWrite(systemSegmentName).join();
        if (null != oldChunkedSegmentStorage) {
            // Add some junk to previous instance after failover
            oldChunkedSegmentStorage.write(oldHandle, oldOffset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
        } else {
            // Only first time.
            for (int j = 1; j < 10; j++) {
                val b0 = "JUNK".getBytes();
                segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b0), b0.length, null).join();
                offset += b0.length;
            }
        }
        offsetToTruncateAt += 4;
        val b1 = "Test".getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
        offset += b1.length;
        val b2 = Integer.toString(i).getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
        offset += b2.length;
        segmentStorageInLoop.truncate(h, offsetToTruncateAt, null).join();
        TestUtils.checkSegmentBounds(metadataStoreAfterCrash, h.getSegmentName(), offsetToTruncateAt, offset);
        val length = Math.toIntExact(offset - offsetToTruncateAt);
        byte[] readBytes = new byte[length];
        val bytesRead = segmentStorageInLoop.read(h, offsetToTruncateAt, readBytes, 0, length, null).get();
        Assert.assertEquals(length, readBytes.length);
        String s = new String(readBytes);
        // Add some garbage
        if (null != oldChunkedSegmentStorage) {
            oldChunkedSegmentStorage.write(oldHandle, oldOffset + 4, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
        }
        // Save these instances so that you can write some junk after bootstrap.
        oldChunkedSegmentStorage = segmentStorageInLoop;
        oldHandle = h;
        oldOffset = offset;
    }
    epoch++;
    @Cleanup ChunkMetadataStore metadataStoreFinal = getMetadataStore();
    ChunkedSegmentStorage segmentStorageFinal = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreFinal, executorService(), config);
    cleanupHelper.add(segmentStorageFinal);
    segmentStorageFinal.initialize(epoch);
    segmentStorageFinal.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorageFinal.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorageFinal);
    checkSystemSegmentsLayout(segmentStorageFinal);
    val info = segmentStorageFinal.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(offset, info.getLength());
    Assert.assertEquals(offsetToTruncateAt, info.getStartOffset());
    byte[] out = new byte[Math.toIntExact(offset - offsetToTruncateAt)];
    val hr = segmentStorageFinal.openRead(systemSegmentName).join();
    segmentStorageFinal.read(hr, offsetToTruncateAt, out, 0, Math.toIntExact(offset - offsetToTruncateAt), null).join();
    val expected = "Test1Test2Test3Test4Test5Test6Test7Test8Test9";
    val actual = new String(out);
    Assert.assertEquals(expected, actual);
}

Example 17 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testMixedChunk.

/**
 * Test for a mix bag of chunks.
 */
@Test
public void testMixedChunk() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    insertChunk(chunkStorage, "deletedChunk", dataSize);
    insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
    insertChunk(chunkStorage, "activeChunk", dataSize);
    insertChunkMetadata(metadataStore, "activeChunk", dataSize, 1);
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("activeChunk", "nonExistingChunk", "deletedChunk")).join();
    // Validate state before
    assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "activeChunk", "nonExistingChunk", "deletedChunk" });
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 3);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(3, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
    Assert.assertTrue(chunkStorage.exists("activeChunk").get());
}

Example 18 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testDeletedChunk.

/**
 * Test for chunk that is marked inactive and added as garbage.
 */
@Test
public void testDeletedChunk() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    insertChunk(chunkStorage, "deletedChunk", dataSize);
    insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Collections.singleton("deletedChunk")).join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("deletedChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
    Assert.assertNull(getChunkMetadata(metadataStore, "deletedChunk"));
}

Example 19 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testDeletedChunkMissingFromStorage.

/**
 * Test for chunk that is marked inactive, added as garbage but missing from storage.
 */
@Test
public void testDeletedChunkMissingFromStorage() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertFalse(chunkStorage.exists("deletedChunk").join());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Collections.singleton("deletedChunk")).join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("deletedChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
    Assert.assertNull(getChunkMetadata(metadataStore, "deletedChunk"));
}

Example 20 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testActiveChunk.

/**
 * Test for chunk that is marked active but added as garbage.
 */
@Test
public void testActiveChunk() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    insertChunk(chunkStorage, "activeChunk", dataSize);
    insertChunkMetadata(metadataStore, "activeChunk", dataSize, 1);
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Collections.singleton("activeChunk")).join();
    // Validate state before
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("activeChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertTrue(chunkStorage.exists("activeChunk").get());
    Assert.assertNotNull(getChunkMetadata(metadataStore, "activeChunk"));
    Assert.assertTrue(chunkStorage.exists("activeChunk").join());
}