Examples with InMemoryTaskQueueManager io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager used on opensource projects

Example 26 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithTwoFailovers.

/**
 * Tests a scenario when there are two fail overs.
 * The test adds a few chunks to the system segments and then fails over.
 * After fail over the zombie instance continues to write junk data to both system segment and journal file.
 * The new instance should read the journal log file and recreate the layout of system segments.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithTwoFailovers() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
    @Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 1;
    val policy = new SegmentRollingPolicy(8);
    val config = getDefaultConfigBuilder(policy).build();
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    // Epoch 1
    @Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
    segmentStorage1.initialize(epoch);
    segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage1.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage1);
    checkSystemSegmentsLayout(segmentStorage1);
    // Simulate some writes to system segment, this should cause some new chunks being added.
    val h = segmentStorage1.openWrite(systemSegmentName).join();
    val b1 = "Hello".getBytes();
    segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
    offset += b1.length;
    checkSystemSegmentsLayout(segmentStorage1);
    // Epoch 2
    epoch++;
    @Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
    segmentStorage2.initialize(epoch);
    segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Bootstrap
    segmentStorage2.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorage2);
    checkSystemSegmentsLayout(segmentStorage2);
    val h2 = segmentStorage2.openWrite(systemSegmentName).join();
    // Write Junk Data to from first instance.
    segmentStorage1.write(h, offset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
    val b2 = " World".getBytes();
    segmentStorage2.write(h2, offset, new ByteArrayInputStream(b2), b2.length, null).join();
    offset += b2.length;
    checkSystemSegmentsLayout(segmentStorage2);
    val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(b1.length + b2.length, info.getLength());
    byte[] out = new byte[b1.length + b2.length];
    val hr = segmentStorage2.openRead(systemSegmentName).join();
    segmentStorage2.read(hr, 0, out, 0, b1.length + b2.length, null).join();
    Assert.assertEquals("Hello World", new String(out));
}

Example 27 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class TestUtils method checkGarbageCollectionQueue.

/**
 * Checks garbage collection queue to ensure new chunks and truncated chunks are added to GC queue.
 *
 * @param chunkedSegmentStorage Instance of {@link ChunkedSegmentStorage}.
 * @param beforeSet set of chunks before.
 * @param afterSet set of chunks after.
 */
public static void checkGarbageCollectionQueue(ChunkedSegmentStorage chunkedSegmentStorage, Set<String> beforeSet, Set<String> afterSet) {
    // Get the enqueued tasks.
    // Need to de-dup
    val tasks = new HashMap<String, GarbageCollector.TaskInfo>();
    val tasksList = ((InMemoryTaskQueueManager) chunkedSegmentStorage.getGarbageCollector().getTaskQueue()).drain(chunkedSegmentStorage.getGarbageCollector().getTaskQueueName(), Integer.MAX_VALUE).stream().collect(Collectors.toList());
    for (val task : tasksList) {
        tasks.put(task.getName(), task);
    }
    // All chunks not in new set must be enqueued for deletion.
    for (val oldChunk : beforeSet) {
        if (!afterSet.contains(oldChunk)) {
            val task = tasks.get(oldChunk);
            Assert.assertNotNull(task);
            Assert.assertEquals(GarbageCollector.TaskInfo.DELETE_CHUNK, task.getTaskType());
        }
    }
    // All chunks not in old set must be enqueued for deletion.
    for (val newChunk : afterSet) {
        if (!beforeSet.contains(newChunk)) {
            val task = tasks.get(newChunk);
            Assert.assertNotNull(task);
            Assert.assertEquals(GarbageCollector.TaskInfo.DELETE_CHUNK, task.getTaskType());
        }
    }
}

Example 28 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testIOException.

/**
 * Test for IO exception.
 */
@Test
public void testIOException() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    insertChunk(chunkStorage, "deletedChunk", dataSize);
    insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    chunkStorage.setReadOnly(chunkStorage.openWrite("deletedChunk").get(), true).join();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("deletedChunk")).join();
    // Validate state before
    assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "deletedChunk" });
    Assert.assertTrue(chunkStorage.exists("deletedChunk").get());
}

Example 29 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testMaxAttemptsWithSegment.

/**
 * Test for Max Attempts.
 */
@Test
public void testMaxAttemptsWithSegment() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = spy(getMetadataStore());
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] {}, false, 0);
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    for (int i = 0; i < 3; i++) {
        val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
        Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
        Assert.assertEquals(1, garbageCollector.getQueueSize().get());
        // Step 2: Inject fault.
        CompletableFuture f = new CompletableFuture();
        f.completeExceptionally(new StorageMetadataException("Test Exception"));
        doReturn(f).when(metadataStore).commit(any());
        garbageCollector.processBatch(list).join();
        // Validate state after
        Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
        Assert.assertEquals(1, garbageCollector.getQueueSize().get());
        Assert.assertNotNull(getSegmentMetadata(metadataStore, "testSegment"));
    }
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).size());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getFailedQueueName()).peek().getName());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertNotNull(getSegmentMetadata(metadataStore, "testSegment"));
}

Example 30 with InMemoryTaskQueueManager

use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.

the class GarbageCollectorTests method testNonExistentChunk.

/**
 * Test for chunk that does not exist in metadata but added as garbage.
 */
@Test
public void testNonExistentChunk() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
    int containerId = CONTAINER_ID;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    Assert.assertFalse(chunkStorage.exists("nonExistingChunk").join());
    // Add some garbage
    garbageCollector.addChunksToGarbage(TXN_ID, Collections.singleton("nonExistingChunk")).join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("nonExistingChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
}