Search in sources :

Example 1 with BackendForTestStream

use of org.apache.flink.runtime.state.testutils.BackendForTestStream in project flink by apache.

the class TaskCheckpointingBehaviourTest method testBlockingNonInterruptibleCheckpoint.

@Test
public void testBlockingNonInterruptibleCheckpoint() throws Exception {
    StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new);
    Task task = createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class));
    // start the task and wait until it is in "restore"
    task.startTaskThread();
    IN_CHECKPOINT_LATCH.await();
    // cancel the task and wait. unless cancellation properly closes
    // the streams, this will never terminate
    task.cancelExecution();
    task.getExecutingThread().join();
    assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    assertNull(task.getFailureCause());
}
Also used : Task(org.apache.flink.runtime.taskmanager.Task) CheckpointResponder(org.apache.flink.runtime.taskmanager.CheckpointResponder) StateBackend(org.apache.flink.runtime.state.StateBackend) OperatorStateBackend(org.apache.flink.runtime.state.OperatorStateBackend) DefaultOperatorStateBackend(org.apache.flink.runtime.state.DefaultOperatorStateBackend) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) BackendForTestStream(org.apache.flink.runtime.state.testutils.BackendForTestStream) Test(org.junit.Test)

Example 2 with BackendForTestStream

use of org.apache.flink.runtime.state.testutils.BackendForTestStream in project flink by apache.

the class RocksDBAsyncSnapshotTest method testCancelFullyAsyncCheckpoints.

/**
 * This tests ensures that canceling of asynchronous snapshots works as expected and does not
 * block.
 */
@Test
public void testCancelFullyAsyncCheckpoints() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.configureForKeyedStream(value -> value, BasicTypeInfo.STRING_TYPE_INFO);
    StreamConfig streamConfig = testHarness.getStreamConfig();
    File dbDir = temporaryFolder.newFolder();
    final EmbeddedRocksDBStateBackend.PriorityQueueStateType timerServicePriorityQueueType = RocksDBOptions.TIMER_SERVICE_FACTORY.defaultValue();
    final int skipStreams;
    if (timerServicePriorityQueueType == EmbeddedRocksDBStateBackend.PriorityQueueStateType.HEAP) {
        // we skip the first created stream, because it is used to checkpoint the timer service,
        // which is
        // currently not asynchronous.
        skipStreams = 1;
    } else if (timerServicePriorityQueueType == EmbeddedRocksDBStateBackend.PriorityQueueStateType.ROCKSDB) {
        skipStreams = 0;
    } else {
        throw new AssertionError(String.format("Unknown timer service priority queue type %s.", timerServicePriorityQueueType));
    }
    // this is the proper instance that we need to call.
    BlockerCheckpointStreamFactory blockerCheckpointStreamFactory = new BlockerCheckpointStreamFactory(4 * 1024 * 1024) {

        int count = skipStreams;

        @Override
        public CheckpointStateOutputStream createCheckpointStateOutputStream(CheckpointedStateScope scope) throws IOException {
            if (count > 0) {
                --count;
                return new BlockingCheckpointOutputStream(new MemCheckpointStreamFactory.MemoryCheckpointOutputStream(maxSize), null, null, Integer.MAX_VALUE);
            } else {
                return super.createCheckpointStateOutputStream(scope);
            }
        }
    };
    // to avoid serialization of the above factory instance, we need to pass it in
    // through a static variable
    StateBackend stateBackend = new BackendForTestStream(new StaticForwardFactory(blockerCheckpointStreamFactory));
    RocksDBStateBackend backend = new RocksDBStateBackend(stateBackend);
    backend.setDbStoragePath(dbDir.getAbsolutePath());
    streamConfig.setStateBackend(backend);
    streamConfig.setStreamOperator(new AsyncCheckpointOperator());
    streamConfig.setOperatorID(new OperatorID());
    TestTaskStateManager taskStateManagerTestMock = new TestTaskStateManager();
    StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, taskStateManagerTestMock);
    blockerCheckpointStreamFactory.setBlockerLatch(new OneShotLatch());
    blockerCheckpointStreamFactory.setWaiterLatch(new OneShotLatch());
    testHarness.invoke(mockEnv);
    testHarness.waitForTaskRunning();
    final OneInputStreamTask<String, String> task = testHarness.getTask();
    task.triggerCheckpointAsync(new CheckpointMetaData(42, 17), CheckpointOptions.forCheckpointWithDefaultLocation()).get();
    testHarness.processElement(new StreamRecord<>("Wohoo", 0));
    blockerCheckpointStreamFactory.getWaiterLatch().await();
    task.cancel();
    blockerCheckpointStreamFactory.getBlockerLatch().trigger();
    testHarness.endInput();
    ExecutorService threadPool = task.getAsyncOperationsThreadPool();
    threadPool.shutdown();
    Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS));
    Set<BlockingCheckpointOutputStream> createdStreams = blockerCheckpointStreamFactory.getAllCreatedStreams();
    for (BlockingCheckpointOutputStream stream : createdStreams) {
        Assert.assertTrue("Not all of the " + createdStreams.size() + " created streams have been closed.", stream.isClosed());
    }
    try {
        testHarness.waitForTaskCompletion();
        fail("Operation completed. Cancel failed.");
    } catch (Exception expected) {
        Throwable cause = expected.getCause();
        if (!(cause instanceof CancelTaskException)) {
            fail("Unexpected exception: " + expected);
        }
    }
}
Also used : OneInputStreamTask(org.apache.flink.streaming.runtime.tasks.OneInputStreamTask) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) StateBackend(org.apache.flink.runtime.state.StateBackend) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) AbstractKeyedStateBackend(org.apache.flink.runtime.state.AbstractKeyedStateBackend) MemCheckpointStreamFactory(org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory) BlockerCheckpointStreamFactory(org.apache.flink.runtime.util.BlockerCheckpointStreamFactory) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) StreamMockEnvironment(org.apache.flink.streaming.runtime.tasks.StreamMockEnvironment) MockInputSplitProvider(org.apache.flink.runtime.operators.testutils.MockInputSplitProvider) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) BlockingCheckpointOutputStream(org.apache.flink.runtime.util.BlockingCheckpointOutputStream) CheckpointMetaData(org.apache.flink.runtime.checkpoint.CheckpointMetaData) CheckpointException(org.apache.flink.runtime.checkpoint.CheckpointException) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) BackendForTestStream(org.apache.flink.runtime.state.testutils.BackendForTestStream) TestTaskStateManager(org.apache.flink.runtime.state.TestTaskStateManager) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) OneInputStreamTaskTestHarness(org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness) ExecutorService(java.util.concurrent.ExecutorService) CheckpointedStateScope(org.apache.flink.runtime.state.CheckpointedStateScope) File(java.io.File) Test(org.junit.Test)

Aggregations

StateBackend (org.apache.flink.runtime.state.StateBackend)2 MemoryStateBackend (org.apache.flink.runtime.state.memory.MemoryStateBackend)2 BackendForTestStream (org.apache.flink.runtime.state.testutils.BackendForTestStream)2 Test (org.junit.Test)2 File (java.io.File)1 IOException (java.io.IOException)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 OneShotLatch (org.apache.flink.core.testutils.OneShotLatch)1 CheckpointException (org.apache.flink.runtime.checkpoint.CheckpointException)1 CheckpointMetaData (org.apache.flink.runtime.checkpoint.CheckpointMetaData)1 CancelTaskException (org.apache.flink.runtime.execution.CancelTaskException)1 OperatorID (org.apache.flink.runtime.jobgraph.OperatorID)1 MockInputSplitProvider (org.apache.flink.runtime.operators.testutils.MockInputSplitProvider)1 AbstractKeyedStateBackend (org.apache.flink.runtime.state.AbstractKeyedStateBackend)1 CheckpointedStateScope (org.apache.flink.runtime.state.CheckpointedStateScope)1 DefaultOperatorStateBackend (org.apache.flink.runtime.state.DefaultOperatorStateBackend)1 OperatorStateBackend (org.apache.flink.runtime.state.OperatorStateBackend)1 TestTaskStateManager (org.apache.flink.runtime.state.TestTaskStateManager)1 MemCheckpointStreamFactory (org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory)1