Examples with CheckpointBarrier org.apache.flink.runtime.io.network.api.CheckpointBarrier used on opensource projects

Example 11 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class TwoInputStreamTaskTest method testCheckpointBarriers.

/**
	 * This test verifies that checkpoint barriers are correctly forwarded.
	 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointBarriers() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask = new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<String, Integer, String>(coMapTask, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 0);
    // This element should be buffered since we received a checkpoint barrier on
    // this input
    testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
    // This one should go through
    testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 1);
    expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
    testHarness.waitForInputProcessing();
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<Integer>(11, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<Integer>(111, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("11", initialTime));
    expectedOutput.add(new StreamRecord<String>("111", initialTime));
    testHarness.waitForInputProcessing();
    // TODO Use count down latches instead as a cleaner solution
    for (int i = 0; i < 20; ++i) {
        if (testHarness.getOutput().size() >= expectedOutput.size()) {
            break;
        } else {
            Thread.sleep(100);
        }
    }
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    // now we should see the barrier and after that the buffered elements
    expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()));
    expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(4, resultElements.size());
}