Examples with CoStreamMap org.apache.flink.streaming.api.operators.co.CoStreamMap used on opensource projects

Example 1 with CoStreamMap

use of org.apache.flink.streaming.api.operators.co.CoStreamMap in project flink by apache.

the class TwoInputStreamTaskTest method testOvertakingCheckpointBarriers.

/**
	 * This test verifies that checkpoint barriers and barrier buffers work correctly with
	 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
	 * some inputs receive barriers from an earlier checkpoint, thereby blocking,
	 * then all inputs receive barriers from a later checkpoint.
	 */
@Test
@SuppressWarnings("unchecked")
public void testOvertakingCheckpointBarriers() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask = new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<String, Integer, String>(coMapTask, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 0);
    // These elements should be buffered until we receive barriers from
    // all inputs
    testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<Integer>(42, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<Integer>(1337, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("42", initialTime));
    expectedOutput.add(new StreamRecord<String>("1337", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Now give a later barrier to all inputs, this should unblock the first channel,
    // thereby allowing the two blocked elements through
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 0, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 1, 1);
    expectedOutput.add(new CancelCheckpointMarker(0));
    expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
    expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
    expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Then give the earlier barrier, these should be ignored
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 2 with CoStreamMap

use of org.apache.flink.streaming.api.operators.co.CoStreamMap in project flink by apache.

the class TwoInputStreamTaskTest method testOpenCloseAndTimestamps.

/**
	 * This test verifies that open() and close() are correctly called. This test also verifies
	 * that timestamps of emitted elements are correct. {@link CoStreamMap} assigns the input
	 * timestamp to emitted elements.
	 */
@Test
@SuppressWarnings("unchecked")
public void testOpenCloseAndTimestamps() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask = new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<String, Integer, String>(coMapTask, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new TestOpenCloseMapFunction());
    streamConfig.setStreamOperator(coMapOperator);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new StreamRecord<String>("Hello", initialTime + 1), 0, 0);
    expectedOutput.add(new StreamRecord<String>("Hello", initialTime + 1));
    // wait until the input is processed to ensure ordering of the output
    testHarness.waitForInputProcessing();
    testHarness.processElement(new StreamRecord<Integer>(1337, initialTime + 2), 1, 0);
    expectedOutput.add(new StreamRecord<String>("1337", initialTime + 2));
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    Assert.assertTrue("RichFunction methods where not called.", TestOpenCloseMapFunction.closeCalled);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 3 with CoStreamMap

use of org.apache.flink.streaming.api.operators.co.CoStreamMap in project flink by apache.

the class StreamTaskCancellationBarrierTest method testDeclineCallOnCancelBarrierTwoInputs.

/**
	 * This test verifies (for onw input tasks) that the Stream tasks react the following way to
	 * receiving a checkpoint cancellation barrier:
	 *
	 *   - send a "decline checkpoint" notification out (to the JobManager)
	 *   - emit a cancellation barrier downstream
	 */
@Test
public void testDeclineCallOnCancelBarrierTwoInputs() throws Exception {
    TwoInputStreamTask<String, String, String> task = new TwoInputStreamTask<String, String, String>();
    TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, String, String> op = new CoStreamMap<>(new UnionCoMap());
    streamConfig.setStreamOperator(op);
    StreamMockEnvironment environment = spy(testHarness.createEnvironment());
    // start the task
    testHarness.invoke(environment);
    testHarness.waitForTaskRunning();
    // emit cancellation barriers
    testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
    testHarness.processEvent(new CancelCheckpointMarker(2L), 1, 0);
    testHarness.waitForInputProcessing();
    // the decline call should go to the coordinator
    verify(environment, times(1)).declineCheckpoint(eq(2L), any(CheckpointDeclineOnCancellationBarrierException.class));
    // a cancellation barrier should be downstream
    Object result = testHarness.getOutput().poll();
    assertNotNull("nothing emitted", result);
    assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
    assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());
    // cancel and shutdown
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 4 with CoStreamMap

use of org.apache.flink.streaming.api.operators.co.CoStreamMap in project flink by apache.

the class TwoInputStreamTaskTest method testWatermarkAndStreamStatusForwarding.

/**
	 * This test verifies that watermarks and stream statuses are correctly forwarded. This also checks whether
	 * watermarks are forwarded only when we have received watermarks from all inputs. The
	 * forwarded watermark must be the minimum of the watermarks of all active inputs.
	 */
@Test
@SuppressWarnings("unchecked")
public void testWatermarkAndStreamStatusForwarding() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask = new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<String, Integer, String>(coMapTask, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new Watermark(initialTime), 0, 0);
    testHarness.processElement(new Watermark(initialTime), 0, 1);
    testHarness.processElement(new Watermark(initialTime), 1, 0);
    // now the output should still be empty
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(new Watermark(initialTime), 1, 1);
    // now the watermark should have propagated, Map simply forward Watermarks
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // contrary to checkpoint barriers these elements are not blocked by watermarks
    testHarness.processElement(new StreamRecord<String>("Hello", initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<Integer>(42, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("Hello", initialTime));
    expectedOutput.add(new StreamRecord<String>("42", initialTime));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(new Watermark(initialTime + 4), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 3), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 3), 1, 0);
    testHarness.processElement(new Watermark(initialTime + 2), 1, 1);
    // check whether we get the minimum of all the watermarks, this must also only occur in
    // the output after the two StreamRecords
    expectedOutput.add(new Watermark(initialTime + 2));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance watermark from one of the inputs, now we should get a new one since the
    // minimum increases
    testHarness.processElement(new Watermark(initialTime + 4), 1, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 3));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance the other two inputs, now we should get a new one since the
    // minimum increases again
    testHarness.processElement(new Watermark(initialTime + 4), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 4), 1, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 4));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // test whether idle input channels are acknowledged correctly when forwarding watermarks
    testHarness.processElement(StreamStatus.IDLE, 0, 1);
    testHarness.processElement(StreamStatus.IDLE, 1, 0);
    testHarness.processElement(new Watermark(initialTime + 6), 0, 0);
    // this watermark should be advanced first
    testHarness.processElement(new Watermark(initialTime + 5), 1, 1);
    // once this is acknowledged,
    testHarness.processElement(StreamStatus.IDLE, 1, 1);
    // watermark (initial + 6) should be forwarded
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 5));
    // We don't expect to see Watermark(6) here because the idle status of one
    // input doesn't propagate to the other input. That is, if input 1 is at WM 6 and input
    // two was at WM 5 before going to IDLE then the output watermark will not jump to WM 6.
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make all input channels idle and check that the operator's idle status is forwarded
    testHarness.processElement(StreamStatus.IDLE, 0, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(StreamStatus.IDLE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make some input channels active again and check that the operator's active status is forwarded only once
    testHarness.processElement(StreamStatus.ACTIVE, 1, 0);
    testHarness.processElement(StreamStatus.ACTIVE, 0, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(StreamStatus.ACTIVE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(2, resultElements.size());
}

Example 5 with CoStreamMap

use of org.apache.flink.streaming.api.operators.co.CoStreamMap in project flink by apache.

the class TwoInputStreamTaskTest method testCheckpointBarriers.

/**
	 * This test verifies that checkpoint barriers are correctly forwarded.
	 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointBarriers() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask = new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<String, Integer, String>(coMapTask, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 0);
    // This element should be buffered since we received a checkpoint barrier on
    // this input
    testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
    // This one should go through
    testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 1);
    expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
    testHarness.waitForInputProcessing();
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<Integer>(11, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<Integer>(111, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("11", initialTime));
    expectedOutput.add(new StreamRecord<String>("111", initialTime));
    testHarness.waitForInputProcessing();
    // TODO Use count down latches instead as a cleaner solution
    for (int i = 0; i < 20; ++i) {
        if (testHarness.getOutput().size() >= expectedOutput.size()) {
            break;
        } else {
            Thread.sleep(100);
        }
    }
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    // now we should see the barrier and after that the buffered elements
    expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()));
    expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(4, resultElements.size());
}