Examples with OneInputStreamOperatorTestHarness org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness used on opensource projects

Example 1 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class AsyncWaitOperatorTest method testAsyncTimeout.

@Test
public void testAsyncTimeout() throws Exception {
    final long timeout = 10L;
    final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new LazyAsyncFunction(), timeout, 2, AsyncDataStream.OutputMode.ORDERED);
    final Environment mockEnvironment = mock(Environment.class);
    final Configuration taskConfiguration = new Configuration();
    final ExecutionConfig executionConfig = new ExecutionConfig();
    final TaskMetricGroup metricGroup = new UnregisteredTaskMetricsGroup();
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final TaskInfo taskInfo = new TaskInfo("foobarTask", 1, 0, 1, 1);
    when(mockEnvironment.getTaskConfiguration()).thenReturn(taskConfiguration);
    when(mockEnvironment.getExecutionConfig()).thenReturn(executionConfig);
    when(mockEnvironment.getMetricGroup()).thenReturn(metricGroup);
    when(mockEnvironment.getTaskManagerInfo()).thenReturn(taskManagerRuntimeInfo);
    when(mockEnvironment.getTaskInfo()).thenReturn(taskInfo);
    when(mockEnvironment.getUserClassLoader()).thenReturn(AsyncWaitOperatorTest.class.getClassLoader());
    final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);
    final long initialTime = 0L;
    final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.setProcessingTime(initialTime);
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.processElement(new StreamRecord<>(1, initialTime));
        testHarness.setProcessingTime(initialTime + 5L);
        testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
    }
    // trigger the timeout of the first stream record
    testHarness.setProcessingTime(initialTime + timeout + 1L);
    // allow the second async stream record to be processed
    LazyAsyncFunction.countDown();
    // wait until all async collectors in the buffer have been emitted out.
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.close();
    }
    expectedOutput.add(new StreamRecord<>(2, initialTime + 5L));
    TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
    ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(mockEnvironment).failExternally(argumentCaptor.capture());
    Throwable failureCause = argumentCaptor.getValue();
    Assert.assertNotNull(failureCause.getCause());
    Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
    Assert.assertNotNull(failureCause.getCause().getCause());
    Assert.assertTrue(failureCause.getCause().getCause() instanceof TimeoutException);
}

Example 2 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class WrappingFunctionSnapshotRestoreTest method testSnapshotAndRestoreWrappedListCheckpointed.

@Test
public void testSnapshotAndRestoreWrappedListCheckpointed() throws Exception {
    StreamMap<Integer, Integer> operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(new InnerTestFunList())));
    OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(5, 12L));
    // snapshot and restore from scratch
    OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
    testHarness.close();
    InnerTestFunList innerTestFun = new InnerTestFunList();
    operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun)));
    testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    Assert.assertTrue(innerTestFun.wasRestored);
    testHarness.close();
}

Example 3 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class WrappingFunctionSnapshotRestoreTest method testSnapshotAndRestoreWrappedCheckpointedFunction.

@Test
public void testSnapshotAndRestoreWrappedCheckpointedFunction() throws Exception {
    StreamMap<Integer, Integer> operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(new InnerTestFun())));
    OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(5, 12L));
    // snapshot and restore from scratch
    OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
    testHarness.close();
    InnerTestFun innerTestFun = new InnerTestFun();
    operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun)));
    testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    Assert.assertTrue(innerTestFun.wasRestored);
    testHarness.close();
}

Example 4 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class TimestampsAndPunctuatedWatermarksOperatorTest method testTimestampsAndPeriodicWatermarksOperator.

@Test
@SuppressWarnings("unchecked")
public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
    final TimestampsAndPunctuatedWatermarksOperator<Tuple2<Long, Boolean>> operator = new TimestampsAndPunctuatedWatermarksOperator<>(new PunctuatedExtractor());
    OneInputStreamOperatorTestHarness<Tuple2<Long, Boolean>, Tuple2<Long, Boolean>> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(3L, true), 0L));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(5L, false), 0L));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(4L, false), 0L));
    // this watermark should be ignored
    testHarness.processWatermark(new Watermark(10));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(4L, false), 0L));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(4L, true), 0L));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(9L, false), 0L));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(5L, false), 0L));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(7L, true), 0L));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(10L, false), 0L));
    testHarness.processWatermark(new Watermark(Long.MAX_VALUE));
    ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
    assertEquals(3L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(3L, ((Watermark) output.poll()).getTimestamp());
    assertEquals(5L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(4L, ((Watermark) output.poll()).getTimestamp());
    assertEquals(9L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(5L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(7L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(7L, ((Watermark) output.poll()).getTimestamp());
    assertEquals(10L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
    assertEquals(Long.MAX_VALUE, ((Watermark) output.poll()).getTimestamp());
}

Example 5 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class AccumulatingAlignedProcessingTimeWindowOperatorTest method checkpointRestoreWithPendingWindowTumblingWithProcessFunction.

@Test
public void checkpointRestoreWithPendingWindowTumblingWithProcessFunction() {
    try {
        final int windowSize = 200;
        // tumbling window that triggers every 200 milliseconds
        AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityProcessFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
        OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setup();
        testHarness.open();
        testHarness.setProcessingTime(0);
        // inject some elements
        final int numElementsFirst = 700;
        final int numElements = 1000;
        for (int i = 0; i < numElementsFirst; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        // draw a snapshot and dispose the window
        int beforeSnapShot = testHarness.getOutput().size();
        StreamStateHandle state = testHarness.snapshotLegacy(1L, System.currentTimeMillis());
        List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput());
        int afterSnapShot = testHarness.getOutput().size();
        assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
        assertTrue(afterSnapShot <= numElementsFirst);
        // inject some random elements, which should not show up in the state
        for (int i = 0; i < 300; i++) {
            testHarness.processElement(new StreamRecord<>(i + numElementsFirst));
        }
        testHarness.close();
        op.dispose();
        // re-create the operator and restore the state
        op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityProcessFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
        testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setup();
        testHarness.restore(state);
        testHarness.open();
        // inject some more elements
        for (int i = numElementsFirst; i < numElements; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        testHarness.setProcessingTime(400);
        // get and verify the result
        List<Integer> finalResult = new ArrayList<>();
        finalResult.addAll(resultAtSnapshot);
        List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput());
        finalResult.addAll(finalPartialResult);
        assertEquals(numElements, finalResult.size());
        Collections.sort(finalResult);
        for (int i = 0; i < numElements; i++) {
            assertEquals(i, finalResult.get(i).intValue());
        }
        testHarness.close();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}