use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.
the class AsyncWaitOperatorTest method testAsyncTimeout.
@Test
public void testAsyncTimeout() throws Exception {
final long timeout = 10L;
final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new LazyAsyncFunction(), timeout, 2, AsyncDataStream.OutputMode.ORDERED);
final Environment mockEnvironment = mock(Environment.class);
final Configuration taskConfiguration = new Configuration();
final ExecutionConfig executionConfig = new ExecutionConfig();
final TaskMetricGroup metricGroup = new UnregisteredTaskMetricsGroup();
final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
final TaskInfo taskInfo = new TaskInfo("foobarTask", 1, 0, 1, 1);
when(mockEnvironment.getTaskConfiguration()).thenReturn(taskConfiguration);
when(mockEnvironment.getExecutionConfig()).thenReturn(executionConfig);
when(mockEnvironment.getMetricGroup()).thenReturn(metricGroup);
when(mockEnvironment.getTaskManagerInfo()).thenReturn(taskManagerRuntimeInfo);
when(mockEnvironment.getTaskInfo()).thenReturn(taskInfo);
when(mockEnvironment.getUserClassLoader()).thenReturn(AsyncWaitOperatorTest.class.getClassLoader());
final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);
final long initialTime = 0L;
final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
testHarness.setProcessingTime(initialTime);
synchronized (testHarness.getCheckpointLock()) {
testHarness.processElement(new StreamRecord<>(1, initialTime));
testHarness.setProcessingTime(initialTime + 5L);
testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
}
// trigger the timeout of the first stream record
testHarness.setProcessingTime(initialTime + timeout + 1L);
// allow the second async stream record to be processed
LazyAsyncFunction.countDown();
// wait until all async collectors in the buffer have been emitted out.
synchronized (testHarness.getCheckpointLock()) {
testHarness.close();
}
expectedOutput.add(new StreamRecord<>(2, initialTime + 5L));
TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
verify(mockEnvironment).failExternally(argumentCaptor.capture());
Throwable failureCause = argumentCaptor.getValue();
Assert.assertNotNull(failureCause.getCause());
Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
Assert.assertNotNull(failureCause.getCause().getCause());
Assert.assertTrue(failureCause.getCause().getCause() instanceof TimeoutException);
}
use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.
the class WrappingFunctionSnapshotRestoreTest method testSnapshotAndRestoreWrappedListCheckpointed.
@Test
public void testSnapshotAndRestoreWrappedListCheckpointed() throws Exception {
StreamMap<Integer, Integer> operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(new InnerTestFunList())));
OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.setup();
testHarness.open();
testHarness.processElement(new StreamRecord<>(5, 12L));
// snapshot and restore from scratch
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
testHarness.close();
InnerTestFunList innerTestFun = new InnerTestFunList();
operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun)));
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.setup();
testHarness.initializeState(snapshot);
testHarness.open();
Assert.assertTrue(innerTestFun.wasRestored);
testHarness.close();
}
use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.
the class WrappingFunctionSnapshotRestoreTest method testSnapshotAndRestoreWrappedCheckpointedFunction.
@Test
public void testSnapshotAndRestoreWrappedCheckpointedFunction() throws Exception {
StreamMap<Integer, Integer> operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(new InnerTestFun())));
OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.setup();
testHarness.open();
testHarness.processElement(new StreamRecord<>(5, 12L));
// snapshot and restore from scratch
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
testHarness.close();
InnerTestFun innerTestFun = new InnerTestFun();
operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun)));
testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.setup();
testHarness.initializeState(snapshot);
testHarness.open();
Assert.assertTrue(innerTestFun.wasRestored);
testHarness.close();
}
use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.
the class TimestampsAndPunctuatedWatermarksOperatorTest method testTimestampsAndPeriodicWatermarksOperator.
@Test
@SuppressWarnings("unchecked")
public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
final TimestampsAndPunctuatedWatermarksOperator<Tuple2<Long, Boolean>> operator = new TimestampsAndPunctuatedWatermarksOperator<>(new PunctuatedExtractor());
OneInputStreamOperatorTestHarness<Tuple2<Long, Boolean>, Tuple2<Long, Boolean>> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
testHarness.open();
testHarness.processElement(new StreamRecord<>(new Tuple2<>(3L, true), 0L));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(5L, false), 0L));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(4L, false), 0L));
// this watermark should be ignored
testHarness.processWatermark(new Watermark(10));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(4L, false), 0L));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(4L, true), 0L));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(9L, false), 0L));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(5L, false), 0L));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(7L, true), 0L));
testHarness.processElement(new StreamRecord<>(new Tuple2<>(10L, false), 0L));
testHarness.processWatermark(new Watermark(Long.MAX_VALUE));
ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
assertEquals(3L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(3L, ((Watermark) output.poll()).getTimestamp());
assertEquals(5L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(4L, ((Watermark) output.poll()).getTimestamp());
assertEquals(9L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(5L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(7L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(7L, ((Watermark) output.poll()).getTimestamp());
assertEquals(10L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
assertEquals(Long.MAX_VALUE, ((Watermark) output.poll()).getTimestamp());
}
use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.
the class AccumulatingAlignedProcessingTimeWindowOperatorTest method checkpointRestoreWithPendingWindowTumblingWithProcessFunction.
@Test
public void checkpointRestoreWithPendingWindowTumblingWithProcessFunction() {
try {
final int windowSize = 200;
// tumbling window that triggers every 200 milliseconds
AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityProcessFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(op);
testHarness.setup();
testHarness.open();
testHarness.setProcessingTime(0);
// inject some elements
final int numElementsFirst = 700;
final int numElements = 1000;
for (int i = 0; i < numElementsFirst; i++) {
testHarness.processElement(new StreamRecord<>(i));
}
// draw a snapshot and dispose the window
int beforeSnapShot = testHarness.getOutput().size();
StreamStateHandle state = testHarness.snapshotLegacy(1L, System.currentTimeMillis());
List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput());
int afterSnapShot = testHarness.getOutput().size();
assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
assertTrue(afterSnapShot <= numElementsFirst);
// inject some random elements, which should not show up in the state
for (int i = 0; i < 300; i++) {
testHarness.processElement(new StreamRecord<>(i + numElementsFirst));
}
testHarness.close();
op.dispose();
// re-create the operator and restore the state
op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityProcessFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
testHarness = new OneInputStreamOperatorTestHarness<>(op);
testHarness.setup();
testHarness.restore(state);
testHarness.open();
// inject some more elements
for (int i = numElementsFirst; i < numElements; i++) {
testHarness.processElement(new StreamRecord<>(i));
}
testHarness.setProcessingTime(400);
// get and verify the result
List<Integer> finalResult = new ArrayList<>();
finalResult.addAll(resultAtSnapshot);
List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput());
finalResult.addAll(finalPartialResult);
assertEquals(numElements, finalResult.size());
Collections.sort(finalResult);
for (int i = 0; i < numElements; i++) {
assertEquals(i, finalResult.get(i).intValue());
}
testHarness.close();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Aggregations