Examples with KeyedOneInputStreamOperatorTestHarness org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness used on opensource projects

Example 46 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class WindowOperatorTest method testCleanupTimerWithEmptyListStateForTumblingWindows2.

@Test
public void testCleanupTimerWithEmptyListStateForTumblingWindows2() throws Exception {
    final int WINDOW_SIZE = 2;
    final long LATENESS = 100;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc = new ListStateDescriptor<>("window-contents", inputType.createSerializer(new ExecutionConfig()));
    WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, String, TimeWindow> operator = new WindowOperator<>(TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalIterableWindowFunction<>(new PassThroughFunction2()), new EventTimeTriggerAccumGC(LATENESS), LATENESS, null);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.open();
    ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
    // normal element
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processWatermark(new Watermark(1599));
    testHarness.processWatermark(new Watermark(1999));
    testHarness.processWatermark(new Watermark(2100));
    testHarness.processWatermark(new Watermark(5000));
    expected.add(new Watermark(1599));
    expected.add(new StreamRecord<>("GOT: (key2,1)", 1999));
    // here it fires and purges
    expected.add(new Watermark(1999));
    // here is the cleanup timer
    expected.add(new Watermark(2100));
    expected.add(new Watermark(5000));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
    testHarness.close();
}

Example 47 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class WindowOperatorTest method testSideOutputDueToLatenessTumbling.

@Test
public void testSideOutputDueToLatenessTumbling() throws Exception {
    final int WINDOW_SIZE = 2;
    final long LATENESS = 0;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
    WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()), EventTimeTrigger.create(), LATENESS, lateOutputTag);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.open();
    ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
    ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
    // normal element
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processWatermark(new Watermark(1985));
    expected.add(new Watermark(1985));
    // this will not be dropped because window.maxTimestamp() + allowedLateness > currentWatermark
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1980));
    testHarness.processWatermark(new Watermark(1999));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 1999));
    expected.add(new Watermark(1999));
    // sideoutput as late, will reuse previous timestamp since only input tuple is sideoutputed
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
    sideExpected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2001));
    testHarness.processWatermark(new Watermark(2999));
    expected.add(new Watermark(2999));
    testHarness.processWatermark(new Watermark(3999));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
    expected.add(new Watermark(3999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
    TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
    testHarness.close();
}

Example 48 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class WindowOperatorTest method testSideOutputDueToLatenessSliding.

@Test
public void testSideOutputDueToLatenessSliding() throws Exception {
    final int WINDOW_SIZE = 3;
    final int WINDOW_SLIDE = 1;
    final long LATENESS = 0;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
    WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(SlidingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS), Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()), EventTimeTrigger.create(), LATENESS, lateOutputTag);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.open();
    ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
    ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processWatermark(new Watermark(1999));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
    expected.add(new Watermark(1999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
    testHarness.processWatermark(new Watermark(3000));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
    expected.add(new Watermark(3000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
    // lateness is set to 0 and window_size = 3 sec and slide 1, the following 2 elements (2400)
    // are assigned to windows ending at 2999, 3999, 4999.
    // The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
    testHarness.processWatermark(new Watermark(6000));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
    expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
    expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
    expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));
    expected.add(new Watermark(6000));
    // sideoutput element due to lateness
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
    sideExpected.add(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
    testHarness.processWatermark(new Watermark(25000));
    expected.add(new Watermark(25000));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
    TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) testHarness.getSideOutput(lateOutputTag), new Tuple2ResultSortComparator());
    testHarness.close();
}

Example 49 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class WindowOperatorTest method testCleanupTimerWithEmptyListStateForSessionWindows.

@Test
public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception {
    final int GAP_SIZE = 3;
    final long LATENESS = 10;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc = new ListStateDescriptor<>("window-contents", inputType.createSerializer(new ExecutionConfig()));
    WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalIterableWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), LATENESS, null);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.open();
    ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processWatermark(new Watermark(4998));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
    expected.add(new Watermark(4998));
    testHarness.processWatermark(new Watermark(14600));
    expected.add(new Watermark(14600));
    ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
    testHarness.close();
}

Example 50 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class WindowOperatorTest method testCountTrigger.

@Test
@SuppressWarnings("unchecked")
public void testCountTrigger() throws Exception {
    closeCalled.set(0);
    final int WINDOW_SIZE = 4;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
    WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, GlobalWindow> operator = new WindowOperator<>(GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()), PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE)), 0, null);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    // The global window actually ignores these timestamps...
    // add elements out-of-order
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
    // do a snapshot, close and restore again
    OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
    testHarness.close();
    ConcurrentLinkedQueue<Object> outputBeforeClose = testHarness.getOutput();
    stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));
    operator = new WindowOperator<>(GlobalWindows.create(), new GlobalWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()), PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE)), 0, null);
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, Iterables.concat(outputBeforeClose, testHarness.getOutput()), new Tuple2ResultSortComparator());
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, Iterables.concat(outputBeforeClose, testHarness.getOutput()), new Tuple2ResultSortComparator());
    testHarness.close();
}