Examples with KeyedOneInputStreamOperatorTestHarness org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness used on opensource projects

Example 16 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class AbstractStreamOperatorTest method testEnsureProcessingTimeTimerRegisteredOnRestore.

/**
	 * Verify that a low-level timer is set for processing-time timers in case of restore.
	 */
@Test
public void testEnsureProcessingTimeTimerRegisteredOnRestore() throws Exception {
    TestOperator testOperator = new TestOperator();
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.open();
    testHarness.setProcessingTime(0L);
    testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);
    testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
    testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);
    testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);
    OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
    TestOperator testOperator1 = new TestOperator();
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness1 = new KeyedOneInputStreamOperatorTestHarness<>(testOperator1, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness1.setProcessingTime(0L);
    testHarness1.setup();
    testHarness1.initializeState(snapshot);
    testHarness1.open();
    testHarness1.setProcessingTime(10L);
    assertThat(extractResult(testHarness1), contains("ON_PROC_TIME:HELLO"));
    testHarness1.setProcessingTime(20L);
    assertThat(extractResult(testHarness1), contains("ON_PROC_TIME:CIAO"));
}

Example 17 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class AbstractStreamOperatorTest method testWatermarkCallbackServiceKeyDeletion.

@Test
public void testWatermarkCallbackServiceKeyDeletion() throws Exception {
    final int MAX_PARALLELISM = 10;
    Tuple2<Integer, String> element1 = new Tuple2<>(7, "start");
    Tuple2<Integer, String> element2 = new Tuple2<>(45, "start");
    Tuple2<Integer, String> element3 = new Tuple2<>(90, "start");
    TestOperatorWithDeletingKeyCallback op = new TestOperatorWithDeletingKeyCallback(45);
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness1 = new KeyedOneInputStreamOperatorTestHarness<>(op, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO, MAX_PARALLELISM, 1, 0);
    testHarness1.open();
    testHarness1.processElement(new StreamRecord<>(element1));
    testHarness1.processElement(new StreamRecord<>(element2));
    testHarness1.processWatermark(10L);
    assertEquals(3L, testHarness1.getOutput().size());
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyElement(testHarness1.getOutput().poll(), 45);
    verifyWatermark(testHarness1.getOutput().poll(), 10);
    testHarness1.processElement(new StreamRecord<>(element3));
    testHarness1.processWatermark(20L);
    // because at the first watermark the operator removed key 45
    assertEquals(3L, testHarness1.getOutput().size());
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyElement(testHarness1.getOutput().poll(), 90);
    verifyWatermark(testHarness1.getOutput().poll(), 20);
    testHarness1.processWatermark(25L);
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyElement(testHarness1.getOutput().poll(), 90);
    verifyWatermark(testHarness1.getOutput().poll(), 25);
    // unregister key and then fail
    op.unregisterKey(90);
    // take a snapshot with some elements in internal sorting queue
    OperatorStateHandles snapshot = testHarness1.snapshot(0, 0);
    testHarness1.close();
    testHarness1 = new KeyedOneInputStreamOperatorTestHarness<>(new TestOperatorWithDeletingKeyCallback(45), new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO, MAX_PARALLELISM, 1, 0);
    testHarness1.setup();
    testHarness1.initializeState(snapshot);
    testHarness1.open();
    testHarness1.processWatermark(30L);
    assertEquals(2L, testHarness1.getOutput().size());
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyWatermark(testHarness1.getOutput().poll(), 30);
    testHarness1.close();
}

Example 18 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class StreamOperatorSnapshotRestoreTest method testOperatorStatesSnapshotRestore.

@Test
public void testOperatorStatesSnapshotRestore() throws Exception {
    //-------------------------------------------------------------------------- snapshot
    TestOneInputStreamOperator op = new TestOneInputStreamOperator(false);
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, new KeySelector<Integer, Integer>() {

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
    0);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    OperatorStateHandles handles = testHarness.snapshot(1L, 1L);
    testHarness.close();
    //-------------------------------------------------------------------------- restore
    op = new TestOneInputStreamOperator(true);
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, new KeySelector<Integer, Integer>() {

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
    0);
    testHarness.initializeState(handles);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    testHarness.close();
}

Example 19 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class KeyedProcessOperatorTest method testTimestampAndWatermarkQuerying.

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {
    KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME));
    OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark(new Watermark(17));
    testHarness.processElement(new StreamRecord<>(5, 12L));
    testHarness.processWatermark(new Watermark(42));
    testHarness.processElement(new StreamRecord<>(6, 13L));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(17L));
    expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
    expectedOutput.add(new Watermark(42L));
    expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 20 with KeyedOneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness in project flink by apache.

the class KeyedProcessOperatorTest method testSnapshotAndRestore.

@Test
public void testSnapshotAndRestore() throws Exception {
    KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction());
    OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(5, 12L));
    // snapshot and restore from scratch
    OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
    testHarness.close();
    operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction());
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.setProcessingTime(5);
    testHarness.processWatermark(new Watermark(6));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>("PROC:1777", 5L));
    expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L));
    expectedOutput.add(new Watermark(6));
    System.out.println("GOT: " + testHarness.getOutput());
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}