Examples with OperatorStateHandles org.apache.flink.streaming.runtime.tasks.OperatorStateHandles used on opensource projects

Example 11 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class AbstractStreamOperatorTest method testWatermarkCallbackServiceKeyDeletion.

@Test
public void testWatermarkCallbackServiceKeyDeletion() throws Exception {
    final int MAX_PARALLELISM = 10;
    Tuple2<Integer, String> element1 = new Tuple2<>(7, "start");
    Tuple2<Integer, String> element2 = new Tuple2<>(45, "start");
    Tuple2<Integer, String> element3 = new Tuple2<>(90, "start");
    TestOperatorWithDeletingKeyCallback op = new TestOperatorWithDeletingKeyCallback(45);
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness1 = new KeyedOneInputStreamOperatorTestHarness<>(op, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO, MAX_PARALLELISM, 1, 0);
    testHarness1.open();
    testHarness1.processElement(new StreamRecord<>(element1));
    testHarness1.processElement(new StreamRecord<>(element2));
    testHarness1.processWatermark(10L);
    assertEquals(3L, testHarness1.getOutput().size());
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyElement(testHarness1.getOutput().poll(), 45);
    verifyWatermark(testHarness1.getOutput().poll(), 10);
    testHarness1.processElement(new StreamRecord<>(element3));
    testHarness1.processWatermark(20L);
    // because at the first watermark the operator removed key 45
    assertEquals(3L, testHarness1.getOutput().size());
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyElement(testHarness1.getOutput().poll(), 90);
    verifyWatermark(testHarness1.getOutput().poll(), 20);
    testHarness1.processWatermark(25L);
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyElement(testHarness1.getOutput().poll(), 90);
    verifyWatermark(testHarness1.getOutput().poll(), 25);
    // unregister key and then fail
    op.unregisterKey(90);
    // take a snapshot with some elements in internal sorting queue
    OperatorStateHandles snapshot = testHarness1.snapshot(0, 0);
    testHarness1.close();
    testHarness1 = new KeyedOneInputStreamOperatorTestHarness<>(new TestOperatorWithDeletingKeyCallback(45), new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO, MAX_PARALLELISM, 1, 0);
    testHarness1.setup();
    testHarness1.initializeState(snapshot);
    testHarness1.open();
    testHarness1.processWatermark(30L);
    assertEquals(2L, testHarness1.getOutput().size());
    verifyElement(testHarness1.getOutput().poll(), 7);
    verifyWatermark(testHarness1.getOutput().poll(), 30);
    testHarness1.close();
}

Example 12 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class AbstractStreamOperatorTest method testWatermarkCallbackServiceScalingDown.

@Test
public void testWatermarkCallbackServiceScalingDown() throws Exception {
    final int MAX_PARALLELISM = 10;
    KeySelector<Tuple2<Integer, String>, Integer> keySelector = new TestKeySelector();
    Tuple2<Integer, String> element1 = new Tuple2<>(7, "first");
    Tuple2<Integer, String> element2 = new Tuple2<>(45, "start");
    Tuple2<Integer, String> element3 = new Tuple2<>(90, "start");
    Tuple2<Integer, String> element4 = new Tuple2<>(10, "start");
    int keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element1), MAX_PARALLELISM);
    assertEquals(1, keygroup);
    assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
    assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
    keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element2), MAX_PARALLELISM);
    assertEquals(6, keygroup);
    assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
    assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
    keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element3), MAX_PARALLELISM);
    assertEquals(2, keygroup);
    assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
    assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
    keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element4), MAX_PARALLELISM);
    assertEquals(9, keygroup);
    assertEquals(2, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
    assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
    // starting the test, we will go from parallelism of 3 to parallelism of 2
    // first operator
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness1 = getTestHarness(MAX_PARALLELISM, 3, 0);
    testHarness1.open();
    // second operator
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness2 = getTestHarness(MAX_PARALLELISM, 3, 1);
    testHarness2.open();
    // third operator
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness3 = getTestHarness(MAX_PARALLELISM, 3, 2);
    testHarness3.open();
    testHarness1.processWatermark(Long.MIN_VALUE);
    testHarness2.processWatermark(Long.MIN_VALUE);
    testHarness3.processWatermark(Long.MIN_VALUE);
    testHarness1.processElement(new StreamRecord<>(element1));
    testHarness1.processElement(new StreamRecord<>(element3));
    testHarness2.processElement(new StreamRecord<>(element2));
    testHarness3.processElement(new StreamRecord<>(element4));
    // so far we only have the initial watermark
    assertEquals(1, testHarness1.getOutput().size());
    verifyWatermark(testHarness1.getOutput().poll(), Long.MIN_VALUE);
    assertEquals(1, testHarness2.getOutput().size());
    verifyWatermark(testHarness2.getOutput().poll(), Long.MIN_VALUE);
    assertEquals(1, testHarness3.getOutput().size());
    verifyWatermark(testHarness3.getOutput().poll(), Long.MIN_VALUE);
    // we take a snapshot and make it look as a single operator
    // this will be the initial state of all downstream tasks.
    OperatorStateHandles snapshot = AbstractStreamOperatorTestHarness.repackageState(testHarness2.snapshot(0, 0), testHarness1.snapshot(0, 0), testHarness3.snapshot(0, 0));
    // first new operator
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness4 = getTestHarness(MAX_PARALLELISM, 2, 0);
    testHarness4.setup();
    testHarness4.initializeState(snapshot);
    testHarness4.open();
    // second new operator
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness5 = getTestHarness(MAX_PARALLELISM, 2, 1);
    testHarness5.setup();
    testHarness5.initializeState(snapshot);
    testHarness5.open();
    testHarness4.processWatermark(10);
    testHarness5.processWatermark(10);
    assertEquals(3, testHarness4.getOutput().size());
    verifyElement(testHarness4.getOutput().poll(), 7);
    verifyElement(testHarness4.getOutput().poll(), 90);
    verifyWatermark(testHarness4.getOutput().poll(), 10);
    assertEquals(3, testHarness5.getOutput().size());
    verifyElement(testHarness5.getOutput().poll(), 45);
    verifyElement(testHarness5.getOutput().poll(), 10);
    verifyWatermark(testHarness5.getOutput().poll(), 10);
    testHarness1.close();
    testHarness2.close();
    testHarness3.close();
    testHarness4.close();
    testHarness5.close();
}

Example 13 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class ListCheckpointedTest method testUDFReturningNull.

@Test
public void testUDFReturningNull() throws Exception {
    TestUserFunction userFunction = new TestUserFunction(null);
    AbstractStreamOperatorTestHarness<Integer> testHarness = new AbstractStreamOperatorTestHarness<>(new StreamMap<>(userFunction), 1, 1, 0);
    testHarness.open();
    OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
    testHarness.initializeState(snapshot);
    Assert.assertTrue(userFunction.isRestored());
}

Example 14 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class ListCheckpointedTest method testUDFReturningData.

@Test
public void testUDFReturningData() throws Exception {
    TestUserFunction userFunction = new TestUserFunction(Arrays.asList(1, 2, 3));
    AbstractStreamOperatorTestHarness<Integer> testHarness = new AbstractStreamOperatorTestHarness<>(new StreamMap<>(userFunction), 1, 1, 0);
    testHarness.open();
    OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
    testHarness.initializeState(snapshot);
    Assert.assertTrue(userFunction.isRestored());
}

Example 15 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class StreamOperatorSnapshotRestoreTest method testOperatorStatesSnapshotRestore.

@Test
public void testOperatorStatesSnapshotRestore() throws Exception {
    //-------------------------------------------------------------------------- snapshot
    TestOneInputStreamOperator op = new TestOneInputStreamOperator(false);
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, new KeySelector<Integer, Integer>() {

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
    0);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    OperatorStateHandles handles = testHarness.snapshot(1L, 1L);
    testHarness.close();
    //-------------------------------------------------------------------------- restore
    op = new TestOneInputStreamOperator(true);
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, new KeySelector<Integer, Integer>() {

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
    0);
    testHarness.initializeState(handles);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    testHarness.close();
}