Examples with OperatorSubtaskState org.apache.flink.runtime.checkpoint.OperatorSubtaskState used on opensource projects

Example 41 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class CoBroadcastWithNonKeyedOperatorTest method testScaleUp.

@Test
public void testScaleUp() throws Exception {
    final Set<String> keysToRegister = new HashSet<>();
    keysToRegister.add("test1");
    keysToRegister.add("test2");
    keysToRegister.add("test3");
    final OperatorSubtaskState mergedSnapshot;
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness1 = getInitializedTestHarness(new TestFunctionWithOutput(keysToRegister), 10, 2, 0, STATE_DESCRIPTOR);
        TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness2 = getInitializedTestHarness(new TestFunctionWithOutput(keysToRegister), 10, 2, 1, STATE_DESCRIPTOR)) {
        // make sure all operators have the same state
        testHarness1.processElement2(new StreamRecord<>(3));
        testHarness2.processElement2(new StreamRecord<>(3));
        mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(testHarness1.snapshot(0L, 0L), testHarness2.snapshot(0L, 0L));
    }
    final Set<String> expected = new HashSet<>(3);
    expected.add("test1=3");
    expected.add("test2=3");
    expected.add("test3=3");
    final OperatorSubtaskState initState1 = repartitionInitState(mergedSnapshot, 10, 2, 3, 0);
    final OperatorSubtaskState initState2 = repartitionInitState(mergedSnapshot, 10, 2, 3, 1);
    final OperatorSubtaskState initState3 = repartitionInitState(mergedSnapshot, 10, 2, 3, 2);
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness1 = getInitializedTestHarness(new TestFunctionWithOutput(keysToRegister), 10, 3, 0, initState1, STATE_DESCRIPTOR);
        TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness2 = getInitializedTestHarness(new TestFunctionWithOutput(keysToRegister), 10, 3, 1, initState2, STATE_DESCRIPTOR);
        TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness3 = getInitializedTestHarness(new TestFunctionWithOutput(keysToRegister), 10, 3, 2, initState3, STATE_DESCRIPTOR)) {
        testHarness1.processElement1(new StreamRecord<>("trigger"));
        testHarness2.processElement1(new StreamRecord<>("trigger"));
        testHarness3.processElement1(new StreamRecord<>("trigger"));
        Queue<?> output1 = testHarness1.getOutput();
        Queue<?> output2 = testHarness2.getOutput();
        Queue<?> output3 = testHarness3.getOutput();
        Assert.assertEquals(expected.size(), output1.size());
        for (Object o : output1) {
            StreamRecord<String> rec = (StreamRecord<String>) o;
            Assert.assertTrue(expected.contains(rec.getValue()));
        }
        Assert.assertEquals(expected.size(), output2.size());
        for (Object o : output2) {
            StreamRecord<String> rec = (StreamRecord<String>) o;
            Assert.assertTrue(expected.contains(rec.getValue()));
        }
        Assert.assertEquals(expected.size(), output3.size());
        for (Object o : output3) {
            StreamRecord<String> rec = (StreamRecord<String>) o;
            Assert.assertTrue(expected.contains(rec.getValue()));
        }
    }
}

Example 42 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class WrappingFunctionSnapshotRestoreTest method testSnapshotAndRestoreWrappedCheckpointedFunction.

@Test
public void testSnapshotAndRestoreWrappedCheckpointedFunction() throws Exception {
    StreamMap<Integer, Integer> operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(new InnerTestFun())));
    OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(5, 12L));
    // snapshot and restore from scratch
    OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
    testHarness.close();
    InnerTestFun innerTestFun = new InnerTestFun();
    operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun)));
    testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    Assert.assertTrue(innerTestFun.wasRestored);
    testHarness.close();
}

Example 43 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class WrappingFunctionSnapshotRestoreTest method testSnapshotAndRestoreWrappedListCheckpointed.

@Test
public void testSnapshotAndRestoreWrappedListCheckpointed() throws Exception {
    StreamMap<Integer, Integer> operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(new InnerTestFunList())));
    OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(5, 12L));
    // snapshot and restore from scratch
    OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
    testHarness.close();
    InnerTestFunList innerTestFun = new InnerTestFunList();
    operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun)));
    testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    Assert.assertTrue(innerTestFun.wasRestored);
    testHarness.close();
}

Example 44 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class CoBroadcastWithKeyedOperatorTest method testScaleUp.

@Test
public void testScaleUp() throws Exception {
    final Set<String> keysToRegister = new HashSet<>();
    keysToRegister.add("test1");
    keysToRegister.add("test2");
    keysToRegister.add("test3");
    final OperatorSubtaskState mergedSnapshot;
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness1 = getInitializedTestHarness(BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new TestFunctionWithOutput(keysToRegister), 10, 2, 0);
        TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness2 = getInitializedTestHarness(BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new TestFunctionWithOutput(keysToRegister), 10, 2, 1)) {
        // make sure all operators have the same state
        testHarness1.processElement2(new StreamRecord<>(3));
        testHarness2.processElement2(new StreamRecord<>(3));
        mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(testHarness1.snapshot(0L, 0L), testHarness2.snapshot(0L, 0L));
    }
    final Set<String> expected = new HashSet<>(3);
    expected.add("test1=3");
    expected.add("test2=3");
    expected.add("test3=3");
    OperatorSubtaskState operatorSubtaskState1 = repartitionInitState(mergedSnapshot, 10, 2, 3, 0);
    OperatorSubtaskState operatorSubtaskState2 = repartitionInitState(mergedSnapshot, 10, 2, 3, 1);
    OperatorSubtaskState operatorSubtaskState3 = repartitionInitState(mergedSnapshot, 10, 2, 3, 2);
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness1 = getInitializedTestHarness(BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new TestFunctionWithOutput(keysToRegister), 10, 3, 0, operatorSubtaskState1);
        TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness2 = getInitializedTestHarness(BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new TestFunctionWithOutput(keysToRegister), 10, 3, 1, operatorSubtaskState2);
        TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness3 = getInitializedTestHarness(BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new TestFunctionWithOutput(keysToRegister), 10, 3, 2, operatorSubtaskState3)) {
        testHarness1.processElement1(new StreamRecord<>("trigger"));
        testHarness2.processElement1(new StreamRecord<>("trigger"));
        testHarness3.processElement1(new StreamRecord<>("trigger"));
        Queue<?> output1 = testHarness1.getOutput();
        Queue<?> output2 = testHarness2.getOutput();
        Queue<?> output3 = testHarness3.getOutput();
        assertEquals(expected.size(), output1.size());
        for (Object o : output1) {
            StreamRecord<String> rec = (StreamRecord<String>) o;
            assertTrue(expected.contains(rec.getValue()));
        }
        assertEquals(expected.size(), output2.size());
        for (Object o : output2) {
            StreamRecord<String> rec = (StreamRecord<String>) o;
            assertTrue(expected.contains(rec.getValue()));
        }
        assertEquals(expected.size(), output3.size());
        for (Object o : output3) {
            StreamRecord<String> rec = (StreamRecord<String>) o;
            assertTrue(expected.contains(rec.getValue()));
        }
    }
}

Example 45 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class RetractableTopNFunctionTest method testConstantRankRangeWithoutOffsetWithoutRowNumber.

@Test
public void testConstantRankRangeWithoutOffsetWithoutRowNumber() throws Exception {
    AbstractTopNFunction func = createFunction(RankType.ROW_NUMBER, new ConstantRankRange(1, 2), true, false);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(func);
    testHarness.open();
    testHarness.processElement(insertRecord("book", 1L, 12));
    testHarness.processElement(insertRecord("book", 2L, 19));
    testHarness.processElement(insertRecord("book", 4L, 11));
    testHarness.processElement(insertRecord("fruit", 4L, 33));
    testHarness.processElement(insertRecord("fruit", 3L, 44));
    testHarness.processElement(insertRecord("fruit", 5L, 22));
    List<Object> expectedOutput = new ArrayList<>();
    expectedOutput.add(insertRecord("book", 1L, 12));
    expectedOutput.add(insertRecord("book", 2L, 19));
    expectedOutput.add(deleteRecord("book", 2L, 19));
    expectedOutput.add(insertRecord("book", 4L, 11));
    expectedOutput.add(insertRecord("fruit", 4L, 33));
    expectedOutput.add(insertRecord("fruit", 3L, 44));
    expectedOutput.add(deleteRecord("fruit", 3L, 44));
    expectedOutput.add(insertRecord("fruit", 5L, 22));
    assertorWithoutRowNumber.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    // do a snapshot, data could be recovered from state
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    testHarness.close();
    expectedOutput.clear();
    func = createFunction(RankType.ROW_NUMBER, new ConstantRankRange(1, 2), true, false);
    testHarness = createTestHarness(func);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.processElement(insertRecord("book", 1L, 10));
    expectedOutput.add(deleteRecord("book", 1L, 12));
    expectedOutput.add(insertRecord("book", 1L, 10));
    assertorWithoutRowNumber.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}