Search in sources :

Example 31 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class KeyedStateInputFormatTest method testReadTime.

@Test
public void testReadTime() throws Exception {
    OperatorID operatorID = OperatorIDGenerator.fromUid("uid");
    OperatorSubtaskState state = createOperatorSubtaskState(new KeyedProcessOperator<>(new StatefulFunctionWithTime()));
    OperatorState operatorState = new OperatorState(operatorID, 1, 128);
    operatorState.putState(0, state);
    KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new TimerReaderFunction(), Types.INT));
    KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];
    KeyedStateReaderFunction<Integer, Integer> userFunction = new TimerReaderFunction();
    List<Integer> data = readInputSplit(split, userFunction);
    Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}
Also used : KeyGroupRangeInputSplit(org.apache.flink.state.api.input.splits.KeyGroupRangeInputSplit) Configuration(org.apache.flink.configuration.Configuration) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) OperatorState(org.apache.flink.runtime.checkpoint.OperatorState) Test(org.junit.Test)

Example 32 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class KeyedStateInputFormatTest method testReadState.

@Test
public void testReadState() throws Exception {
    OperatorID operatorID = OperatorIDGenerator.fromUid("uid");
    OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
    OperatorState operatorState = new OperatorState(operatorID, 1, 128);
    operatorState.putState(0, state);
    KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
    KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];
    KeyedStateReaderFunction<Integer, Integer> userFunction = new ReaderFunction();
    List<Integer> data = readInputSplit(split, userFunction);
    Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 2, 3), data);
}
Also used : KeyGroupRangeInputSplit(org.apache.flink.state.api.input.splits.KeyGroupRangeInputSplit) Configuration(org.apache.flink.configuration.Configuration) KeyedStateReaderFunction(org.apache.flink.state.api.functions.KeyedStateReaderFunction) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) OperatorState(org.apache.flink.runtime.checkpoint.OperatorState) Test(org.junit.Test)

Example 33 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class ListStateInputFormatTest method testReadListOperatorState.

@Test
public void testReadListOperatorState() throws Exception {
    try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) {
        testHarness.open();
        testHarness.processElement(1, 0);
        testHarness.processElement(2, 0);
        testHarness.processElement(3, 0);
        OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0);
        OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4);
        state.putState(0, subtaskState);
        OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0);
        ListStateInputFormat<Integer> format = new ListStateInputFormat<>(state, new Configuration(), null, descriptor);
        format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
        format.open(split);
        List<Integer> results = new ArrayList<>();
        while (!format.reachedEnd()) {
            results.add(format.nextRecord(0));
        }
        results.sort(Comparator.naturalOrder());
        Assert.assertEquals("Failed to read correct list state from state backend", Arrays.asList(1, 2, 3), results);
    }
}
Also used : MockStreamingRuntimeContext(org.apache.flink.streaming.util.MockStreamingRuntimeContext) OperatorStateInputSplit(org.apache.flink.state.api.input.splits.OperatorStateInputSplit) Configuration(org.apache.flink.configuration.Configuration) ArrayList(java.util.ArrayList) OperatorState(org.apache.flink.runtime.checkpoint.OperatorState) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) Test(org.junit.Test)

Example 34 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class KeyedStateInputFormatTest method testInvalidProcessReaderFunctionFails.

@Test(expected = IOException.class)
public void testInvalidProcessReaderFunctionFails() throws Exception {
    OperatorID operatorID = OperatorIDGenerator.fromUid("uid");
    OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
    OperatorState operatorState = new OperatorState(operatorID, 1, 128);
    operatorState.putState(0, state);
    KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
    KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];
    KeyedStateReaderFunction<Integer, Integer> userFunction = new InvalidReaderFunction();
    readInputSplit(split, userFunction);
    Assert.fail("KeyedStateReaderFunction did not fail on invalid RuntimeContext use");
}
Also used : KeyGroupRangeInputSplit(org.apache.flink.state.api.input.splits.KeyGroupRangeInputSplit) Configuration(org.apache.flink.configuration.Configuration) KeyedStateReaderFunction(org.apache.flink.state.api.functions.KeyedStateReaderFunction) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) OperatorState(org.apache.flink.runtime.checkpoint.OperatorState) Test(org.junit.Test)

Example 35 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class UnionStateInputFormatTest method testReadUnionOperatorState.

@Test
public void testReadUnionOperatorState() throws Exception {
    try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) {
        testHarness.open();
        testHarness.processElement(1, 0);
        testHarness.processElement(2, 0);
        testHarness.processElement(3, 0);
        OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0);
        OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4);
        state.putState(0, subtaskState);
        OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0);
        UnionStateInputFormat<Integer> format = new UnionStateInputFormat<>(state, new Configuration(), null, descriptor);
        format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
        format.open(split);
        List<Integer> results = new ArrayList<>();
        while (!format.reachedEnd()) {
            results.add(format.nextRecord(0));
        }
        results.sort(Comparator.naturalOrder());
        Assert.assertEquals("Failed to read correct list state from state backend", Arrays.asList(1, 2, 3), results);
    }
}
Also used : MockStreamingRuntimeContext(org.apache.flink.streaming.util.MockStreamingRuntimeContext) OperatorStateInputSplit(org.apache.flink.state.api.input.splits.OperatorStateInputSplit) Configuration(org.apache.flink.configuration.Configuration) ArrayList(java.util.ArrayList) OperatorState(org.apache.flink.runtime.checkpoint.OperatorState) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) Test(org.junit.Test)

Aggregations

OperatorSubtaskState (org.apache.flink.runtime.checkpoint.OperatorSubtaskState)178 Test (org.junit.Test)142 Watermark (org.apache.flink.streaming.api.watermark.Watermark)52 ConcurrentLinkedQueue (java.util.concurrent.ConcurrentLinkedQueue)37 RowData (org.apache.flink.table.data.RowData)31 ArrayList (java.util.ArrayList)28 KeyedOneInputStreamOperatorTestHarness (org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness)25 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)23 Map (java.util.Map)22 OperatorID (org.apache.flink.runtime.jobgraph.OperatorID)21 OneInputStreamOperatorTestHarness (org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness)19 HashMap (java.util.HashMap)18 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)18 TypeHint (org.apache.flink.api.common.typeinfo.TypeHint)16 Event (org.apache.flink.cep.Event)16 SubEvent (org.apache.flink.cep.SubEvent)16 TimeWindow (org.apache.flink.streaming.api.windowing.windows.TimeWindow)15 GenericRowData (org.apache.flink.table.data.GenericRowData)15 Ignore (org.junit.Ignore)15 TaskStateSnapshot (org.apache.flink.runtime.checkpoint.TaskStateSnapshot)14