Examples with OperatorSubtaskState org.apache.flink.runtime.checkpoint.OperatorSubtaskState used on opensource projects

Example 96 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class CEPOperatorTest method testCEPOperatorCleanupProcessingTime.

@Test
public void testCEPOperatorCleanupProcessingTime() throws Exception {
    Event startEvent1 = new Event(42, "start", 1.0);
    Event startEvent2 = new Event(42, "start", 2.0);
    SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
    SubEvent middleEvent2 = new SubEvent(42, "foo2", 1.0, 10.0);
    SubEvent middleEvent3 = new SubEvent(42, "foo3", 1.0, 10.0);
    Event endEvent1 = new Event(42, "end", 1.0);
    Event endEvent2 = new Event(42, "end", 2.0);
    Event startEventK2 = new Event(43, "start", 1.0);
    CepOperator<Event, Integer, Map<String, List<Event>>> operator = getKeyedCepOperator(true);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);
    try {
        harness.open();
        harness.setProcessingTime(0L);
        harness.processElement(new StreamRecord<>(startEvent1, 1L));
        harness.processElement(new StreamRecord<>(startEventK2, 1L));
        harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 2L));
        harness.processElement(new StreamRecord<Event>(middleEvent1, 2L));
        harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3L));
        assertTrue(!operator.hasNonEmptyPQ(42));
        assertTrue(!operator.hasNonEmptyPQ(43));
        assertTrue(operator.hasNonEmptySharedBuffer(42));
        assertTrue(operator.hasNonEmptySharedBuffer(43));
        harness.setProcessingTime(3L);
        harness.processElement(new StreamRecord<>(startEvent2, 3L));
        harness.processElement(new StreamRecord<Event>(middleEvent2, 4L));
        OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
        harness.close();
        CepOperator<Event, Integer, Map<String, List<Event>>> operator2 = getKeyedCepOperator(true);
        harness = CepOperatorTestUtilities.getCepTestHarness(operator2);
        harness.setup();
        harness.initializeState(snapshot);
        harness.open();
        harness.setProcessingTime(3L);
        harness.processElement(new StreamRecord<>(endEvent1, 5L));
        verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent1, endEvent1);
        verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent2, endEvent1);
        verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent2, endEvent1);
        harness.setProcessingTime(11L);
        harness.processElement(new StreamRecord<Event>(middleEvent3, 11L));
        harness.processElement(new StreamRecord<>(endEvent2, 12L));
        verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent2, endEvent2);
        verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent3, endEvent2);
        harness.setProcessingTime(21L);
        assertTrue(operator2.hasNonEmptySharedBuffer(42));
        harness.processElement(new StreamRecord<>(startEvent1, 21L));
        assertTrue(operator2.hasNonEmptySharedBuffer(42));
        harness.setProcessingTime(49L);
        // TODO: 3/13/17 we have to have another event in order to clean up
        harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 2L));
        // the pattern expired
        assertTrue(!operator2.hasNonEmptySharedBuffer(42));
        assertEquals(0L, harness.numEventTimeTimers());
        assertTrue(!operator2.hasNonEmptyPQ(42));
        assertTrue(!operator2.hasNonEmptyPQ(43));
    } finally {
        harness.close();
    }
}

Example 97 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class CEPOperatorTest method testKeyedCEPOperatorCheckpointingWithRocksDB.

@Test
public void testKeyedCEPOperatorCheckpointingWithRocksDB() throws Exception {
    String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
    RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend(), TernaryBoolean.FALSE);
    rocksDBStateBackend.setDbStoragePath(rocksDbPath);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = getCepTestHarness(false);
    try {
        harness.setStateBackend(rocksDBStateBackend);
        harness.open();
        Event startEvent = new Event(42, "start", 1.0);
        SubEvent middleEvent = new SubEvent(42, "foo", 1.0, 10.0);
        Event endEvent = new Event(42, "end", 1.0);
        harness.processElement(new StreamRecord<>(startEvent, 1L));
        harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 2L));
        // simulate snapshot/restore with some elements in internal sorting queue
        OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
        harness.close();
        harness = getCepTestHarness(false);
        rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
        rocksDBStateBackend.setDbStoragePath(rocksDbPath);
        harness.setStateBackend(rocksDBStateBackend);
        harness.setup();
        harness.initializeState(snapshot);
        harness.open();
        harness.processWatermark(new Watermark(Long.MIN_VALUE));
        harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3L));
        // if element timestamps are not correctly checkpointed/restored this will lead to
        // a pruning time underflow exception in NFA
        harness.processWatermark(new Watermark(2L));
        // simulate snapshot/restore with empty element queue but NFA state
        OperatorSubtaskState snapshot2 = harness.snapshot(1L, 1L);
        harness.close();
        harness = getCepTestHarness(false);
        rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
        rocksDBStateBackend.setDbStoragePath(rocksDbPath);
        harness.setStateBackend(rocksDBStateBackend);
        harness.setup();
        harness.initializeState(snapshot2);
        harness.open();
        harness.processElement(new StreamRecord<Event>(middleEvent, 3L));
        harness.processElement(new StreamRecord<>(new Event(42, "start", 1.0), 4L));
        harness.processElement(new StreamRecord<>(endEvent, 5L));
        harness.processWatermark(new Watermark(Long.MAX_VALUE));
        // get and verify the output
        Queue<Object> result = harness.getOutput();
        assertEquals(2, result.size());
        verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
        verifyWatermark(result.poll(), Long.MAX_VALUE);
    } finally {
        harness.close();
    }
}

Example 98 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class KeyedStateInputFormatTest method testMaxParallelismRespected.

@Test
public void testMaxParallelismRespected() throws Exception {
    OperatorID operatorID = OperatorIDGenerator.fromUid("uid");
    OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
    OperatorState operatorState = new OperatorState(operatorID, 1, 128);
    operatorState.putState(0, state);
    KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
    KeyGroupRangeInputSplit[] splits = format.createInputSplits(129);
    Assert.assertEquals("Failed to properly partition operator state into input splits", 128, splits.length);
}

Example 99 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class KeyedStateInputFormatTest method testReadMultipleOutputPerKey.

@Test
public void testReadMultipleOutputPerKey() throws Exception {
    OperatorID operatorID = OperatorIDGenerator.fromUid("uid");
    OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
    OperatorState operatorState = new OperatorState(operatorID, 1, 128);
    operatorState.putState(0, state);
    KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
    KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];
    KeyedStateReaderFunction<Integer, Integer> userFunction = new DoubleReaderFunction();
    List<Integer> data = readInputSplit(split, userFunction);
    Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}

Example 100 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class WindowReaderTest method getOperatorState.

private static OperatorState getOperatorState(WindowOperator<Integer, Integer, ?, Void, ?> operator) throws Exception {
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> harness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<>(), Types.INT, MAX_PARALLELISM, 1, 0);
    harness.open();
    harness.processElement(1, 0);
    harness.processElement(1, 1);
    OperatorSubtaskState state = harness.snapshot(0, 0L);
    harness.close();
    OperatorID operatorID = OperatorIDGenerator.fromUid(UID);
    OperatorState operatorState = new OperatorState(operatorID, 1, MAX_PARALLELISM);
    operatorState.putState(0, state);
    return operatorState;
}