Examples with OperatorStateHandles org.apache.flink.streaming.runtime.tasks.OperatorStateHandles used on opensource projects

Example 6 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class AbstractStreamOperatorTestHarness method initializeStateFromLegacyCheckpoint.

public void initializeStateFromLegacyCheckpoint(String checkpointFilename) throws Exception {
    FileInputStream fin = new FileInputStream(checkpointFilename);
    StreamTaskState state = MigrationInstantiationUtil.deserializeObject(fin, ClassLoader.getSystemClassLoader());
    fin.close();
    if (!setupCalled) {
        setup();
    }
    StreamStateHandle stateHandle = SavepointV0Serializer.convertOperatorAndFunctionState(state);
    List<KeyGroupsStateHandle> keyGroupStatesList = new ArrayList<>();
    if (state.getKvStates() != null) {
        KeyGroupsStateHandle keyedStateHandle = SavepointV0Serializer.convertKeyedBackendState(state.getKvStates(), environment.getTaskInfo().getIndexOfThisSubtask(), 0);
        keyGroupStatesList.add(keyedStateHandle);
    }
    // finally calling the initializeState() with the legacy operatorStateHandles
    initializeState(new OperatorStateHandles(0, stateHandle, keyGroupStatesList, Collections.<KeyGroupsStateHandle>emptyList(), Collections.<OperatorStateHandle>emptyList(), Collections.<OperatorStateHandle>emptyList()));
}

Example 7 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class AbstractStreamOperatorTestHarness method initializeState.

/**
	 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#initializeState(OperatorStateHandles)}.
	 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#setup(StreamTask, StreamConfig, Output)}
	 * if it was not called before.
	 *
	 * <p>This will reshape the state handles to include only those key-group states
	 * in the local key-group range and the operator states that would be assigned to the local
	 * subtask.
	 */
public void initializeState(OperatorStateHandles operatorStateHandles) throws Exception {
    if (!setupCalled) {
        setup();
    }
    if (operatorStateHandles != null) {
        int numKeyGroups = getEnvironment().getTaskInfo().getMaxNumberOfParallelSubtasks();
        int numSubtasks = getEnvironment().getTaskInfo().getNumberOfParallelSubtasks();
        int subtaskIndex = getEnvironment().getTaskInfo().getIndexOfThisSubtask();
        // create a new OperatorStateHandles that only contains the state for our key-groups
        List<KeyGroupRange> keyGroupPartitions = StateAssignmentOperation.createKeyGroupPartitions(numKeyGroups, numSubtasks);
        KeyGroupRange localKeyGroupRange = keyGroupPartitions.get(subtaskIndex);
        List<KeyGroupsStateHandle> localManagedKeyGroupState = null;
        if (operatorStateHandles.getManagedKeyedState() != null) {
            localManagedKeyGroupState = StateAssignmentOperation.getKeyGroupsStateHandles(operatorStateHandles.getManagedKeyedState(), localKeyGroupRange);
        }
        List<KeyGroupsStateHandle> localRawKeyGroupState = null;
        if (operatorStateHandles.getRawKeyedState() != null) {
            localRawKeyGroupState = StateAssignmentOperation.getKeyGroupsStateHandles(operatorStateHandles.getRawKeyedState(), localKeyGroupRange);
        }
        List<OperatorStateHandle> managedOperatorState = new ArrayList<>();
        if (operatorStateHandles.getManagedOperatorState() != null) {
            managedOperatorState.addAll(operatorStateHandles.getManagedOperatorState());
        }
        Collection<OperatorStateHandle> localManagedOperatorState = operatorStateRepartitioner.repartitionState(managedOperatorState, numSubtasks).get(subtaskIndex);
        List<OperatorStateHandle> rawOperatorState = new ArrayList<>();
        if (operatorStateHandles.getRawOperatorState() != null) {
            rawOperatorState.addAll(operatorStateHandles.getRawOperatorState());
        }
        Collection<OperatorStateHandle> localRawOperatorState = operatorStateRepartitioner.repartitionState(rawOperatorState, numSubtasks).get(subtaskIndex);
        OperatorStateHandles massagedOperatorStateHandles = new OperatorStateHandles(0, operatorStateHandles.getLegacyOperatorState(), localManagedKeyGroupState, localRawKeyGroupState, localManagedOperatorState, localRawOperatorState);
        operator.initializeState(massagedOperatorStateHandles);
    } else {
        operator.initializeState(null);
    }
    initializeCalled = true;
}

Example 8 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class AbstractStreamOperatorTestHarness method snapshot.

/**
	 * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions)}.
	 */
public OperatorStateHandles snapshot(long checkpointId, long timestamp) throws Exception {
    CheckpointStreamFactory streamFactory = stateBackend.createStreamFactory(new JobID(), "test_op");
    OperatorSnapshotResult operatorStateResult = operator.snapshotState(checkpointId, timestamp, CheckpointOptions.forFullCheckpoint());
    KeyGroupsStateHandle keyedManaged = FutureUtil.runIfNotDoneAndGet(operatorStateResult.getKeyedStateManagedFuture());
    KeyGroupsStateHandle keyedRaw = FutureUtil.runIfNotDoneAndGet(operatorStateResult.getKeyedStateRawFuture());
    OperatorStateHandle opManaged = FutureUtil.runIfNotDoneAndGet(operatorStateResult.getOperatorStateManagedFuture());
    OperatorStateHandle opRaw = FutureUtil.runIfNotDoneAndGet(operatorStateResult.getOperatorStateRawFuture());
    return new OperatorStateHandles(0, null, keyedManaged != null ? Collections.singletonList(keyedManaged) : null, keyedRaw != null ? Collections.singletonList(keyedRaw) : null, opManaged != null ? Collections.singletonList(opManaged) : null, opRaw != null ? Collections.singletonList(opRaw) : null);
}

Example 9 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class CEPOperatorTest method testKeyedCEPOperatorCheckpointing.

@Test
public void testKeyedCEPOperatorCheckpointing() throws Exception {
    OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness = getCepTestHarness(false);
    harness.open();
    Event startEvent = new Event(42, "start", 1.0);
    SubEvent middleEvent = new SubEvent(42, "foo", 1.0, 10.0);
    Event endEvent = new Event(42, "end", 1.0);
    harness.processElement(new StreamRecord<>(startEvent, 1L));
    harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 2L));
    // simulate snapshot/restore with some elements in internal sorting queue
    OperatorStateHandles snapshot = harness.snapshot(0L, 0L);
    harness.close();
    harness = getCepTestHarness(false);
    harness.setup();
    harness.initializeState(snapshot);
    harness.open();
    harness.processWatermark(new Watermark(Long.MIN_VALUE));
    harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3L));
    // if element timestamps are not correctly checkpointed/restored this will lead to
    // a pruning time underflow exception in NFA
    harness.processWatermark(new Watermark(2L));
    harness.processElement(new StreamRecord<Event>(middleEvent, 3L));
    harness.processElement(new StreamRecord<>(new Event(42, "start", 1.0), 4L));
    harness.processElement(new StreamRecord<>(endEvent, 5L));
    // simulate snapshot/restore with empty element queue but NFA state
    OperatorStateHandles snapshot2 = harness.snapshot(1L, 1L);
    harness.close();
    harness = getCepTestHarness(false);
    harness.setup();
    harness.initializeState(snapshot2);
    harness.open();
    harness.processWatermark(new Watermark(Long.MAX_VALUE));
    // get and verify the output
    Queue<Object> result = harness.getOutput();
    assertEquals(2, result.size());
    verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
    verifyWatermark(result.poll(), Long.MAX_VALUE);
    harness.close();
}

Example 10 with OperatorStateHandles

use of org.apache.flink.streaming.runtime.tasks.OperatorStateHandles in project flink by apache.

the class AbstractStreamOperatorTest method testEnsureProcessingTimeTimerRegisteredOnRestore.

/**
	 * Verify that a low-level timer is set for processing-time timers in case of restore.
	 */
@Test
public void testEnsureProcessingTimeTimerRegisteredOnRestore() throws Exception {
    TestOperator testOperator = new TestOperator();
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.open();
    testHarness.setProcessingTime(0L);
    testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0);
    testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0);
    testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0);
    testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0);
    OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
    TestOperator testOperator1 = new TestOperator();
    KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness1 = new KeyedOneInputStreamOperatorTestHarness<>(testOperator1, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness1.setProcessingTime(0L);
    testHarness1.setup();
    testHarness1.initializeState(snapshot);
    testHarness1.open();
    testHarness1.setProcessingTime(10L);
    assertThat(extractResult(testHarness1), contains("ON_PROC_TIME:HELLO"));
    testHarness1.setProcessingTime(20L);
    assertThat(extractResult(testHarness1), contains("ON_PROC_TIME:CIAO"));
}