Examples with PrioritizedOperatorSubtaskState org.apache.flink.runtime.checkpoint.PrioritizedOperatorSubtaskState used on opensource projects

Example 1 with PrioritizedOperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.PrioritizedOperatorSubtaskState in project flink by apache.

the class TestTaskStateManager method prioritizedOperatorState.

@Nonnull
@Override
public PrioritizedOperatorSubtaskState prioritizedOperatorState(OperatorID operatorID) {
    TaskStateSnapshot jmTaskStateSnapshot = getLastJobManagerTaskStateSnapshot();
    TaskStateSnapshot tmTaskStateSnapshot = getLastTaskManagerTaskStateSnapshot();
    if (jmTaskStateSnapshot == null) {
        return PrioritizedOperatorSubtaskState.emptyNotRestored();
    } else {
        OperatorSubtaskState jmOpState = jmTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);
        if (jmOpState == null) {
            return PrioritizedOperatorSubtaskState.emptyNotRestored();
        } else {
            List<OperatorSubtaskState> tmStateCollection = Collections.emptyList();
            if (tmTaskStateSnapshot != null) {
                OperatorSubtaskState tmOpState = tmTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);
                if (tmOpState != null) {
                    tmStateCollection = Collections.singletonList(tmOpState);
                }
            }
            PrioritizedOperatorSubtaskState.Builder builder = new PrioritizedOperatorSubtaskState.Builder(jmOpState, tmStateCollection, reportedCheckpointId);
            return builder.build();
        }
    }
}

Example 2 with PrioritizedOperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.PrioritizedOperatorSubtaskState in project flink by apache.

the class CoordinatorEventsExactlyOnceITCase method readSnapshot.

@Nullable
static StreamStateHandle readSnapshot(TaskStateManager stateManager, OperatorID operatorId) {
    final PrioritizedOperatorSubtaskState poss = stateManager.prioritizedOperatorState(operatorId);
    if (!poss.isRestored()) {
        return null;
    }
    final StateObjectCollection<OperatorStateHandle> opState = stateManager.prioritizedOperatorState(operatorId).getPrioritizedManagedOperatorState().get(0);
    final OperatorStateHandle handle = Iterators.getOnlyElement(opState.iterator());
    return handle.getDelegateStateHandle();
}

Example 3 with PrioritizedOperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.PrioritizedOperatorSubtaskState in project flink by apache.

the class StreamOperatorContextBuilderTest method testStateBackendLoading.

@Test(expected = CustomStateBackendFactory.ExpectedException.class)
public void testStateBackendLoading() throws Exception {
    Configuration configuration = new Configuration();
    configuration.set(StateBackendOptions.STATE_BACKEND, CustomStateBackendFactory.class.getCanonicalName());
    StreamOperatorContextBuilder builder = new StreamOperatorContextBuilder(new MockStreamingRuntimeContext(true, 1, 0), configuration, new OperatorState(new OperatorID(), 1, 128), new PrioritizedOperatorSubtaskStateInputSplit() {

        @Override
        public PrioritizedOperatorSubtaskState getPrioritizedOperatorSubtaskState() {
            return PrioritizedOperatorSubtaskState.emptyNotRestored();
        }

        @Override
        public int getSplitNumber() {
            return 0;
        }
    }, new CloseableRegistry(), null);
    builder.build(LOG);
}

Example 4 with PrioritizedOperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.PrioritizedOperatorSubtaskState in project flink by apache.

the class StreamTaskStateInitializerImpl method streamOperatorStateContext.

// -----------------------------------------------------------------------------------------------------------------
@Override
public StreamOperatorStateContext streamOperatorStateContext(@Nonnull OperatorID operatorID, @Nonnull String operatorClassName, @Nonnull ProcessingTimeService processingTimeService, @Nonnull KeyContext keyContext, @Nullable TypeSerializer<?> keySerializer, @Nonnull CloseableRegistry streamTaskCloseableRegistry, @Nonnull MetricGroup metricGroup, double managedMemoryFraction, boolean isUsingCustomRawKeyedState) throws Exception {
    TaskInfo taskInfo = environment.getTaskInfo();
    OperatorSubtaskDescriptionText operatorSubtaskDescription = new OperatorSubtaskDescriptionText(operatorID, operatorClassName, taskInfo.getIndexOfThisSubtask(), taskInfo.getNumberOfParallelSubtasks());
    final String operatorIdentifierText = operatorSubtaskDescription.toString();
    final PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates = taskStateManager.prioritizedOperatorState(operatorID);
    CheckpointableKeyedStateBackend<?> keyedStatedBackend = null;
    OperatorStateBackend operatorStateBackend = null;
    CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs = null;
    CloseableIterable<StatePartitionStreamProvider> rawOperatorStateInputs = null;
    InternalTimeServiceManager<?> timeServiceManager;
    try {
        // -------------- Keyed State Backend --------------
        keyedStatedBackend = keyedStatedBackend(keySerializer, operatorIdentifierText, prioritizedOperatorSubtaskStates, streamTaskCloseableRegistry, metricGroup, managedMemoryFraction);
        // -------------- Operator State Backend --------------
        operatorStateBackend = operatorStateBackend(operatorIdentifierText, prioritizedOperatorSubtaskStates, streamTaskCloseableRegistry);
        // -------------- Raw State Streams --------------
        rawKeyedStateInputs = rawKeyedStateInputs(prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState().iterator());
        streamTaskCloseableRegistry.registerCloseable(rawKeyedStateInputs);
        rawOperatorStateInputs = rawOperatorStateInputs(prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState().iterator());
        streamTaskCloseableRegistry.registerCloseable(rawOperatorStateInputs);
        // -------------- Internal Timer Service Manager --------------
        if (keyedStatedBackend != null) {
            // if the operator indicates that it is using custom raw keyed state,
            // then whatever was written in the raw keyed state snapshot was NOT written
            // by the internal timer services (because there is only ever one user of raw keyed
            // state);
            // in this case, timers should not attempt to restore timers from the raw keyed
            // state.
            final Iterable<KeyGroupStatePartitionStreamProvider> restoredRawKeyedStateTimers = (prioritizedOperatorSubtaskStates.isRestored() && !isUsingCustomRawKeyedState) ? rawKeyedStateInputs : Collections.emptyList();
            timeServiceManager = timeServiceManagerProvider.create(keyedStatedBackend, environment.getUserCodeClassLoader().asClassLoader(), keyContext, processingTimeService, restoredRawKeyedStateTimers);
        } else {
            timeServiceManager = null;
        }
        return new StreamOperatorStateContextImpl(prioritizedOperatorSubtaskStates.getRestoredCheckpointId(), operatorStateBackend, keyedStatedBackend, timeServiceManager, rawOperatorStateInputs, rawKeyedStateInputs);
    } catch (Exception ex) {
        // cleanup if something went wrong before results got published.
        if (keyedStatedBackend != null) {
            if (streamTaskCloseableRegistry.unregisterCloseable(keyedStatedBackend)) {
                IOUtils.closeQuietly(keyedStatedBackend);
            }
            // release resource (e.g native resource)
            keyedStatedBackend.dispose();
        }
        if (operatorStateBackend != null) {
            if (streamTaskCloseableRegistry.unregisterCloseable(operatorStateBackend)) {
                IOUtils.closeQuietly(operatorStateBackend);
            }
            operatorStateBackend.dispose();
        }
        if (streamTaskCloseableRegistry.unregisterCloseable(rawKeyedStateInputs)) {
            IOUtils.closeQuietly(rawKeyedStateInputs);
        }
        if (streamTaskCloseableRegistry.unregisterCloseable(rawOperatorStateInputs)) {
            IOUtils.closeQuietly(rawOperatorStateInputs);
        }
        throw new Exception("Exception while creating StreamOperatorStateContext.", ex);
    }
}

Example 5 with PrioritizedOperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.PrioritizedOperatorSubtaskState in project flink by apache.

the class TaskStateManagerImplTest method testStateReportingAndRetrieving.

/**
 * Test reporting and retrieving prioritized local and remote state.
 */
@Test
public void testStateReportingAndRetrieving() {
    JobID jobID = new JobID();
    ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    TestCheckpointResponder testCheckpointResponder = new TestCheckpointResponder();
    TestTaskLocalStateStore testTaskLocalStateStore = new TestTaskLocalStateStore();
    InMemoryStateChangelogStorage changelogStorage = new InMemoryStateChangelogStorage();
    TaskStateManager taskStateManager = taskStateManager(jobID, executionAttemptID, testCheckpointResponder, null, testTaskLocalStateStore, changelogStorage);
    // ---------------------------------------- test reporting
    // -----------------------------------------
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(74L, 11L);
    CheckpointMetrics checkpointMetrics = new CheckpointMetrics();
    TaskStateSnapshot jmTaskStateSnapshot = new TaskStateSnapshot();
    OperatorID operatorID_1 = new OperatorID(1L, 1L);
    OperatorID operatorID_2 = new OperatorID(2L, 2L);
    OperatorID operatorID_3 = new OperatorID(3L, 3L);
    Assert.assertFalse(taskStateManager.prioritizedOperatorState(operatorID_1).isRestored());
    Assert.assertFalse(taskStateManager.prioritizedOperatorState(operatorID_2).isRestored());
    Assert.assertFalse(taskStateManager.prioritizedOperatorState(operatorID_3).isRestored());
    KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
    // Remote state of operator 1 has only managed keyed state.
    OperatorSubtaskState jmOperatorSubtaskState_1 = OperatorSubtaskState.builder().setManagedKeyedState(StateHandleDummyUtil.createNewKeyedStateHandle(keyGroupRange)).build();
    // Remote state of operator 1 has only raw keyed state.
    OperatorSubtaskState jmOperatorSubtaskState_2 = OperatorSubtaskState.builder().setRawKeyedState(StateHandleDummyUtil.createNewKeyedStateHandle(keyGroupRange)).build();
    jmTaskStateSnapshot.putSubtaskStateByOperatorID(operatorID_1, jmOperatorSubtaskState_1);
    jmTaskStateSnapshot.putSubtaskStateByOperatorID(operatorID_2, jmOperatorSubtaskState_2);
    TaskStateSnapshot tmTaskStateSnapshot = new TaskStateSnapshot();
    // Only operator 1 has a local alternative for the managed keyed state.
    OperatorSubtaskState tmOperatorSubtaskState_1 = OperatorSubtaskState.builder().setManagedKeyedState(StateHandleDummyUtil.createNewKeyedStateHandle(keyGroupRange)).build();
    tmTaskStateSnapshot.putSubtaskStateByOperatorID(operatorID_1, tmOperatorSubtaskState_1);
    taskStateManager.reportTaskStateSnapshots(checkpointMetaData, checkpointMetrics, jmTaskStateSnapshot, tmTaskStateSnapshot);
    TestCheckpointResponder.AcknowledgeReport acknowledgeReport = testCheckpointResponder.getAcknowledgeReports().get(0);
    // checks that the checkpoint responder and the local state store received state as
    // expected.
    Assert.assertEquals(checkpointMetaData.getCheckpointId(), acknowledgeReport.getCheckpointId());
    Assert.assertEquals(checkpointMetrics, acknowledgeReport.getCheckpointMetrics());
    Assert.assertEquals(executionAttemptID, acknowledgeReport.getExecutionAttemptID());
    Assert.assertEquals(jobID, acknowledgeReport.getJobID());
    Assert.assertEquals(jmTaskStateSnapshot, acknowledgeReport.getSubtaskState());
    Assert.assertEquals(tmTaskStateSnapshot, testTaskLocalStateStore.retrieveLocalState(checkpointMetaData.getCheckpointId()));
    // -------------------------------------- test prio retrieving
    // ---------------------------------------
    JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(checkpointMetaData.getCheckpointId(), acknowledgeReport.getSubtaskState());
    taskStateManager = taskStateManager(jobID, executionAttemptID, testCheckpointResponder, taskRestore, testTaskLocalStateStore, changelogStorage);
    // this has remote AND local managed keyed state.
    PrioritizedOperatorSubtaskState prioritized_1 = taskStateManager.prioritizedOperatorState(operatorID_1);
    // this has only remote raw keyed state.
    PrioritizedOperatorSubtaskState prioritized_2 = taskStateManager.prioritizedOperatorState(operatorID_2);
    // not restored.
    PrioritizedOperatorSubtaskState prioritized_3 = taskStateManager.prioritizedOperatorState(operatorID_3);
    Assert.assertTrue(prioritized_1.isRestored());
    Assert.assertTrue(prioritized_2.isRestored());
    Assert.assertTrue(prioritized_3.isRestored());
    Assert.assertTrue(taskStateManager.prioritizedOperatorState(new OperatorID()).isRestored());
    // checks for operator 1.
    Iterator<StateObjectCollection<KeyedStateHandle>> prioritizedManagedKeyedState_1 = prioritized_1.getPrioritizedManagedKeyedState().iterator();
    Assert.assertTrue(prioritizedManagedKeyedState_1.hasNext());
    StateObjectCollection<KeyedStateHandle> current = prioritizedManagedKeyedState_1.next();
    KeyedStateHandle keyedStateHandleExp = tmOperatorSubtaskState_1.getManagedKeyedState().iterator().next();
    KeyedStateHandle keyedStateHandleAct = current.iterator().next();
    Assert.assertTrue(keyedStateHandleExp == keyedStateHandleAct);
    Assert.assertTrue(prioritizedManagedKeyedState_1.hasNext());
    current = prioritizedManagedKeyedState_1.next();
    keyedStateHandleExp = jmOperatorSubtaskState_1.getManagedKeyedState().iterator().next();
    keyedStateHandleAct = current.iterator().next();
    Assert.assertTrue(keyedStateHandleExp == keyedStateHandleAct);
    Assert.assertFalse(prioritizedManagedKeyedState_1.hasNext());
    // checks for operator 2.
    Iterator<StateObjectCollection<KeyedStateHandle>> prioritizedRawKeyedState_2 = prioritized_2.getPrioritizedRawKeyedState().iterator();
    Assert.assertTrue(prioritizedRawKeyedState_2.hasNext());
    current = prioritizedRawKeyedState_2.next();
    keyedStateHandleExp = jmOperatorSubtaskState_2.getRawKeyedState().iterator().next();
    keyedStateHandleAct = current.iterator().next();
    Assert.assertTrue(keyedStateHandleExp == keyedStateHandleAct);
    Assert.assertFalse(prioritizedRawKeyedState_2.hasNext());
}