Examples with FullyFinishedOperatorState org.apache.flink.runtime.checkpoint.FullyFinishedOperatorState used on opensource projects

Example 1 with FullyFinishedOperatorState

use of org.apache.flink.runtime.checkpoint.FullyFinishedOperatorState in project flink by apache.

the class CheckpointTestUtils method createOperatorStates.

/**
 * Creates a random collection of OperatorState objects containing various types of state
 * handles.
 *
 * @param basePath The basePath for savepoint, will be null for checkpoint.
 * @param numAllRunningTaskStates Number of tasks belong to all running vertex.
 * @param numPartlyFinishedTaskStates Number of tasks belong to partly finished vertex.
 * @param numFullyFinishedTaskStates Number of tasks belong to fully finished vertex.
 * @param numSubtasksPerTask Number of subtasks for each task.
 */
public static Collection<OperatorState> createOperatorStates(Random random, @Nullable String basePath, int numAllRunningTaskStates, int numPartlyFinishedTaskStates, int numFullyFinishedTaskStates, int numSubtasksPerTask) {
    List<OperatorState> taskStates = new ArrayList<>(numAllRunningTaskStates + numPartlyFinishedTaskStates + numFullyFinishedTaskStates);
    for (int stateIdx = 0; stateIdx < numAllRunningTaskStates; ++stateIdx) {
        OperatorState taskState = new OperatorState(new OperatorID(), numSubtasksPerTask, 128);
        randomlySetCoordinatorState(taskState, random);
        randomlySetSubtaskState(taskState, IntStream.range(0, numSubtasksPerTask).toArray(), random, basePath);
        taskStates.add(taskState);
    }
    for (int stateIdx = 0; stateIdx < numPartlyFinishedTaskStates; ++stateIdx) {
        OperatorState taskState = new OperatorState(new OperatorID(), numSubtasksPerTask, 128);
        randomlySetCoordinatorState(taskState, random);
        randomlySetSubtaskState(taskState, IntStream.range(0, numSubtasksPerTask / 2).toArray(), random, basePath);
        IntStream.range(numSubtasksPerTask / 2, numSubtasksPerTask).forEach(index -> taskState.putState(index, FinishedOperatorSubtaskState.INSTANCE));
        taskStates.add(taskState);
    }
    for (int stateIdx = 0; stateIdx < numFullyFinishedTaskStates; ++stateIdx) {
        taskStates.add(new FullyFinishedOperatorState(new OperatorID(), numSubtasksPerTask, 128));
    }
    return taskStates;
}

Example 2 with FullyFinishedOperatorState

use of org.apache.flink.runtime.checkpoint.FullyFinishedOperatorState in project flink by apache.

the class MetadataV3Serializer method deserializeOperatorState.

@Override
protected OperatorState deserializeOperatorState(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
    final OperatorID jobVertexId = new OperatorID(dis.readLong(), dis.readLong());
    final int parallelism = dis.readInt();
    final int maxParallelism = dis.readInt();
    ByteStreamStateHandle coordinateState = deserializeAndCheckByteStreamStateHandle(dis, context);
    final int numSubTaskStates = dis.readInt();
    if (numSubTaskStates < 0) {
        checkState(coordinateState == null, "Coordinator State should be null for fully finished operator state");
        return new FullyFinishedOperatorState(jobVertexId, parallelism, maxParallelism);
    }
    final OperatorState operatorState = new OperatorState(jobVertexId, parallelism, maxParallelism);
    // Coordinator state
    operatorState.setCoordinatorState(coordinateState);
    // Sub task states
    for (int j = 0; j < numSubTaskStates; j++) {
        SubtaskAndFinishedState subtaskAndFinishedState = deserializeSubtaskIndexAndFinishedState(dis);
        if (subtaskAndFinishedState.isFinished) {
            operatorState.putState(subtaskAndFinishedState.subtaskIndex, FinishedOperatorSubtaskState.INSTANCE);
        } else {
            final OperatorSubtaskState subtaskState = deserializeSubtaskState(dis, context);
            operatorState.putState(subtaskAndFinishedState.subtaskIndex, subtaskState);
        }
    }
    return operatorState;
}