Examples with OperatorState org.apache.flink.runtime.checkpoint.OperatorState used on opensource projects

Example 56 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithMasterAndTaskState.

private void testCheckpointWithMasterAndTaskState(String basePath) throws Exception {
    final Random rnd = new Random();
    final int maxNumMasterStates = 5;
    final int maxTaskStates = 20;
    final int maxNumSubtasks = 20;
    for (int i = 0; i < 100; ++i) {
        final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
        final int numTasks = rnd.nextInt(maxTaskStates) + 1;
        final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
        final Collection<OperatorState> taskStates = CheckpointTestUtils.createOperatorStates(rnd, basePath, numTasks, 0, 0, numSubtasks);
        final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
        final Collection<MasterState> masterStates = CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
        testCheckpointSerialization(checkpointId, taskStates, masterStates, basePath);
    }
}

Example 57 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithFinishedTasks.

private void testCheckpointWithFinishedTasks(String basePath) throws Exception {
    final Random rnd = new Random();
    final int maxNumMasterStates = 5;
    final int maxNumSubtasks = 20;
    final int maxAllRunningTaskStates = 20;
    final int maxPartlyFinishedStates = 10;
    final int maxFullyFinishedSubtasks = 10;
    final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
    final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
    final int numAllRunningTasks = rnd.nextInt(maxAllRunningTaskStates) + 1;
    final int numPartlyFinishedTasks = rnd.nextInt(maxPartlyFinishedStates) + 1;
    final int numFullyFinishedTasks = rnd.nextInt(maxFullyFinishedSubtasks) + 1;
    final Collection<OperatorState> taskStates = CheckpointTestUtils.createOperatorStates(rnd, basePath, numAllRunningTasks, numPartlyFinishedTasks, numFullyFinishedTasks, numSubtasks);
    final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
    final Collection<MasterState> masterStates = CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
    testCheckpointSerialization(checkpointId, taskStates, masterStates, basePath);
}

Example 58 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class CheckpointMetadata method dispose.

@Override
public void dispose() throws Exception {
    for (OperatorState operatorState : operatorStates) {
        operatorState.discardState();
    }
    operatorStates.clear();
    masterStates.clear();
}

Example 59 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV3Serializer method deserializeOperatorState.

@Override
protected OperatorState deserializeOperatorState(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
    final OperatorID jobVertexId = new OperatorID(dis.readLong(), dis.readLong());
    final int parallelism = dis.readInt();
    final int maxParallelism = dis.readInt();
    ByteStreamStateHandle coordinateState = deserializeAndCheckByteStreamStateHandle(dis, context);
    final int numSubTaskStates = dis.readInt();
    if (numSubTaskStates < 0) {
        checkState(coordinateState == null, "Coordinator State should be null for fully finished operator state");
        return new FullyFinishedOperatorState(jobVertexId, parallelism, maxParallelism);
    }
    final OperatorState operatorState = new OperatorState(jobVertexId, parallelism, maxParallelism);
    // Coordinator state
    operatorState.setCoordinatorState(coordinateState);
    // Sub task states
    for (int j = 0; j < numSubTaskStates; j++) {
        SubtaskAndFinishedState subtaskAndFinishedState = deserializeSubtaskIndexAndFinishedState(dis);
        if (subtaskAndFinishedState.isFinished) {
            operatorState.putState(subtaskAndFinishedState.subtaskIndex, FinishedOperatorSubtaskState.INSTANCE);
        } else {
            final OperatorSubtaskState subtaskState = deserializeSubtaskState(dis, context);
            operatorState.putState(subtaskAndFinishedState.subtaskIndex, subtaskState);
        }
    }
    return operatorState;
}

Example 60 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class TestUtils method createOperatorState.

private static Collection<OperatorState> createOperatorState(OperatorID... operatorIds) {
    Random random = new Random();
    Collection<OperatorState> operatorStates = new ArrayList<>(operatorIds.length);
    for (OperatorID operatorId : operatorIds) {
        final OperatorState operatorState = new OperatorState(operatorId, 1, 42);
        final OperatorSubtaskState subtaskState = OperatorSubtaskState.builder().setManagedOperatorState(new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("foobar", new byte[0]))).setInputChannelState(singleton(createNewInputChannelStateHandle(10, random))).setResultSubpartitionState(singleton(createNewResultSubpartitionStateHandle(10, random))).build();
        operatorState.putState(0, subtaskState);
        operatorStates.add(operatorState);
    }
    return operatorStates;
}