Examples with OperatorState org.apache.flink.runtime.checkpoint.OperatorState used on opensource projects

Example 51 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class SavepointWriter method fromExistingSavepoint.

/**
 * Loads an existing savepoint. Useful if you want to modify or extend the state of an existing
 * application. The savepoint will be written using the state backend defined via the clusters
 * configuration.
 *
 * @param path The path to an existing savepoint on disk.
 * @return A {@link SavepointWriter}.
 * @see #fromExistingSavepoint(String, StateBackend)
 * @see #withConfiguration(ConfigOption, Object)
 */
public static SavepointWriter fromExistingSavepoint(String path) throws IOException {
    CheckpointMetadata metadata = SavepointLoader.loadSavepointMetadata(path);
    int maxParallelism = metadata.getOperatorStates().stream().map(OperatorState::getMaxParallelism).max(Comparator.naturalOrder()).orElseThrow(() -> new RuntimeException("Savepoint must contain at least one operator state."));
    SavepointMetadataV2 savepointMetadata = new SavepointMetadataV2(maxParallelism, metadata.getMasterStates(), metadata.getOperatorStates());
    return new SavepointWriter(savepointMetadata, null);
}

Example 52 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeMetadata.

protected CheckpointMetadata deserializeMetadata(DataInputStream dis, @Nullable String externalPointer) throws IOException {
    final DeserializationContext context = externalPointer == null ? null : new DeserializationContext(externalPointer);
    // first: checkpoint ID
    final long checkpointId = dis.readLong();
    if (checkpointId < 0) {
        throw new IOException("invalid checkpoint ID: " + checkpointId);
    }
    // second: master state
    final List<MasterState> masterStates;
    final int numMasterStates = dis.readInt();
    if (numMasterStates == 0) {
        masterStates = Collections.emptyList();
    } else if (numMasterStates > 0) {
        masterStates = new ArrayList<>(numMasterStates);
        for (int i = 0; i < numMasterStates; i++) {
            masterStates.add(deserializeMasterState(dis));
        }
    } else {
        throw new IOException("invalid number of master states: " + numMasterStates);
    }
    // third: operator states
    final int numTaskStates = dis.readInt();
    final List<OperatorState> operatorStates = new ArrayList<>(numTaskStates);
    for (int i = 0; i < numTaskStates; i++) {
        operatorStates.add(deserializeOperatorState(dis, context));
    }
    return new CheckpointMetadata(checkpointId, operatorStates, masterStates);
}

Example 53 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV2V3SerializerBase method serializeMetadata.

// ------------------------------------------------------------------------
// (De)serialization entry points
// ------------------------------------------------------------------------
protected void serializeMetadata(CheckpointMetadata checkpointMetadata, DataOutputStream dos) throws IOException {
    // first: checkpoint ID
    dos.writeLong(checkpointMetadata.getCheckpointId());
    // second: master state
    final Collection<MasterState> masterStates = checkpointMetadata.getMasterStates();
    dos.writeInt(masterStates.size());
    for (MasterState ms : masterStates) {
        serializeMasterState(ms, dos);
    }
    // third: operator states
    Collection<OperatorState> operatorStates = checkpointMetadata.getOperatorStates();
    dos.writeInt(operatorStates.size());
    for (OperatorState operatorState : operatorStates) {
        serializeOperatorState(operatorState, dos);
    }
}

Example 54 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class SchedulerUtilsTest method buildCheckpoint.

private CompletedCheckpoint buildCheckpoint(KeyedStateHandle incremental) {
    OperatorID operatorID = new OperatorID();
    OperatorState operatorState = new OperatorState(operatorID, 1, 1);
    operatorState.putState(0, OperatorSubtaskState.builder().setManagedKeyedState(incremental).build());
    return new CompletedCheckpoint(new JobID(), 1, 1, 1, singletonMap(operatorID, operatorState), emptyList(), CheckpointProperties.forCheckpoint(NEVER_RETAIN_AFTER_TERMINATION), new TestCompletedCheckpointStorageLocation());
}

Example 55 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithOnlyTaskState.

private void testCheckpointWithOnlyTaskState(String basePath) throws Exception {
    final Random rnd = new Random();
    final int maxTaskStates = 20;
    final int maxNumSubtasks = 20;
    for (int i = 0; i < 100; ++i) {
        final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
        final int numTasks = rnd.nextInt(maxTaskStates) + 1;
        final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
        final Collection<OperatorState> taskStates = CheckpointTestUtils.createOperatorStates(rnd, basePath, numTasks, 0, 0, numSubtasks);
        final Collection<MasterState> masterStates = Collections.emptyList();
        testCheckpointSerialization(checkpointId, taskStates, masterStates, basePath);
    }
}