Examples with MasterState org.apache.flink.runtime.checkpoint.MasterState used on opensource projects

Example 6 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MasterHooks method restoreMasterHooks.

// ------------------------------------------------------------------------
// checkpoint restoring
// ------------------------------------------------------------------------
/**
 * Calls the restore method given checkpoint master hooks and passes the given master state to
 * them where state with a matching name is found.
 *
 * <p>If state is found and no hook with the same name is found, the method throws an exception,
 * unless the {@code allowUnmatchedState} flag is set.
 *
 * @param masterHooks The hooks to call restore on
 * @param states The state to pass to the hooks
 * @param checkpointId The checkpoint ID of the restored checkpoint
 * @param allowUnmatchedState If true, the method fails if not all states are picked up by a
 *     hook.
 * @param log The logger for log messages
 * @throws FlinkException Thrown, if the hooks throw an exception, or the state+ deserialization
 *     fails.
 */
public static void restoreMasterHooks(final Map<String, MasterTriggerRestoreHook<?>> masterHooks, final Collection<MasterState> states, final long checkpointId, final boolean allowUnmatchedState, final Logger log) throws FlinkException {
    // early out
    if (states == null || states.isEmpty() || masterHooks == null || masterHooks.isEmpty()) {
        log.info("No master state to restore");
        return;
    }
    log.info("Calling master restore hooks");
    // collect the hooks
    final LinkedHashMap<String, MasterTriggerRestoreHook<?>> allHooks = new LinkedHashMap<>(masterHooks);
    // first, deserialize all hook state
    final ArrayList<Tuple2<MasterTriggerRestoreHook<?>, Object>> hooksAndStates = new ArrayList<>();
    for (MasterState state : states) {
        if (state != null) {
            final String name = state.name();
            final MasterTriggerRestoreHook<?> hook = allHooks.remove(name);
            if (hook != null) {
                log.debug("Found state to restore for hook '{}'", name);
                Object deserializedState = deserializeState(state, hook);
                hooksAndStates.add(new Tuple2<>(hook, deserializedState));
            } else if (!allowUnmatchedState) {
                throw new IllegalStateException("Found state '" + state.name() + "' which is not resumed by any hook.");
            } else {
                log.info("Dropping unmatched state from '{}'", name);
            }
        }
    }
    // now that all is deserialized, call the hooks
    for (Tuple2<MasterTriggerRestoreHook<?>, Object> hookAndState : hooksAndStates) {
        restoreHook(hookAndState.f1, hookAndState.f0, checkpointId);
    }
    // trigger the remaining hooks without checkpointed state
    for (MasterTriggerRestoreHook<?> hook : allHooks.values()) {
        restoreHook(null, hook, checkpointId);
    }
}

Example 7 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeMasterState.

protected MasterState deserializeMasterState(DataInputStream dis) throws IOException {
    final int magicNumber = dis.readInt();
    if (magicNumber != MASTER_STATE_MAGIC_NUMBER) {
        throw new IOException("incorrect magic number in master styte byte sequence");
    }
    final int numBytes = dis.readInt();
    if (numBytes <= 0) {
        throw new IOException("found zero or negative length for master state bytes");
    }
    final byte[] data = new byte[numBytes];
    dis.readFully(data);
    final DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
    final int version = in.readInt();
    final String name = in.readUTF();
    final byte[] bytes = new byte[in.readInt()];
    in.readFully(bytes);
    // check that the data is not corrupt
    if (in.read() != -1) {
        throw new IOException("found trailing bytes in master state");
    }
    return new MasterState(name, bytes, version);
}

Example 8 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeMetadata.

protected CheckpointMetadata deserializeMetadata(DataInputStream dis, @Nullable String externalPointer) throws IOException {
    final DeserializationContext context = externalPointer == null ? null : new DeserializationContext(externalPointer);
    // first: checkpoint ID
    final long checkpointId = dis.readLong();
    if (checkpointId < 0) {
        throw new IOException("invalid checkpoint ID: " + checkpointId);
    }
    // second: master state
    final List<MasterState> masterStates;
    final int numMasterStates = dis.readInt();
    if (numMasterStates == 0) {
        masterStates = Collections.emptyList();
    } else if (numMasterStates > 0) {
        masterStates = new ArrayList<>(numMasterStates);
        for (int i = 0; i < numMasterStates; i++) {
            masterStates.add(deserializeMasterState(dis));
        }
    } else {
        throw new IOException("invalid number of master states: " + numMasterStates);
    }
    // third: operator states
    final int numTaskStates = dis.readInt();
    final List<OperatorState> operatorStates = new ArrayList<>(numTaskStates);
    for (int i = 0; i < numTaskStates; i++) {
        operatorStates.add(deserializeOperatorState(dis, context));
    }
    return new CheckpointMetadata(checkpointId, operatorStates, masterStates);
}

Example 9 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MetadataV2V3SerializerBase method serializeMetadata.

// ------------------------------------------------------------------------
// (De)serialization entry points
// ------------------------------------------------------------------------
protected void serializeMetadata(CheckpointMetadata checkpointMetadata, DataOutputStream dos) throws IOException {
    // first: checkpoint ID
    dos.writeLong(checkpointMetadata.getCheckpointId());
    // second: master state
    final Collection<MasterState> masterStates = checkpointMetadata.getMasterStates();
    dos.writeInt(masterStates.size());
    for (MasterState ms : masterStates) {
        serializeMasterState(ms, dos);
    }
    // third: operator states
    Collection<OperatorState> operatorStates = checkpointMetadata.getOperatorStates();
    dos.writeInt(operatorStates.size());
    for (OperatorState operatorState : operatorStates) {
        serializeOperatorState(operatorState, dos);
    }
}

Example 10 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithOnlyTaskState.

private void testCheckpointWithOnlyTaskState(String basePath) throws Exception {
    final Random rnd = new Random();
    final int maxTaskStates = 20;
    final int maxNumSubtasks = 20;
    for (int i = 0; i < 100; ++i) {
        final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
        final int numTasks = rnd.nextInt(maxTaskStates) + 1;
        final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
        final Collection<OperatorState> taskStates = CheckpointTestUtils.createOperatorStates(rnd, basePath, numTasks, 0, 0, numSubtasks);
        final Collection<MasterState> masterStates = Collections.emptyList();
        testCheckpointSerialization(checkpointId, taskStates, masterStates, basePath);
    }
}