Examples with StreamStateHandle org.apache.flink.runtime.state.StreamStateHandle used on opensource projects

Example 56 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class Checkpoints method loadAndValidateCheckpoint.

public static CompletedCheckpoint loadAndValidateCheckpoint(JobID jobId, Map<JobVertexID, ExecutionJobVertex> tasks, CompletedCheckpointStorageLocation location, ClassLoader classLoader, boolean allowNonRestoredState, CheckpointProperties checkpointProperties, RestoreMode restoreMode) throws IOException {
    checkNotNull(jobId, "jobId");
    checkNotNull(tasks, "tasks");
    checkNotNull(location, "location");
    checkNotNull(classLoader, "classLoader");
    final StreamStateHandle metadataHandle = location.getMetadataHandle();
    final String checkpointPointer = location.getExternalPointer();
    // (1) load the savepoint
    final CheckpointMetadata checkpointMetadata;
    try (InputStream in = metadataHandle.openInputStream()) {
        DataInputStream dis = new DataInputStream(in);
        checkpointMetadata = loadCheckpointMetadata(dis, classLoader, checkpointPointer);
    }
    // generate mapping from operator to task
    Map<OperatorID, ExecutionJobVertex> operatorToJobVertexMapping = new HashMap<>();
    for (ExecutionJobVertex task : tasks.values()) {
        for (OperatorIDPair operatorIDPair : task.getOperatorIDs()) {
            operatorToJobVertexMapping.put(operatorIDPair.getGeneratedOperatorID(), task);
            operatorIDPair.getUserDefinedOperatorID().ifPresent(id -> operatorToJobVertexMapping.put(id, task));
        }
    }
    // (2) validate it (parallelism, etc)
    HashMap<OperatorID, OperatorState> operatorStates = new HashMap<>(checkpointMetadata.getOperatorStates().size());
    for (OperatorState operatorState : checkpointMetadata.getOperatorStates()) {
        ExecutionJobVertex executionJobVertex = operatorToJobVertexMapping.get(operatorState.getOperatorID());
        if (executionJobVertex != null) {
            if (executionJobVertex.getMaxParallelism() == operatorState.getMaxParallelism() || executionJobVertex.canRescaleMaxParallelism(operatorState.getMaxParallelism())) {
                operatorStates.put(operatorState.getOperatorID(), operatorState);
            } else {
                String msg = String.format("Failed to rollback to checkpoint/savepoint %s. " + "Max parallelism mismatch between checkpoint/savepoint state and new program. " + "Cannot map operator %s with max parallelism %d to new program with " + "max parallelism %d. This indicates that the program has been changed " + "in a non-compatible way after the checkpoint/savepoint.", checkpointMetadata, operatorState.getOperatorID(), operatorState.getMaxParallelism(), executionJobVertex.getMaxParallelism());
                throw new IllegalStateException(msg);
            }
        } else if (allowNonRestoredState) {
            LOG.info("Skipping savepoint state for operator {}.", operatorState.getOperatorID());
        } else {
            if (operatorState.getCoordinatorState() != null) {
                throwNonRestoredStateException(checkpointPointer, operatorState.getOperatorID());
            }
            for (OperatorSubtaskState operatorSubtaskState : operatorState.getStates()) {
                if (operatorSubtaskState.hasState()) {
                    throwNonRestoredStateException(checkpointPointer, operatorState.getOperatorID());
                }
            }
            LOG.info("Skipping empty savepoint state for operator {}.", operatorState.getOperatorID());
        }
    }
    return new CompletedCheckpoint(jobId, checkpointMetadata.getCheckpointId(), 0L, 0L, operatorStates, checkpointMetadata.getMasterStates(), checkpointProperties, restoreMode == RestoreMode.CLAIM ? new ClaimModeCompletedStorageLocation(location) : location);
}

Example 57 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class Checkpoints method disposeSavepoint.

// ------------------------------------------------------------------------
// Savepoint Disposal Hooks
// ------------------------------------------------------------------------
public static void disposeSavepoint(String pointer, CheckpointStorage checkpointStorage, ClassLoader classLoader) throws IOException, FlinkException {
    checkNotNull(pointer, "location");
    checkNotNull(checkpointStorage, "stateBackend");
    checkNotNull(classLoader, "classLoader");
    final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(pointer);
    final StreamStateHandle metadataHandle = checkpointLocation.getMetadataHandle();
    // load the savepoint object (the metadata) to have all the state handles that we need
    // to dispose of all state
    final CheckpointMetadata metadata;
    try (InputStream in = metadataHandle.openInputStream();
        DataInputStream dis = new DataInputStream(in)) {
        metadata = loadCheckpointMetadata(dis, classLoader, pointer);
    }
    Exception exception = null;
    // addressable any more even if the following disposal fails
    try {
        metadataHandle.discardState();
    } catch (Exception e) {
        exception = e;
    }
    // now dispose the savepoint data
    try {
        metadata.dispose();
    } catch (Exception e) {
        exception = ExceptionUtils.firstOrSuppressed(e, exception);
    }
    // now dispose the location (directory, table, whatever)
    try {
        checkpointLocation.disposeStorageLocation();
    } catch (Exception e) {
        exception = ExceptionUtils.firstOrSuppressed(e, exception);
    }
    // forward exceptions caught in the process
    if (exception != null) {
        ExceptionUtils.rethrowIOException(exception);
    }
}

Example 58 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class BlockingCheckpointOutputStream method closeAndGetHandle.

@Nullable
@Override
public StreamStateHandle closeAndGetHandle() throws IOException {
    if (!closed.compareAndSet(false, true)) {
        throw new IOException("Stream was already closed!");
    }
    if (delegate instanceof CheckpointStateOutputStream) {
        StreamStateHandle streamStateHandle = ((CheckpointStateOutputStream) delegate).closeAndGetHandle();
        unblockAll();
        return streamStateHandle;
    } else {
        unblockAll();
        throw new IOException("Delegate is not a CheckpointStateOutputStream!");
    }
}

Example 59 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeStreamStateHandle.

@Nullable
static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
    final int type = dis.read();
    if (NULL_HANDLE == type) {
        return null;
    } else if (FILE_STREAM_STATE_HANDLE == type) {
        long size = dis.readLong();
        String pathString = dis.readUTF();
        return new FileStateHandle(new Path(pathString), size);
    } else if (BYTE_STREAM_STATE_HANDLE == type) {
        String handleName = dis.readUTF();
        int numBytes = dis.readInt();
        byte[] data = new byte[numBytes];
        dis.readFully(data);
        return new ByteStreamStateHandle(handleName, data);
    } else if (RELATIVE_STREAM_STATE_HANDLE == type) {
        if (context == null) {
            throw new IOException("Cannot deserialize a RelativeFileStateHandle without a context to make it relative to.");
        }
        String relativePath = dis.readUTF();
        long size = dis.readLong();
        Path statePath = new Path(context.getExclusiveDirPath(), relativePath);
        return new RelativeFileStateHandle(statePath, relativePath, size);
    } else if (KEY_GROUPS_HANDLE == type) {
        int startKeyGroup = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
        long[] offsets = new long[numKeyGroups];
        for (int i = 0; i < numKeyGroups; ++i) {
            offsets[i] = dis.readLong();
        }
        KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, offsets);
        StreamStateHandle stateHandle = deserializeStreamStateHandle(dis, context);
        return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
    } else {
        throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
    }
}

Example 60 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeIncrementalStateHandle.

private static IncrementalRemoteKeyedStateHandle deserializeIncrementalStateHandle(DataInputStream dis, @Nullable DeserializationContext context, int stateHandleType) throws IOException {
    boolean isV2Format = INCREMENTAL_KEY_GROUPS_HANDLE_V2 == stateHandleType;
    long checkpointId = dis.readLong();
    String backendId = dis.readUTF();
    int startKeyGroup = dis.readInt();
    int numKeyGroups = dis.readInt();
    long checkpointedSize = isV2Format ? dis.readLong() : UNKNOWN_CHECKPOINTED_SIZE;
    KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
    StreamStateHandle metaDataStateHandle = deserializeStreamStateHandle(dis, context);
    Map<StateHandleID, StreamStateHandle> sharedStates = deserializeStreamStateHandleMap(dis, context);
    Map<StateHandleID, StreamStateHandle> privateStates = deserializeStreamStateHandleMap(dis, context);
    UUID uuid;
    try {
        uuid = UUID.fromString(backendId);
    } catch (Exception ex) {
        // compatibility with old format pre FLINK-6964:
        uuid = UUID.nameUUIDFromBytes(backendId.getBytes(StandardCharsets.UTF_8));
    }
    StateHandleID stateHandleId = isV2Format ? new StateHandleID(dis.readUTF()) : StateHandleID.randomStateHandleId();
    return IncrementalRemoteKeyedStateHandle.restore(uuid, keyGroupRange, checkpointId, sharedStates, privateStates, metaDataStateHandle, checkpointedSize, stateHandleId);
}