Examples with StateSnapshot org.apache.flink.runtime.state.StateSnapshot used on opensource projects

Example 1 with StateSnapshot

use of org.apache.flink.runtime.state.StateSnapshot in project flink by apache.

the class HeapSnapshotStrategy method asyncSnapshot.

@Override
public SnapshotResultSupplier<KeyedStateHandle> asyncSnapshot(HeapSnapshotResources<K> syncPartResource, long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
    List<StateMetaInfoSnapshot> metaInfoSnapshots = syncPartResource.getMetaInfoSnapshots();
    if (metaInfoSnapshots.isEmpty()) {
        return snapshotCloseableRegistry -> SnapshotResult.empty();
    }
    final KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(// get a serialized form already at state registration time in the future
    syncPartResource.getKeySerializer(), metaInfoSnapshots, !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator));
    final SupplierWithException<CheckpointStreamWithResultProvider, Exception> checkpointStreamSupplier = localRecoveryConfig.isLocalRecoveryEnabled() && !checkpointOptions.getCheckpointType().isSavepoint() ? () -> createDuplicatingStream(checkpointId, CheckpointedStateScope.EXCLUSIVE, streamFactory, localRecoveryConfig.getLocalStateDirectoryProvider().orElseThrow(LocalRecoveryConfig.localRecoveryNotEnabled())) : () -> createSimpleStream(CheckpointedStateScope.EXCLUSIVE, streamFactory);
    return (snapshotCloseableRegistry) -> {
        final Map<StateUID, Integer> stateNamesToId = syncPartResource.getStateNamesToId();
        final Map<StateUID, StateSnapshot> cowStateStableSnapshots = syncPartResource.getCowStateStableSnapshots();
        final CheckpointStreamWithResultProvider streamWithResultProvider = checkpointStreamSupplier.get();
        snapshotCloseableRegistry.registerCloseable(streamWithResultProvider);
        final CheckpointStateOutputStream localStream = streamWithResultProvider.getCheckpointOutputStream();
        final DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(localStream);
        serializationProxy.write(outView);
        final long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()];
        for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) {
            int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos);
            keyGroupRangeOffsets[keyGroupPos] = localStream.getPos();
            outView.writeInt(keyGroupId);
            for (Map.Entry<StateUID, StateSnapshot> stateSnapshot : cowStateStableSnapshots.entrySet()) {
                StateSnapshot.StateKeyGroupWriter partitionedSnapshot = stateSnapshot.getValue().getKeyGroupWriter();
                try (OutputStream kgCompressionOut = keyGroupCompressionDecorator.decorateWithCompression(localStream)) {
                    DataOutputViewStreamWrapper kgCompressionView = new DataOutputViewStreamWrapper(kgCompressionOut);
                    kgCompressionView.writeShort(stateNamesToId.get(stateSnapshot.getKey()));
                    partitionedSnapshot.writeStateInKeyGroup(kgCompressionView, keyGroupId);
                }
            // this will just close the outer compression stream
            }
        }
        if (snapshotCloseableRegistry.unregisterCloseable(streamWithResultProvider)) {
            KeyGroupRangeOffsets kgOffs = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets);
            SnapshotResult<StreamStateHandle> result = streamWithResultProvider.closeAndFinalizeCheckpointStreamResult();
            return toKeyedStateHandleSnapshotResult(result, kgOffs, KeyGroupsStateHandle::new);
        } else {
            throw new IOException("Stream already unregistered.");
        }
    };
}

Example 2 with StateSnapshot

use of org.apache.flink.runtime.state.StateSnapshot in project flink by apache.

the class HeapKeyValueStateIterator method moveToNextState.

private boolean moveToNextState() throws IOException {
    if (statesIterator.hasNext()) {
        this.currentState = statesIterator.next();
        this.newKVState = true;
    } else if (keyGroupIterator.hasNext()) {
        this.currentKeyGroup = keyGroupIterator.next();
        resetStates();
        this.newKeyGroup = true;
        this.newKVState = true;
    } else {
        return false;
    }
    StateSnapshot stateSnapshot = this.stateStableSnapshots.get(currentState);
    setCurrentStateIterator(stateSnapshot);
    // set to a valid entry
    return true;
}

Example 3 with StateSnapshot

use of org.apache.flink.runtime.state.StateSnapshot in project flink by apache.

the class HeapSnapshotResources method processSnapshotMetaInfoForAllStates.

private static void processSnapshotMetaInfoForAllStates(List<StateMetaInfoSnapshot> metaInfoSnapshots, Map<StateUID, StateSnapshot> cowStateStableSnapshots, Map<StateUID, Integer> stateNamesToId, Map<String, ? extends StateSnapshotRestore> registeredStates, StateMetaInfoSnapshot.BackendStateType stateType) {
    for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
        final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
        stateNamesToId.put(stateUid, stateNamesToId.size());
        StateSnapshotRestore state = kvState.getValue();
        if (null != state) {
            final StateSnapshot stateSnapshot = state.stateSnapshot();
            metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
            cowStateStableSnapshots.put(stateUid, stateSnapshot);
        }
    }
}