Examples with StateMetaInfoSnapshot org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot used on opensource projects

Example 1 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class HeapSnapshotStrategy method asyncSnapshot.

@Override
public SnapshotResultSupplier<KeyedStateHandle> asyncSnapshot(HeapSnapshotResources<K> syncPartResource, long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
    List<StateMetaInfoSnapshot> metaInfoSnapshots = syncPartResource.getMetaInfoSnapshots();
    if (metaInfoSnapshots.isEmpty()) {
        return snapshotCloseableRegistry -> SnapshotResult.empty();
    }
    final KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(// get a serialized form already at state registration time in the future
    syncPartResource.getKeySerializer(), metaInfoSnapshots, !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator));
    final SupplierWithException<CheckpointStreamWithResultProvider, Exception> checkpointStreamSupplier = localRecoveryConfig.isLocalRecoveryEnabled() && !checkpointOptions.getCheckpointType().isSavepoint() ? () -> createDuplicatingStream(checkpointId, CheckpointedStateScope.EXCLUSIVE, streamFactory, localRecoveryConfig.getLocalStateDirectoryProvider().orElseThrow(LocalRecoveryConfig.localRecoveryNotEnabled())) : () -> createSimpleStream(CheckpointedStateScope.EXCLUSIVE, streamFactory);
    return (snapshotCloseableRegistry) -> {
        final Map<StateUID, Integer> stateNamesToId = syncPartResource.getStateNamesToId();
        final Map<StateUID, StateSnapshot> cowStateStableSnapshots = syncPartResource.getCowStateStableSnapshots();
        final CheckpointStreamWithResultProvider streamWithResultProvider = checkpointStreamSupplier.get();
        snapshotCloseableRegistry.registerCloseable(streamWithResultProvider);
        final CheckpointStateOutputStream localStream = streamWithResultProvider.getCheckpointOutputStream();
        final DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(localStream);
        serializationProxy.write(outView);
        final long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()];
        for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) {
            int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos);
            keyGroupRangeOffsets[keyGroupPos] = localStream.getPos();
            outView.writeInt(keyGroupId);
            for (Map.Entry<StateUID, StateSnapshot> stateSnapshot : cowStateStableSnapshots.entrySet()) {
                StateSnapshot.StateKeyGroupWriter partitionedSnapshot = stateSnapshot.getValue().getKeyGroupWriter();
                try (OutputStream kgCompressionOut = keyGroupCompressionDecorator.decorateWithCompression(localStream)) {
                    DataOutputViewStreamWrapper kgCompressionView = new DataOutputViewStreamWrapper(kgCompressionOut);
                    kgCompressionView.writeShort(stateNamesToId.get(stateSnapshot.getKey()));
                    partitionedSnapshot.writeStateInKeyGroup(kgCompressionView, keyGroupId);
                }
            // this will just close the outer compression stream
            }
        }
        if (snapshotCloseableRegistry.unregisterCloseable(streamWithResultProvider)) {
            KeyGroupRangeOffsets kgOffs = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets);
            SnapshotResult<StreamStateHandle> result = streamWithResultProvider.closeAndFinalizeCheckpointStreamResult();
            return toKeyedStateHandleSnapshotResult(result, kgOffs, KeyGroupsStateHandle::new);
        } else {
            throw new IOException("Stream already unregistered.");
        }
    };
}

Example 2 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class HeapSavepointRestoreOperation method readKeyGroupStateData.

private void readKeyGroupStateData(KeyGroup keyGroup, TypeSerializer<K> keySerializer, Map<Integer, StateMetaInfoSnapshot> kvStatesById) throws Exception {
    try (ThrowingIterator<KeyGroupEntry> entries = keyGroup.getKeyGroupEntries()) {
        while (entries.hasNext()) {
            KeyGroupEntry groupEntry = entries.next();
            StateMetaInfoSnapshot infoSnapshot = kvStatesById.get(groupEntry.getKvStateId());
            switch(infoSnapshot.getBackendStateType()) {
                case KEY_VALUE:
                    readKVStateData(keySerializer, groupEntry, infoSnapshot);
                    break;
                case PRIORITY_QUEUE:
                    readPriorityQueue(groupEntry, infoSnapshot);
                    break;
                case OPERATOR:
                case BROADCAST:
                    throw new IllegalStateException("Expected only keyed state. Received: " + infoSnapshot.getBackendStateType());
            }
        }
    }
}

Example 3 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class OperatorStateRestoreOperation method restore.

@Override
public Void restore() throws Exception {
    if (stateHandles.isEmpty()) {
        return null;
    }
    for (OperatorStateHandle stateHandle : stateHandles) {
        if (stateHandle == null) {
            continue;
        }
        FSDataInputStream in = stateHandle.openInputStream();
        closeStreamOnCancelRegistry.registerCloseable(in);
        ClassLoader restoreClassLoader = Thread.currentThread().getContextClassLoader();
        try {
            Thread.currentThread().setContextClassLoader(userClassloader);
            OperatorBackendSerializationProxy backendSerializationProxy = new OperatorBackendSerializationProxy(userClassloader);
            backendSerializationProxy.read(new DataInputViewStreamWrapper(in));
            List<StateMetaInfoSnapshot> restoredOperatorMetaInfoSnapshots = backendSerializationProxy.getOperatorStateMetaInfoSnapshots();
            // Recreate all PartitionableListStates from the meta info
            for (StateMetaInfoSnapshot restoredSnapshot : restoredOperatorMetaInfoSnapshots) {
                final RegisteredOperatorStateBackendMetaInfo<?> restoredMetaInfo = new RegisteredOperatorStateBackendMetaInfo<>(restoredSnapshot);
                if (restoredMetaInfo.getPartitionStateSerializer() instanceof UnloadableDummyTypeSerializer) {
                    throw new IOException("Unable to restore operator state [" + restoredSnapshot.getName() + "]." + " The previous typeSerializer of the operator state must be present; the typeSerializer could" + " have been removed from the classpath, or its implementation have changed and could" + " not be loaded. This is a temporary restriction that will be fixed in future versions.");
                }
                PartitionableListState<?> listState = registeredOperatorStates.get(restoredSnapshot.getName());
                if (null == listState) {
                    listState = new PartitionableListState<>(restoredMetaInfo);
                    registeredOperatorStates.put(listState.getStateMetaInfo().getName(), listState);
                } else {
                // TODO with eager state registration in place, check here for
                // typeSerializer migration strategies
                }
            }
            // ... and then get back the broadcast state.
            List<StateMetaInfoSnapshot> restoredBroadcastMetaInfoSnapshots = backendSerializationProxy.getBroadcastStateMetaInfoSnapshots();
            for (StateMetaInfoSnapshot restoredSnapshot : restoredBroadcastMetaInfoSnapshots) {
                final RegisteredBroadcastStateBackendMetaInfo<?, ?> restoredMetaInfo = new RegisteredBroadcastStateBackendMetaInfo<>(restoredSnapshot);
                if (restoredMetaInfo.getKeySerializer() instanceof UnloadableDummyTypeSerializer || restoredMetaInfo.getValueSerializer() instanceof UnloadableDummyTypeSerializer) {
                    throw new IOException("Unable to restore broadcast state [" + restoredSnapshot.getName() + "]." + " The previous key and value serializers of the state must be present; the serializers could" + " have been removed from the classpath, or their implementations have changed and could" + " not be loaded. This is a temporary restriction that will be fixed in future versions.");
                }
                BackendWritableBroadcastState<?, ?> broadcastState = registeredBroadcastStates.get(restoredSnapshot.getName());
                if (broadcastState == null) {
                    broadcastState = new HeapBroadcastState<>(restoredMetaInfo);
                    registeredBroadcastStates.put(broadcastState.getStateMetaInfo().getName(), broadcastState);
                } else {
                // TODO with eager state registration in place, check here for
                // typeSerializer migration strategies
                }
            }
            // Restore all the states
            for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> nameToOffsets : stateHandle.getStateNameToPartitionOffsets().entrySet()) {
                final String stateName = nameToOffsets.getKey();
                PartitionableListState<?> listStateForName = registeredOperatorStates.get(stateName);
                if (listStateForName == null) {
                    BackendWritableBroadcastState<?, ?> broadcastStateForName = registeredBroadcastStates.get(stateName);
                    Preconditions.checkState(broadcastStateForName != null, "Found state without " + "corresponding meta info: " + stateName);
                    deserializeBroadcastStateValues(broadcastStateForName, in, nameToOffsets.getValue());
                } else {
                    deserializeOperatorStateValues(listStateForName, in, nameToOffsets.getValue());
                }
            }
        } finally {
            Thread.currentThread().setContextClassLoader(restoreClassLoader);
            if (closeStreamOnCancelRegistry.unregisterCloseable(in)) {
                IOUtils.closeQuietly(in);
            }
        }
    }
    return null;
}

Example 4 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class DefaultOperatorStateBackendSnapshotStrategy method asyncSnapshot.

@Override
public SnapshotResultSupplier<OperatorStateHandle> asyncSnapshot(DefaultOperatorStateBackendSnapshotResources syncPartResource, long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
    Map<String, PartitionableListState<?>> registeredOperatorStatesDeepCopies = syncPartResource.getRegisteredOperatorStatesDeepCopies();
    Map<String, BackendWritableBroadcastState<?, ?>> registeredBroadcastStatesDeepCopies = syncPartResource.getRegisteredBroadcastStatesDeepCopies();
    if (registeredBroadcastStatesDeepCopies.isEmpty() && registeredOperatorStatesDeepCopies.isEmpty()) {
        return snapshotCloseableRegistry -> SnapshotResult.empty();
    }
    return (snapshotCloseableRegistry) -> {
        CheckpointStateOutputStream localOut = streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
        snapshotCloseableRegistry.registerCloseable(localOut);
        // get the registered operator state infos ...
        List<StateMetaInfoSnapshot> operatorMetaInfoSnapshots = new ArrayList<>(registeredOperatorStatesDeepCopies.size());
        for (Map.Entry<String, PartitionableListState<?>> entry : registeredOperatorStatesDeepCopies.entrySet()) {
            operatorMetaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
        }
        // ... get the registered broadcast operator state infos ...
        List<StateMetaInfoSnapshot> broadcastMetaInfoSnapshots = new ArrayList<>(registeredBroadcastStatesDeepCopies.size());
        for (Map.Entry<String, BackendWritableBroadcastState<?, ?>> entry : registeredBroadcastStatesDeepCopies.entrySet()) {
            broadcastMetaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
        }
        // ... write them all in the checkpoint stream ...
        DataOutputView dov = new DataOutputViewStreamWrapper(localOut);
        OperatorBackendSerializationProxy backendSerializationProxy = new OperatorBackendSerializationProxy(operatorMetaInfoSnapshots, broadcastMetaInfoSnapshots);
        backendSerializationProxy.write(dov);
        // ... and then go for the states ...
        // we put BOTH normal and broadcast state metadata here
        int initialMapCapacity = registeredOperatorStatesDeepCopies.size() + registeredBroadcastStatesDeepCopies.size();
        final Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData = new HashMap<>(initialMapCapacity);
        for (Map.Entry<String, PartitionableListState<?>> entry : registeredOperatorStatesDeepCopies.entrySet()) {
            PartitionableListState<?> value = entry.getValue();
            long[] partitionOffsets = value.write(localOut);
            OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
            writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
        }
        // ... and the broadcast states themselves ...
        for (Map.Entry<String, BackendWritableBroadcastState<?, ?>> entry : registeredBroadcastStatesDeepCopies.entrySet()) {
            BackendWritableBroadcastState<?, ?> value = entry.getValue();
            long[] partitionOffsets = { value.write(localOut) };
            OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
            writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
        }
        // ... and, finally, create the state handle.
        OperatorStateHandle retValue = null;
        if (snapshotCloseableRegistry.unregisterCloseable(localOut)) {
            StreamStateHandle stateHandle = localOut.closeAndGetHandle();
            if (stateHandle != null) {
                retValue = new OperatorStreamStateHandle(writtenStatesMetaData, stateHandle);
            }
            return SnapshotResult.of(retValue);
        } else {
            throw new IOException("Stream was already unregistered.");
        }
    };
}

Example 5 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class RocksDBIncrementalRestoreOperation method restoreFromLocalState.

private void restoreFromLocalState(IncrementalLocalKeyedStateHandle localKeyedStateHandle) throws Exception {
    KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(localKeyedStateHandle.getMetaDataState());
    List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
    Path restoreSourcePath = localKeyedStateHandle.getDirectoryStateHandle().getDirectory();
    logger.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.", operatorIdentifier, backendUID);
    this.rocksHandle.openDB(createColumnFamilyDescriptors(stateMetaInfoSnapshots, true), stateMetaInfoSnapshots, restoreSourcePath);
}