Examples with StateMetaInfoSnapshot org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot used on opensource projects

Example 11 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class HeapMetaInfoRestoreOperation method createOrCheckStateForMetaInfo.

Map<Integer, StateMetaInfoSnapshot> createOrCheckStateForMetaInfo(List<StateMetaInfoSnapshot> restoredMetaInfo, Map<String, StateTable<K, ?, ?>> registeredKVStates, Map<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates) {
    final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
    for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
        final StateSnapshotRestore registeredState;
        switch(metaInfoSnapshot.getBackendStateType()) {
            case KEY_VALUE:
                registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
                    registeredKVStates.put(metaInfoSnapshot.getName(), stateTableFactory.newStateTable(keyContext, registeredKeyedBackendStateMetaInfo, keySerializerProvider.currentSchemaSerializer()));
                }
                break;
            case PRIORITY_QUEUE:
                registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    registeredPQStates.put(metaInfoSnapshot.getName(), createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot)));
                }
                break;
            default:
                throw new IllegalStateException("Unexpected state type: " + metaInfoSnapshot.getBackendStateType() + ".");
        }
        // always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle
        // related
        kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
    }
    return kvStatesById;
}

Example 12 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class HeapRestoreOperation method restore.

@Override
public Void restore() throws Exception {
    registeredKVStates.clear();
    registeredPQStates.clear();
    boolean keySerializerRestored = false;
    for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
        if (keyedStateHandle == null) {
            continue;
        }
        if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
            throw unexpectedStateHandleException(KeyGroupsStateHandle.class, keyedStateHandle.getClass());
        }
        LOG.info("Starting to restore from state handle: {}.", keyedStateHandle);
        KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
        FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
        cancelStreamRegistry.registerCloseable(fsDataInputStream);
        try {
            DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);
            KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(userCodeClassLoader);
            serializationProxy.read(inView);
            if (!keySerializerRestored) {
                // fetch current serializer now because if it is incompatible, we can't access
                // it anymore to improve the error message
                TypeSerializer<K> currentSerializer = keySerializerProvider.currentSchemaSerializer();
                // check for key serializer compatibility; this also reconfigures the
                // key serializer to be compatible, if it is required and is possible
                TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat = keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
                if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
                    throw new StateMigrationException("The new key serializer (" + currentSerializer + ") must be compatible with the previous key serializer (" + keySerializerProvider.previousSchemaSerializer() + ").");
                }
                keySerializerRestored = true;
            }
            List<StateMetaInfoSnapshot> restoredMetaInfos = serializationProxy.getStateMetaInfoSnapshots();
            final Map<Integer, StateMetaInfoSnapshot> kvStatesById = this.heapMetaInfoRestoreOperation.createOrCheckStateForMetaInfo(restoredMetaInfos, registeredKVStates, registeredPQStates);
            readStateHandleStateData(fsDataInputStream, inView, keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(), serializationProxy.isUsingKeyGroupCompression());
            LOG.info("Finished restoring from state handle: {}.", keyedStateHandle);
        } finally {
            if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
                IOUtils.closeQuietly(fsDataInputStream);
            }
        }
    }
    return null;
}

Example 13 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class HeapRestoreOperation method readKeyGroupStateData.

private void readKeyGroupStateData(InputStream inputStream, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int keyGroupIndex, int numStates, int readVersion) throws IOException {
    DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(inputStream);
    for (int i = 0; i < numStates; i++) {
        final int kvStateId = inView.readShort();
        final StateMetaInfoSnapshot stateMetaInfoSnapshot = kvStatesById.get(kvStateId);
        final StateSnapshotRestore registeredState;
        switch(stateMetaInfoSnapshot.getBackendStateType()) {
            case KEY_VALUE:
                registeredState = registeredKVStates.get(stateMetaInfoSnapshot.getName());
                break;
            case PRIORITY_QUEUE:
                registeredState = registeredPQStates.get(stateMetaInfoSnapshot.getName());
                break;
            default:
                throw new IllegalStateException("Unexpected state type: " + stateMetaInfoSnapshot.getBackendStateType() + ".");
        }
        StateSnapshotKeyGroupReader keyGroupReader = registeredState.keyGroupReader(readVersion);
        keyGroupReader.readMappingsInKeyGroup(inView, keyGroupIndex);
    }
}

Example 14 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class HeapSavepointRestoreOperation method restore.

@Override
public Void restore() throws Exception {
    registeredKVStates.clear();
    registeredPQStates.clear();
    try (ThrowingIterator<SavepointRestoreResult> restore = this.savepointRestoreOperation.restore()) {
        while (restore.hasNext()) {
            SavepointRestoreResult restoreResult = restore.next();
            List<StateMetaInfoSnapshot> restoredMetaInfos = restoreResult.getStateMetaInfoSnapshots();
            final Map<Integer, StateMetaInfoSnapshot> kvStatesById = this.heapMetaInfoRestoreOperation.createOrCheckStateForMetaInfo(restoredMetaInfos, registeredKVStates, registeredPQStates);
            try (ThrowingIterator<KeyGroup> keyGroups = restoreResult.getRestoredKeyGroups()) {
                while (keyGroups.hasNext()) {
                    readKeyGroupStateData(keyGroups.next(), keySerializerProvider.previousSchemaSerializer(), kvStatesById);
                }
            }
        }
    }
    return null;
}

Example 15 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class RocksIncrementalSnapshotStrategy method syncPrepareResources.

@Override
public IncrementalRocksDBSnapshotResources syncPrepareResources(long checkpointId) throws Exception {
    final SnapshotDirectory snapshotDirectory = prepareLocalSnapshotDirectory(checkpointId);
    LOG.trace("Local RocksDB checkpoint goes to backup path {}.", snapshotDirectory);
    final List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = new ArrayList<>(kvStateInformation.size());
    final PreviousSnapshot previousSnapshot = snapshotMetaData(checkpointId, stateMetaInfoSnapshots);
    takeDBNativeCheckpoint(snapshotDirectory);
    return new IncrementalRocksDBSnapshotResources(snapshotDirectory, previousSnapshot, stateMetaInfoSnapshots);
}