Examples with StateMigrationException org.apache.flink.util.StateMigrationException used on opensource projects

Example 1 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class RocksDBMapState method migrateSerializedValue.

@Override
public void migrateSerializedValue(DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<Map<UK, UV>> priorSerializer, TypeSerializer<Map<UK, UV>> newSerializer) throws StateMigrationException {
    checkArgument(priorSerializer instanceof MapSerializer);
    checkArgument(newSerializer instanceof MapSerializer);
    TypeSerializer<UV> priorMapValueSerializer = ((MapSerializer<UK, UV>) priorSerializer).getValueSerializer();
    TypeSerializer<UV> newMapValueSerializer = ((MapSerializer<UK, UV>) newSerializer).getValueSerializer();
    try {
        boolean isNull = serializedOldValueInput.readBoolean();
        UV mapUserValue = null;
        if (!isNull) {
            mapUserValue = priorMapValueSerializer.deserialize(serializedOldValueInput);
        }
        serializedMigratedValueOutput.writeBoolean(mapUserValue == null);
        newMapValueSerializer.serialize(mapUserValue, serializedMigratedValueOutput);
    } catch (Exception e) {
        throw new StateMigrationException("Error while trying to migrate RocksDB map state.", e);
    }
}

Example 2 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class RocksDBListState method migrateSerializedValue.

@Override
public void migrateSerializedValue(DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<List<V>> priorSerializer, TypeSerializer<List<V>> newSerializer) throws StateMigrationException {
    Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
    Preconditions.checkArgument(newSerializer instanceof ListSerializer);
    TypeSerializer<V> priorElementSerializer = ((ListSerializer<V>) priorSerializer).getElementSerializer();
    TypeSerializer<V> newElementSerializer = ((ListSerializer<V>) newSerializer).getElementSerializer();
    try {
        while (serializedOldValueInput.available() > 0) {
            V element = ListDelimitedSerializer.deserializeNextElement(serializedOldValueInput, priorElementSerializer);
            newElementSerializer.serialize(element, serializedMigratedValueOutput);
            if (serializedOldValueInput.available() > 0) {
                serializedMigratedValueOutput.write(DELIMITER);
            }
        }
    } catch (Exception e) {
        throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
    }
}

Example 3 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class StateBackendTestBase method testListStateRestoreWithWrongSerializers.

@Test
@SuppressWarnings("unchecked")
public void testListStateRestoreWithWrongSerializers() throws Exception {
    assumeTrue(supportsMetaInfoVerification());
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);
        ListState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        state.add("1");
        backend.setCurrentKey(2);
        state.add("2");
        // draw a snapshot
        KeyedStateHandle snapshot1 = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        @SuppressWarnings("unchecked") TypeSerializer<String> fakeStringSerializer = (TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;
        try {
            kvId = new ListStateDescriptor<>("id", fakeStringSerializer);
            state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
            state.get();
            fail("should recognize wrong serializers");
        } catch (StateMigrationException ignored) {
        // expected
        }
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 4 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class HeapRestoreOperation method restore.

@Override
public Void restore() throws Exception {
    registeredKVStates.clear();
    registeredPQStates.clear();
    boolean keySerializerRestored = false;
    for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
        if (keyedStateHandle == null) {
            continue;
        }
        if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
            throw unexpectedStateHandleException(KeyGroupsStateHandle.class, keyedStateHandle.getClass());
        }
        LOG.info("Starting to restore from state handle: {}.", keyedStateHandle);
        KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
        FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
        cancelStreamRegistry.registerCloseable(fsDataInputStream);
        try {
            DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);
            KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(userCodeClassLoader);
            serializationProxy.read(inView);
            if (!keySerializerRestored) {
                // fetch current serializer now because if it is incompatible, we can't access
                // it anymore to improve the error message
                TypeSerializer<K> currentSerializer = keySerializerProvider.currentSchemaSerializer();
                // check for key serializer compatibility; this also reconfigures the
                // key serializer to be compatible, if it is required and is possible
                TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat = keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
                if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
                    throw new StateMigrationException("The new key serializer (" + currentSerializer + ") must be compatible with the previous key serializer (" + keySerializerProvider.previousSchemaSerializer() + ").");
                }
                keySerializerRestored = true;
            }
            List<StateMetaInfoSnapshot> restoredMetaInfos = serializationProxy.getStateMetaInfoSnapshots();
            final Map<Integer, StateMetaInfoSnapshot> kvStatesById = this.heapMetaInfoRestoreOperation.createOrCheckStateForMetaInfo(restoredMetaInfos, registeredKVStates, registeredPQStates);
            readStateHandleStateData(fsDataInputStream, inView, keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(), serializationProxy.isUsingKeyGroupCompression());
            LOG.info("Finished restoring from state handle: {}.", keyedStateHandle);
        } finally {
            if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
                IOUtils.closeQuietly(fsDataInputStream);
            }
        }
    }
    return null;
}

Example 5 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class RocksDBKeyedStateBackend method updateRestoredStateMetaInfo.

private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer) throws Exception {
    RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;
    // fetch current serializer now because if it is incompatible, we can't access
    // it anymore to improve the error message
    TypeSerializer<N> previousNamespaceSerializer = restoredKvStateMetaInfo.getNamespaceSerializer();
    TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
    if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
        throw new StateMigrationException("The new namespace serializer (" + namespaceSerializer + ") must be compatible with the old namespace serializer (" + previousNamespaceSerializer + ").");
    }
    restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);
    // fetch current serializer now because if it is incompatible, we can't access
    // it anymore to improve the error message
    TypeSerializer<SV> previousStateSerializer = restoredKvStateMetaInfo.getStateSerializer();
    TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
    if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
        migrateStateValues(stateDesc, oldStateInfo);
    } else if (newStateSerializerCompatibility.isIncompatible()) {
        throw new StateMigrationException("The new state serializer (" + stateSerializer + ") must not be incompatible with the old state serializer (" + previousStateSerializer + ").");
    }
    return restoredKvStateMetaInfo;
}