Examples with StateMigrationException org.apache.flink.util.StateMigrationException used on opensource projects

Example 6 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class RocksDBKeyedStateBackend method migrateStateValues.

/**
 * Migrate only the state value, that is the "value" that is stored in RocksDB. We don't migrate
 * the key here, which is made up of key group, key, namespace and map key (in case of
 * MapState).
 */
@SuppressWarnings("unchecked")
private <N, S extends State, SV> void migrateStateValues(StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> stateMetaInfo) throws Exception {
    if (stateDesc.getType() == StateDescriptor.Type.MAP) {
        TypeSerializerSnapshot<SV> previousSerializerSnapshot = stateMetaInfo.f1.getPreviousStateSerializerSnapshot();
        checkState(previousSerializerSnapshot != null, "the previous serializer snapshot should exist.");
        checkState(previousSerializerSnapshot instanceof MapSerializerSnapshot, "previous serializer snapshot should be a MapSerializerSnapshot.");
        TypeSerializer<SV> newSerializer = stateMetaInfo.f1.getStateSerializer();
        checkState(newSerializer instanceof MapSerializer, "new serializer should be a MapSerializer.");
        MapSerializer<?, ?> mapSerializer = (MapSerializer<?, ?>) newSerializer;
        MapSerializerSnapshot<?, ?> mapSerializerSnapshot = (MapSerializerSnapshot<?, ?>) previousSerializerSnapshot;
        if (!checkMapStateKeySchemaCompatibility(mapSerializerSnapshot, mapSerializer)) {
            throw new StateMigrationException("The new serializer for a MapState requires state migration in order for the job to proceed, since the key schema has changed. However, migration for MapState currently only allows value schema evolutions.");
        }
    }
    LOG.info("Performing state migration for state {} because the state serializer's schema, i.e. serialization format, has changed.", stateDesc);
    // we need to get an actual state instance because migration is different
    // for different state types. For example, ListState needs to deal with
    // individual elements
    StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
    if (stateFactory == null) {
        String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass());
        throw new FlinkRuntimeException(message);
    }
    State state = stateFactory.createState(stateDesc, stateMetaInfo, RocksDBKeyedStateBackend.this);
    if (!(state instanceof AbstractRocksDBState)) {
        throw new FlinkRuntimeException("State should be an AbstractRocksDBState but is " + state);
    }
    @SuppressWarnings("unchecked") AbstractRocksDBState<?, ?, SV> rocksDBState = (AbstractRocksDBState<?, ?, SV>) state;
    Snapshot rocksDBSnapshot = db.getSnapshot();
    try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, stateMetaInfo.f0, readOptions);
        RocksDBWriteBatchWrapper batchWriter = new RocksDBWriteBatchWrapper(db, getWriteOptions(), getWriteBatchSize())) {
        iterator.seekToFirst();
        DataInputDeserializer serializedValueInput = new DataInputDeserializer();
        DataOutputSerializer migratedSerializedValueOutput = new DataOutputSerializer(512);
        while (iterator.isValid()) {
            serializedValueInput.setBuffer(iterator.value());
            rocksDBState.migrateSerializedValue(serializedValueInput, migratedSerializedValueOutput, stateMetaInfo.f1.getPreviousStateSerializer(), stateMetaInfo.f1.getStateSerializer());
            batchWriter.put(stateMetaInfo.f0, iterator.key(), migratedSerializedValueOutput.getCopyOfBuffer());
            migratedSerializedValueOutput.clear();
            iterator.next();
        }
    } finally {
        db.releaseSnapshot(rocksDBSnapshot);
        rocksDBSnapshot.close();
    }
}

Example 7 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class AbstractRocksDBState method migrateSerializedValue.

public void migrateSerializedValue(DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<V> priorSerializer, TypeSerializer<V> newSerializer) throws StateMigrationException {
    try {
        V value = priorSerializer.deserialize(serializedOldValueInput);
        newSerializer.serialize(value, serializedMigratedValueOutput);
    } catch (Exception e) {
        throw new StateMigrationException("Error while trying to migrate RocksDB state.", e);
    }
}

Example 8 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class RocksDBPriorityQueueSetFactory method tryRegisterPriorityQueueMetaInfo.

@Nonnull
private <T> RocksDBKeyedStateBackend.RocksDbKvStateInfo tryRegisterPriorityQueueMetaInfo(@Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
    RocksDBKeyedStateBackend.RocksDbKvStateInfo stateInfo = kvStateInformation.get(stateName);
    if (stateInfo == null) {
        // Currently this class is for timer service and TTL feature is not applicable here,
        // so no need to register compact filter when creating column family
        RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo = new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
        stateInfo = RocksDBOperationUtils.createStateInfo(metaInfo, db, columnFamilyOptionsFactory, null, writeBufferManagerCapacity);
        RocksDBOperationUtils.registerKvStateInformation(kvStateInformation, nativeMetricMonitor, stateName, stateInfo);
    } else {
        // TODO we implement the simple way of supporting the current functionality, mimicking
        // keyed state
        // because this should be reworked in FLINK-9376 and then we should have a common
        // algorithm over
        // StateMetaInfoSnapshot that avoids this code duplication.
        @SuppressWarnings("unchecked") RegisteredPriorityQueueStateBackendMetaInfo<T> castedMetaInfo = (RegisteredPriorityQueueStateBackendMetaInfo<T>) stateInfo.metaInfo;
        TypeSerializer<T> previousElementSerializer = castedMetaInfo.getPreviousElementSerializer();
        if (previousElementSerializer != byteOrderedElementSerializer) {
            TypeSerializerSchemaCompatibility<T> compatibilityResult = castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer);
            // migrating them. Therefore, here we only check for incompatibility.
            if (compatibilityResult.isIncompatible()) {
                throw new FlinkRuntimeException(new StateMigrationException("The new priority queue serializer must not be incompatible."));
            }
            // update meta info with new serializer
            stateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(stateInfo.columnFamilyHandle, new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer));
            kvStateInformation.put(stateName, stateInfo);
        }
    }
    return stateInfo;
}

Example 9 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class StateBackendTestBase method testReducingStateRestoreWithWrongSerializers.

@Test
@SuppressWarnings("unchecked")
public void testReducingStateRestoreWithWrongSerializers() throws Exception {
    assumeTrue(supportsMetaInfoVerification());
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), StringSerializer.INSTANCE);
        ReducingState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        state.add("1");
        backend.setCurrentKey(2);
        state.add("2");
        // draw a snapshot
        KeyedStateHandle snapshot1 = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        @SuppressWarnings("unchecked") TypeSerializer<String> fakeStringSerializer = (TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;
        try {
            kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), fakeStringSerializer);
            state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
            state.get();
            fail("should recognize wrong serializers");
        } catch (StateMigrationException ignored) {
        // expected
        }
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 10 with StateMigrationException

use of org.apache.flink.util.StateMigrationException in project flink by apache.

the class StateBackendTestBase method testMapStateRestoreWithWrongSerializers.

@Test
@SuppressWarnings("unchecked")
public void testMapStateRestoreWithWrongSerializers() throws Exception {
    assumeTrue(supportsMetaInfoVerification());
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        MapStateDescriptor<String, String> kvId = new MapStateDescriptor<>("id", StringSerializer.INSTANCE, StringSerializer.INSTANCE);
        MapState<String, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        state.put("1", "First");
        backend.setCurrentKey(2);
        state.put("2", "Second");
        // draw a snapshot
        KeyedStateHandle snapshot1 = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        @SuppressWarnings("unchecked") TypeSerializer<String> fakeStringSerializer = (TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;
        try {
            kvId = new MapStateDescriptor<>("id", fakeStringSerializer, StringSerializer.INSTANCE);
            state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
            state.entries();
            fail("should recognize wrong serializers");
        } catch (StateMigrationException ignored) {
        // expected
        }
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}