use of org.apache.flink.api.common.typeutils.base.MapSerializerSnapshot in project flink by apache.
the class RocksDBKeyedStateBackend method migrateStateValues.
/**
* Migrate only the state value, that is the "value" that is stored in RocksDB. We don't migrate
* the key here, which is made up of key group, key, namespace and map key (in case of
* MapState).
*/
@SuppressWarnings("unchecked")
private <N, S extends State, SV> void migrateStateValues(StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> stateMetaInfo) throws Exception {
if (stateDesc.getType() == StateDescriptor.Type.MAP) {
TypeSerializerSnapshot<SV> previousSerializerSnapshot = stateMetaInfo.f1.getPreviousStateSerializerSnapshot();
checkState(previousSerializerSnapshot != null, "the previous serializer snapshot should exist.");
checkState(previousSerializerSnapshot instanceof MapSerializerSnapshot, "previous serializer snapshot should be a MapSerializerSnapshot.");
TypeSerializer<SV> newSerializer = stateMetaInfo.f1.getStateSerializer();
checkState(newSerializer instanceof MapSerializer, "new serializer should be a MapSerializer.");
MapSerializer<?, ?> mapSerializer = (MapSerializer<?, ?>) newSerializer;
MapSerializerSnapshot<?, ?> mapSerializerSnapshot = (MapSerializerSnapshot<?, ?>) previousSerializerSnapshot;
if (!checkMapStateKeySchemaCompatibility(mapSerializerSnapshot, mapSerializer)) {
throw new StateMigrationException("The new serializer for a MapState requires state migration in order for the job to proceed, since the key schema has changed. However, migration for MapState currently only allows value schema evolutions.");
}
}
LOG.info("Performing state migration for state {} because the state serializer's schema, i.e. serialization format, has changed.", stateDesc);
// we need to get an actual state instance because migration is different
// for different state types. For example, ListState needs to deal with
// individual elements
StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
if (stateFactory == null) {
String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass());
throw new FlinkRuntimeException(message);
}
State state = stateFactory.createState(stateDesc, stateMetaInfo, RocksDBKeyedStateBackend.this);
if (!(state instanceof AbstractRocksDBState)) {
throw new FlinkRuntimeException("State should be an AbstractRocksDBState but is " + state);
}
@SuppressWarnings("unchecked") AbstractRocksDBState<?, ?, SV> rocksDBState = (AbstractRocksDBState<?, ?, SV>) state;
Snapshot rocksDBSnapshot = db.getSnapshot();
try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, stateMetaInfo.f0, readOptions);
RocksDBWriteBatchWrapper batchWriter = new RocksDBWriteBatchWrapper(db, getWriteOptions(), getWriteBatchSize())) {
iterator.seekToFirst();
DataInputDeserializer serializedValueInput = new DataInputDeserializer();
DataOutputSerializer migratedSerializedValueOutput = new DataOutputSerializer(512);
while (iterator.isValid()) {
serializedValueInput.setBuffer(iterator.value());
rocksDBState.migrateSerializedValue(serializedValueInput, migratedSerializedValueOutput, stateMetaInfo.f1.getPreviousStateSerializer(), stateMetaInfo.f1.getStateSerializer());
batchWriter.put(stateMetaInfo.f0, iterator.key(), migratedSerializedValueOutput.getCopyOfBuffer());
migratedSerializedValueOutput.clear();
iterator.next();
}
} finally {
db.releaseSnapshot(rocksDBSnapshot);
rocksDBSnapshot.close();
}
}
Aggregations