Examples with RegisteredPriorityQueueStateBackendMetaInfo org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo used on opensource projects

Example 1 with RegisteredPriorityQueueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo in project flink by apache.

the class ChangelogBackendLogApplier method restorePqMetaData.

private static RegisteredPriorityQueueStateBackendMetaInfo restorePqMetaData(ChangelogKeyedStateBackend<?> backend, StateMetaInfoSnapshot snapshot) {
    RegisteredPriorityQueueStateBackendMetaInfo meta = new RegisteredPriorityQueueStateBackendMetaInfo(snapshot);
    backend.create(meta.getName(), meta.getElementSerializer());
    return meta;
}

Example 2 with RegisteredPriorityQueueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo in project flink by apache.

the class RocksDBHeapTimersFullRestoreOperation method applyRestoreResult.

private void applyRestoreResult(SavepointRestoreResult savepointRestoreResult) throws IOException, RocksDBException, StateMigrationException {
    List<StateMetaInfoSnapshot> restoredMetaInfos = savepointRestoreResult.getStateMetaInfoSnapshots();
    Map<Integer, ColumnFamilyHandle> columnFamilyHandles = new HashMap<>();
    Map<Integer, HeapPriorityQueueSnapshotRestoreWrapper<?>> restoredPQStates = new HashMap<>();
    for (int i = 0; i < restoredMetaInfos.size(); i++) {
        StateMetaInfoSnapshot restoredMetaInfo = restoredMetaInfos.get(i);
        if (restoredMetaInfo.getBackendStateType() == BackendStateType.PRIORITY_QUEUE) {
            String stateName = restoredMetaInfo.getName();
            HeapPriorityQueueSnapshotRestoreWrapper<?> queueWrapper = registeredPQStates.computeIfAbsent(stateName, key -> createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(restoredMetaInfo)));
            restoredPQStates.put(i, queueWrapper);
        } else {
            RocksDbKvStateInfo registeredStateCFHandle = this.rocksHandle.getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
            columnFamilyHandles.put(i, registeredStateCFHandle.columnFamilyHandle);
        }
    }
    try (ThrowingIterator<KeyGroup> keyGroups = savepointRestoreResult.getRestoredKeyGroups()) {
        restoreKVStateData(keyGroups, columnFamilyHandles, restoredPQStates);
    }
}

Example 3 with RegisteredPriorityQueueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo in project flink by apache.

the class HeapMetaInfoRestoreOperation method createOrCheckStateForMetaInfo.

Map<Integer, StateMetaInfoSnapshot> createOrCheckStateForMetaInfo(List<StateMetaInfoSnapshot> restoredMetaInfo, Map<String, StateTable<K, ?, ?>> registeredKVStates, Map<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates) {
    final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
    for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
        final StateSnapshotRestore registeredState;
        switch(metaInfoSnapshot.getBackendStateType()) {
            case KEY_VALUE:
                registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
                    registeredKVStates.put(metaInfoSnapshot.getName(), stateTableFactory.newStateTable(keyContext, registeredKeyedBackendStateMetaInfo, keySerializerProvider.currentSchemaSerializer()));
                }
                break;
            case PRIORITY_QUEUE:
                registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    registeredPQStates.put(metaInfoSnapshot.getName(), createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot)));
                }
                break;
            default:
                throw new IllegalStateException("Unexpected state type: " + metaInfoSnapshot.getBackendStateType() + ".");
        }
        // always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle
        // related
        kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
    }
    return kvStatesById;
}

Example 4 with RegisteredPriorityQueueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo in project flink by apache.

the class RocksDBPriorityQueueSetFactory method tryRegisterPriorityQueueMetaInfo.

@Nonnull
private <T> RocksDBKeyedStateBackend.RocksDbKvStateInfo tryRegisterPriorityQueueMetaInfo(@Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
    RocksDBKeyedStateBackend.RocksDbKvStateInfo stateInfo = kvStateInformation.get(stateName);
    if (stateInfo == null) {
        // Currently this class is for timer service and TTL feature is not applicable here,
        // so no need to register compact filter when creating column family
        RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo = new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
        stateInfo = RocksDBOperationUtils.createStateInfo(metaInfo, db, columnFamilyOptionsFactory, null, writeBufferManagerCapacity);
        RocksDBOperationUtils.registerKvStateInformation(kvStateInformation, nativeMetricMonitor, stateName, stateInfo);
    } else {
        // TODO we implement the simple way of supporting the current functionality, mimicking
        // keyed state
        // because this should be reworked in FLINK-9376 and then we should have a common
        // algorithm over
        // StateMetaInfoSnapshot that avoids this code duplication.
        @SuppressWarnings("unchecked") RegisteredPriorityQueueStateBackendMetaInfo<T> castedMetaInfo = (RegisteredPriorityQueueStateBackendMetaInfo<T>) stateInfo.metaInfo;
        TypeSerializer<T> previousElementSerializer = castedMetaInfo.getPreviousElementSerializer();
        if (previousElementSerializer != byteOrderedElementSerializer) {
            TypeSerializerSchemaCompatibility<T> compatibilityResult = castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer);
            // migrating them. Therefore, here we only check for incompatibility.
            if (compatibilityResult.isIncompatible()) {
                throw new FlinkRuntimeException(new StateMigrationException("The new priority queue serializer must not be incompatible."));
            }
            // update meta info with new serializer
            stateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(stateInfo.columnFamilyHandle, new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer));
            kvStateInformation.put(stateName, stateInfo);
        }
    }
    return stateInfo;
}

Example 5 with RegisteredPriorityQueueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo in project flink by apache.

the class PriorityQueueStateChangeLoggerImplTest method getLogger.

@Override
protected StateChangeLogger<String, Void> getLogger(TestingStateChangelogWriter writer, InternalKeyContextImpl<String> keyContext) {
    StringSerializer valueSerializer = new StringSerializer();
    RegisteredPriorityQueueStateBackendMetaInfo<String> metaInfo = new RegisteredPriorityQueueStateBackendMetaInfo<>("test", valueSerializer);
    return new PriorityQueueStateChangeLoggerImpl<>(valueSerializer, keyContext, writer, metaInfo, Short.MIN_VALUE);
}