Examples with RegisteredKeyValueStateBackendMetaInfo org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo used on opensource projects

Example 11 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class KvStateChangeLoggerImplTest method getLogger.

@Override
protected StateChangeLogger<String, String> getLogger(TestingStateChangelogWriter writer, InternalKeyContextImpl<String> keyContext) {
    StringSerializer keySerializer = new StringSerializer();
    StringSerializer nsSerializer = new StringSerializer();
    StringSerializer valueSerializer = new StringSerializer();
    RegisteredKeyValueStateBackendMetaInfo<String, String> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(VALUE, "test", nsSerializer, valueSerializer);
    return new KvStateChangeLoggerImpl<>(keySerializer, nsSerializer, valueSerializer, keyContext, writer, metaInfo, StateTtlConfig.DISABLED, "default", Short.MIN_VALUE);
}

Example 12 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class CopyOnWriteStateTableTest method createStateTableForSnapshotRelease.

private CopyOnWriteStateTable<Integer, Integer, Float> createStateTableForSnapshotRelease(int numberOfKeyGroups) {
    RegisteredKeyValueStateBackendMetaInfo<Integer, Float> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", IntSerializer.INSTANCE, FloatSerializer.INSTANCE);
    MockInternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>(0, numberOfKeyGroups - 1, numberOfKeyGroups);
    CopyOnWriteStateTable<Integer, Integer, Float> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, IntSerializer.INSTANCE);
    ThreadLocalRandom random = ThreadLocalRandom.current();
    for (int i = 0; i < 1000; i++) {
        mockKeyContext.setCurrentKeyAndKeyGroup(i);
        table.put(random.nextInt(), random.nextFloat());
    }
    return table;
}

Example 13 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class RocksDBKeyedStateBackend method getKeys.

@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
    RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
    if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) {
        return Stream.empty();
    }
    RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo = (RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo;
    final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer();
    final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8);
    boolean ambiguousKeyPossible = CompositeKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer);
    final byte[] nameSpaceBytes;
    try {
        CompositeKeySerializationUtils.writeNameSpace(namespace, namespaceSerializer, namespaceOutputView, ambiguousKeyPossible);
        nameSpaceBytes = namespaceOutputView.getCopyOfBuffer();
    } catch (IOException ex) {
        throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
    }
    RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle, readOptions);
    iterator.seekToFirst();
    final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes, ambiguousKeyPossible, nameSpaceBytes);
    Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false);
    return targetStream.onClose(iteratorWrapper::close);
}

Example 14 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class HeapMetaInfoRestoreOperation method createOrCheckStateForMetaInfo.

Map<Integer, StateMetaInfoSnapshot> createOrCheckStateForMetaInfo(List<StateMetaInfoSnapshot> restoredMetaInfo, Map<String, StateTable<K, ?, ?>> registeredKVStates, Map<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates) {
    final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
    for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
        final StateSnapshotRestore registeredState;
        switch(metaInfoSnapshot.getBackendStateType()) {
            case KEY_VALUE:
                registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
                    registeredKVStates.put(metaInfoSnapshot.getName(), stateTableFactory.newStateTable(keyContext, registeredKeyedBackendStateMetaInfo, keySerializerProvider.currentSchemaSerializer()));
                }
                break;
            case PRIORITY_QUEUE:
                registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    registeredPQStates.put(metaInfoSnapshot.getName(), createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot)));
                }
                break;
            default:
                throw new IllegalStateException("Unexpected state type: " + metaInfoSnapshot.getBackendStateType() + ".");
        }
        // always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle
        // related
        kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
    }
    return kvStatesById;
}

Example 15 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class HeapMetaInfoRestoreOperation method createOrCheckStateForMetaInfo.

Map<Integer, StateMetaInfoSnapshot> createOrCheckStateForMetaInfo(List<StateMetaInfoSnapshot> restoredMetaInfo, Map<String, StateTable<K, ?, ?>> registeredKVStates, Map<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates) {
    final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
    for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
        final StateSnapshotRestore registeredState;
        switch(metaInfoSnapshot.getBackendStateType()) {
            case KEY_VALUE:
                registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
                    registeredKVStates.put(metaInfoSnapshot.getName(), stateTableFactory.newStateTable(keyContext, registeredKeyedBackendStateMetaInfo, keySerializerProvider.currentSchemaSerializer()));
                }
                break;
            case PRIORITY_QUEUE:
                registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    registeredPQStates.put(metaInfoSnapshot.getName(), createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot)));
                }
                break;
            default:
                throw new IllegalStateException("Unexpected state type: " + metaInfoSnapshot.getBackendStateType() + ".");
        }
        // always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle
        // related
        kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
    }
    return kvStatesById;
}