Examples with RegisteredKeyValueStateBackendMetaInfo org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo used on opensource projects

Example 16 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class CopyOnWriteStateTableTest method createStateTableForSnapshotRelease.

private CopyOnWriteStateTable<Integer, Integer, Float> createStateTableForSnapshotRelease(int numberOfKeyGroups) {
    RegisteredKeyValueStateBackendMetaInfo<Integer, Float> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", IntSerializer.INSTANCE, FloatSerializer.INSTANCE);
    MockInternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>(0, numberOfKeyGroups - 1, numberOfKeyGroups);
    CopyOnWriteStateTable<Integer, Integer, Float> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, IntSerializer.INSTANCE);
    ThreadLocalRandom random = ThreadLocalRandom.current();
    for (int i = 0; i < 1000; i++) {
        mockKeyContext.setCurrentKeyAndKeyGroup(i);
        table.put(random.nextInt(), random.nextFloat());
    }
    return table;
}

Example 17 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class RocksDBKeyedStateBackend method getKeys.

@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
    RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
    if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) {
        return Stream.empty();
    }
    RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo = (RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo;
    final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer();
    final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8);
    boolean ambiguousKeyPossible = CompositeKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer);
    final byte[] nameSpaceBytes;
    try {
        CompositeKeySerializationUtils.writeNameSpace(namespace, namespaceSerializer, namespaceOutputView, ambiguousKeyPossible);
        nameSpaceBytes = namespaceOutputView.getCopyOfBuffer();
    } catch (IOException ex) {
        throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
    }
    RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle, readOptions);
    iterator.seekToFirst();
    final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes, ambiguousKeyPossible, nameSpaceBytes);
    Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false);
    return targetStream.onClose(iteratorWrapper::close);
}

Example 18 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class ChangelogBackendLogApplier method restoreKvMetaData.

private static RegisteredKeyValueStateBackendMetaInfo restoreKvMetaData(ChangelogKeyedStateBackend<?> backend, StateMetaInfoSnapshot snapshot, DataInputView in) throws Exception {
    RegisteredKeyValueStateBackendMetaInfo meta = new RegisteredKeyValueStateBackendMetaInfo(snapshot);
    StateTtlConfig ttlConfig = readTtlConfig(in);
    Object defaultValue = readDefaultValue(in, meta);
    // Use regular API to create states in both changelog and the base backends the metadata is
    // persisted in log before data changes.
    // An alternative solution to load metadata "natively" by the base backends would require
    // base state to be always present, i.e. the 1st checkpoint would have to be "full" always.
    StateDescriptor stateDescriptor = toStateDescriptor(meta, defaultValue);
    // todo: support changing ttl (FLINK-23143)
    if (ttlConfig.isEnabled()) {
        stateDescriptor.enableTimeToLive(ttlConfig);
    }
    backend.getOrCreateKeyedState(meta.getNamespaceSerializer(), stateDescriptor);
    return meta;
}

Example 19 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink-mirror by flink-ci.

the class HeapMetaInfoRestoreOperation method createOrCheckStateForMetaInfo.

Map<Integer, StateMetaInfoSnapshot> createOrCheckStateForMetaInfo(List<StateMetaInfoSnapshot> restoredMetaInfo, Map<String, StateTable<K, ?, ?>> registeredKVStates, Map<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates) {
    final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
    for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
        final StateSnapshotRestore registeredState;
        switch(metaInfoSnapshot.getBackendStateType()) {
            case KEY_VALUE:
                registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
                    registeredKVStates.put(metaInfoSnapshot.getName(), stateTableFactory.newStateTable(keyContext, registeredKeyedBackendStateMetaInfo, keySerializerProvider.currentSchemaSerializer()));
                }
                break;
            case PRIORITY_QUEUE:
                registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    registeredPQStates.put(metaInfoSnapshot.getName(), createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot)));
                }
                break;
            default:
                throw new IllegalStateException("Unexpected state type: " + metaInfoSnapshot.getBackendStateType() + ".");
        }
        // always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle
        // related
        kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
    }
    return kvStatesById;
}

Example 20 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink-mirror by flink-ci.

the class CopyOnWriteStateTableTest method testSerializerDuplicationInSnapshot.

/**
 * This tests that serializers used for snapshots are duplicates of the ones used in processing
 * to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {
    final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();
    RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", namespaceSerializer, stateSerializer);
    InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
    CopyOnWriteStateTable<Integer, Integer, Integer> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);
    table.put(0, 0, 0, 0);
    table.put(1, 0, 0, 1);
    table.put(2, 0, 1, 2);
    final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();
    final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
    namespaceSerializer.disable();
    keySerializer.disable();
    stateSerializer.disable();
    partitionedSnapshot.writeStateInKeyGroup(new DataOutputViewStreamWrapper(new ByteArrayOutputStreamWithPos(1024)), 0);
}