Examples with RegisteredKeyValueStateBackendMetaInfo org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo used on opensource projects

Example 21 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class ChangelogBackendLogApplier method restoreKvMetaData.

private static RegisteredKeyValueStateBackendMetaInfo restoreKvMetaData(ChangelogKeyedStateBackend<?> backend, StateMetaInfoSnapshot snapshot, DataInputView in) throws Exception {
    RegisteredKeyValueStateBackendMetaInfo meta = new RegisteredKeyValueStateBackendMetaInfo(snapshot);
    StateTtlConfig ttlConfig = readTtlConfig(in);
    Object defaultValue = readDefaultValue(in, meta);
    // Use regular API to create states in both changelog and the base backends the metadata is
    // persisted in log before data changes.
    // An alternative solution to load metadata "natively" by the base backends would require
    // base state to be always present, i.e. the 1st checkpoint would have to be "full" always.
    StateDescriptor stateDescriptor = toStateDescriptor(meta, defaultValue);
    // todo: support changing ttl (FLINK-23143)
    if (ttlConfig.isEnabled()) {
        stateDescriptor.enableTimeToLive(ttlConfig);
    }
    backend.getOrCreateKeyedState(meta.getNamespaceSerializer(), stateDescriptor);
    return meta;
}

Example 22 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class ChangelogKeyedStateBackend method createInternalState.

@Nonnull
@Override
@SuppressWarnings("unchecked")
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(@Nonnull TypeSerializer<N> namespaceSerializer, @Nonnull StateDescriptor<S, SV> stateDesc, @Nonnull StateSnapshotTransformer.StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
    StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
    if (stateFactory == null) {
        String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass());
        throw new FlinkRuntimeException(message);
    }
    RegisteredKeyValueStateBackendMetaInfo<N, SV> meta = new RegisteredKeyValueStateBackendMetaInfo<>(stateDesc.getType(), stateDesc.getName(), namespaceSerializer, stateDesc.getSerializer(), (StateSnapshotTransformer.StateSnapshotTransformFactory<SV>) snapshotTransformFactory);
    InternalKvState<K, N, SV> state = keyedStateBackend.createInternalState(namespaceSerializer, stateDesc, snapshotTransformFactory);
    KvStateChangeLoggerImpl<K, SV, N> kvStateChangeLogger = new KvStateChangeLoggerImpl<>(state.getKeySerializer(), state.getNamespaceSerializer(), state.getValueSerializer(), keyedStateBackend.getKeyContext(), stateChangelogWriter, meta, stateDesc.getTtlConfig(), stateDesc.getDefaultValue(), ++lastCreatedStateId);
    closer.register(kvStateChangeLogger);
    IS is = stateFactory.create(state, kvStateChangeLogger, keyedStateBackend);
    changelogStates.put(stateDesc.getName(), (ChangelogState) is);
    return is;
}

Example 23 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class CopyOnWriteStateTableTest method testSerializerDuplicationInSnapshot.

/**
 * This tests that serializers used for snapshots are duplicates of the ones used in processing
 * to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {
    final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();
    RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", namespaceSerializer, stateSerializer);
    InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
    CopyOnWriteStateTable<Integer, Integer, Integer> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);
    table.put(0, 0, 0, 0);
    table.put(1, 0, 0, 1);
    table.put(2, 0, 1, 2);
    final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();
    final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
    namespaceSerializer.disable();
    keySerializer.disable();
    stateSerializer.disable();
    partitionedSnapshot.writeStateInKeyGroup(new DataOutputViewStreamWrapper(new ByteArrayOutputStreamWithPos(1024)), 0);
}

Example 24 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink-mirror by flink-ci.

the class ChangelogKeyedStateBackend method createInternalState.

@Nonnull
@Override
@SuppressWarnings("unchecked")
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(@Nonnull TypeSerializer<N> namespaceSerializer, @Nonnull StateDescriptor<S, SV> stateDesc, @Nonnull StateSnapshotTransformer.StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
    StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
    if (stateFactory == null) {
        String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass());
        throw new FlinkRuntimeException(message);
    }
    RegisteredKeyValueStateBackendMetaInfo<N, SV> meta = new RegisteredKeyValueStateBackendMetaInfo<>(stateDesc.getType(), stateDesc.getName(), namespaceSerializer, stateDesc.getSerializer(), (StateSnapshotTransformer.StateSnapshotTransformFactory<SV>) snapshotTransformFactory);
    InternalKvState<K, N, SV> state = keyedStateBackend.createInternalState(namespaceSerializer, stateDesc, snapshotTransformFactory);
    KvStateChangeLoggerImpl<K, SV, N> kvStateChangeLogger = new KvStateChangeLoggerImpl<>(state.getKeySerializer(), state.getNamespaceSerializer(), state.getValueSerializer(), keyedStateBackend.getKeyContext(), stateChangelogWriter, meta, stateDesc.getTtlConfig(), stateDesc.getDefaultValue(), ++lastCreatedStateId);
    closer.register(kvStateChangeLogger);
    IS is = stateFactory.create(state, kvStateChangeLogger, keyedStateBackend);
    changelogStates.put(stateDesc.getName(), (ChangelogState) is);
    return is;
}