Examples with RegisteredKeyValueStateBackendMetaInfo org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo used on opensource projects

Example 1 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class ChangelogKeyedStateBackend method createInternalState.

@Nonnull
@Override
@SuppressWarnings("unchecked")
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(@Nonnull TypeSerializer<N> namespaceSerializer, @Nonnull StateDescriptor<S, SV> stateDesc, @Nonnull StateSnapshotTransformer.StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
    StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
    if (stateFactory == null) {
        String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass());
        throw new FlinkRuntimeException(message);
    }
    RegisteredKeyValueStateBackendMetaInfo<N, SV> meta = new RegisteredKeyValueStateBackendMetaInfo<>(stateDesc.getType(), stateDesc.getName(), namespaceSerializer, stateDesc.getSerializer(), (StateSnapshotTransformer.StateSnapshotTransformFactory<SV>) snapshotTransformFactory);
    InternalKvState<K, N, SV> state = keyedStateBackend.createInternalState(namespaceSerializer, stateDesc, snapshotTransformFactory);
    KvStateChangeLoggerImpl<K, SV, N> kvStateChangeLogger = new KvStateChangeLoggerImpl<>(state.getKeySerializer(), state.getNamespaceSerializer(), state.getValueSerializer(), keyedStateBackend.getKeyContext(), stateChangelogWriter, meta, stateDesc.getTtlConfig(), stateDesc.getDefaultValue(), ++lastCreatedStateId);
    closer.register(kvStateChangeLogger);
    IS is = stateFactory.create(state, kvStateChangeLogger, keyedStateBackend);
    changelogStates.put(stateDesc.getName(), (ChangelogState) is);
    return is;
}

Example 2 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class KvStateChangeLoggerImplTest method getLogger.

@Override
protected StateChangeLogger<String, String> getLogger(TestingStateChangelogWriter writer, InternalKeyContextImpl<String> keyContext) {
    StringSerializer keySerializer = new StringSerializer();
    StringSerializer nsSerializer = new StringSerializer();
    StringSerializer valueSerializer = new StringSerializer();
    RegisteredKeyValueStateBackendMetaInfo<String, String> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(VALUE, "test", nsSerializer, valueSerializer);
    return new KvStateChangeLoggerImpl<>(keySerializer, nsSerializer, valueSerializer, keyContext, writer, metaInfo, StateTtlConfig.DISABLED, "default", Short.MIN_VALUE);
}

Example 3 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class CopyOnWriteStateTableTest method testSerializerDuplicationInSnapshot.

/**
 * This tests that serializers used for snapshots are duplicates of the ones used in processing
 * to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {
    final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();
    RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", namespaceSerializer, stateSerializer);
    InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
    CopyOnWriteStateTable<Integer, Integer, Integer> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);
    table.put(0, 0, 0, 0);
    table.put(1, 0, 0, 1);
    table.put(2, 0, 1, 2);
    final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();
    final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
    namespaceSerializer.disable();
    keySerializer.disable();
    stateSerializer.disable();
    partitionedSnapshot.writeStateInKeyGroup(new DataOutputViewStreamWrapper(new ByteArrayOutputStreamWithPos(1024)), 0);
}

Example 4 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class RocksIncrementalSnapshotStrategyTest method createSnapshotStrategy.

public RocksIncrementalSnapshotStrategy createSnapshotStrategy(CloseableRegistry closeableRegistry) throws IOException, RocksDBException {
    ColumnFamilyHandle columnFamilyHandle = rocksDBResource.createNewColumnFamily("test");
    RocksDB rocksDB = rocksDBResource.getRocksDB();
    byte[] key = "checkpoint".getBytes();
    byte[] val = "incrementalTest".getBytes();
    rocksDB.put(columnFamilyHandle, key, val);
    // construct RocksIncrementalSnapshotStrategy
    long lastCompletedCheckpointId = -1L;
    ResourceGuard rocksDBResourceGuard = new ResourceGuard();
    SortedMap<Long, Map<StateHandleID, StreamStateHandle>> materializedSstFiles = new TreeMap<>();
    LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation = new LinkedHashMap<>();
    RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(RocksDBOptions.CHECKPOINT_TRANSFER_THREAD_NUM.defaultValue());
    int keyGroupPrefixBytes = CompositeKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(2);
    RegisteredKeyValueStateBackendMetaInfo<Integer, ArrayList<Integer>> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", IntSerializer.INSTANCE, new ArrayListSerializer<>(IntSerializer.INSTANCE));
    RocksDBKeyedStateBackend.RocksDbKvStateInfo rocksDbKvStateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(columnFamilyHandle, metaInfo);
    kvStateInformation.putIfAbsent("test", rocksDbKvStateInfo);
    return new RocksIncrementalSnapshotStrategy<>(rocksDB, rocksDBResourceGuard, IntSerializer.INSTANCE, kvStateInformation, new KeyGroupRange(0, 1), keyGroupPrefixBytes, TestLocalRecoveryConfig.disabled(), closeableRegistry, tmp.newFolder(), UUID.randomUUID(), materializedSstFiles, rocksDBStateUploader, lastCompletedCheckpointId);
}

Example 5 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by apache.

the class HeapMetaInfoRestoreOperation method createOrCheckStateForMetaInfo.

Map<Integer, StateMetaInfoSnapshot> createOrCheckStateForMetaInfo(List<StateMetaInfoSnapshot> restoredMetaInfo, Map<String, StateTable<K, ?, ?>> registeredKVStates, Map<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates) {
    final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
    for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
        final StateSnapshotRestore registeredState;
        switch(metaInfoSnapshot.getBackendStateType()) {
            case KEY_VALUE:
                registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
                    registeredKVStates.put(metaInfoSnapshot.getName(), stateTableFactory.newStateTable(keyContext, registeredKeyedBackendStateMetaInfo, keySerializerProvider.currentSchemaSerializer()));
                }
                break;
            case PRIORITY_QUEUE:
                registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
                if (registeredState == null) {
                    registeredPQStates.put(metaInfoSnapshot.getName(), createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot)));
                }
                break;
            default:
                throw new IllegalStateException("Unexpected state type: " + metaInfoSnapshot.getBackendStateType() + ".");
        }
        // always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle
        // related
        kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
    }
    return kvStatesById;
}