Examples with RegisteredKeyValueStateBackendMetaInfo org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo used on opensource projects

Example 6 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink-mirror by flink-ci.

the class KvStateChangeLoggerImplTest method getLogger.

@Override
protected StateChangeLogger<String, String> getLogger(TestingStateChangelogWriter writer, InternalKeyContextImpl<String> keyContext) {
    StringSerializer keySerializer = new StringSerializer();
    StringSerializer nsSerializer = new StringSerializer();
    StringSerializer valueSerializer = new StringSerializer();
    RegisteredKeyValueStateBackendMetaInfo<String, String> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(VALUE, "test", nsSerializer, valueSerializer);
    return new KvStateChangeLoggerImpl<>(keySerializer, nsSerializer, valueSerializer, keyContext, writer, metaInfo, StateTtlConfig.DISABLED, "default", Short.MIN_VALUE);
}

Example 7 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink-mirror by flink-ci.

the class ChangelogBackendLogApplier method restoreKvMetaData.

private static RegisteredKeyValueStateBackendMetaInfo restoreKvMetaData(ChangelogKeyedStateBackend<?> backend, StateMetaInfoSnapshot snapshot, DataInputView in) throws Exception {
    RegisteredKeyValueStateBackendMetaInfo meta = new RegisteredKeyValueStateBackendMetaInfo(snapshot);
    StateTtlConfig ttlConfig = readTtlConfig(in);
    Object defaultValue = readDefaultValue(in, meta);
    // Use regular API to create states in both changelog and the base backends the metadata is
    // persisted in log before data changes.
    // An alternative solution to load metadata "natively" by the base backends would require
    // base state to be always present, i.e. the 1st checkpoint would have to be "full" always.
    StateDescriptor stateDescriptor = toStateDescriptor(meta, defaultValue);
    // todo: support changing ttl (FLINK-23143)
    if (ttlConfig.isEnabled()) {
        stateDescriptor.enableTimeToLive(ttlConfig);
    }
    backend.getOrCreateKeyedState(meta.getNamespaceSerializer(), stateDescriptor);
    return meta;
}

Example 8 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink-mirror by flink-ci.

the class RocksIncrementalSnapshotStrategyTest method createSnapshotStrategy.

public RocksIncrementalSnapshotStrategy createSnapshotStrategy(CloseableRegistry closeableRegistry) throws IOException, RocksDBException {
    ColumnFamilyHandle columnFamilyHandle = rocksDBResource.createNewColumnFamily("test");
    RocksDB rocksDB = rocksDBResource.getRocksDB();
    byte[] key = "checkpoint".getBytes();
    byte[] val = "incrementalTest".getBytes();
    rocksDB.put(columnFamilyHandle, key, val);
    // construct RocksIncrementalSnapshotStrategy
    long lastCompletedCheckpointId = -1L;
    ResourceGuard rocksDBResourceGuard = new ResourceGuard();
    SortedMap<Long, Map<StateHandleID, StreamStateHandle>> materializedSstFiles = new TreeMap<>();
    LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation = new LinkedHashMap<>();
    RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(RocksDBOptions.CHECKPOINT_TRANSFER_THREAD_NUM.defaultValue());
    int keyGroupPrefixBytes = CompositeKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(2);
    RegisteredKeyValueStateBackendMetaInfo<Integer, ArrayList<Integer>> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", IntSerializer.INSTANCE, new ArrayListSerializer<>(IntSerializer.INSTANCE));
    RocksDBKeyedStateBackend.RocksDbKvStateInfo rocksDbKvStateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(columnFamilyHandle, metaInfo);
    kvStateInformation.putIfAbsent("test", rocksDbKvStateInfo);
    return new RocksIncrementalSnapshotStrategy<>(rocksDB, rocksDBResourceGuard, IntSerializer.INSTANCE, kvStateInformation, new KeyGroupRange(0, 1), keyGroupPrefixBytes, TestLocalRecoveryConfig.disabled(), closeableRegistry, tmp.newFolder(), UUID.randomUUID(), materializedSstFiles, rocksDBStateUploader, lastCompletedCheckpointId);
}

Example 9 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink-mirror by flink-ci.

the class RocksDBKeyedStateBackend method getKeys.

@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
    RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
    if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) {
        return Stream.empty();
    }
    RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo = (RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo;
    final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer();
    final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8);
    boolean ambiguousKeyPossible = CompositeKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer);
    final byte[] nameSpaceBytes;
    try {
        CompositeKeySerializationUtils.writeNameSpace(namespace, namespaceSerializer, namespaceOutputView, ambiguousKeyPossible);
        nameSpaceBytes = namespaceOutputView.getCopyOfBuffer();
    } catch (IOException ex) {
        throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
    }
    RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle, readOptions);
    iterator.seekToFirst();
    final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes, ambiguousKeyPossible, nameSpaceBytes);
    Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false);
    return targetStream.onClose(iteratorWrapper::close);
}

Example 10 with RegisteredKeyValueStateBackendMetaInfo

use of org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo in project flink by splunk.

the class RocksIncrementalSnapshotStrategyTest method createSnapshotStrategy.

public RocksIncrementalSnapshotStrategy createSnapshotStrategy(CloseableRegistry closeableRegistry) throws IOException, RocksDBException {
    ColumnFamilyHandle columnFamilyHandle = rocksDBResource.createNewColumnFamily("test");
    RocksDB rocksDB = rocksDBResource.getRocksDB();
    byte[] key = "checkpoint".getBytes();
    byte[] val = "incrementalTest".getBytes();
    rocksDB.put(columnFamilyHandle, key, val);
    // construct RocksIncrementalSnapshotStrategy
    long lastCompletedCheckpointId = -1L;
    ResourceGuard rocksDBResourceGuard = new ResourceGuard();
    SortedMap<Long, Map<StateHandleID, StreamStateHandle>> materializedSstFiles = new TreeMap<>();
    LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation = new LinkedHashMap<>();
    RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(RocksDBOptions.CHECKPOINT_TRANSFER_THREAD_NUM.defaultValue());
    int keyGroupPrefixBytes = CompositeKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(2);
    RegisteredKeyValueStateBackendMetaInfo<Integer, ArrayList<Integer>> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", IntSerializer.INSTANCE, new ArrayListSerializer<>(IntSerializer.INSTANCE));
    RocksDBKeyedStateBackend.RocksDbKvStateInfo rocksDbKvStateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(columnFamilyHandle, metaInfo);
    kvStateInformation.putIfAbsent("test", rocksDbKvStateInfo);
    return new RocksIncrementalSnapshotStrategy<>(rocksDB, rocksDBResourceGuard, IntSerializer.INSTANCE, kvStateInformation, new KeyGroupRange(0, 1), keyGroupPrefixBytes, TestLocalRecoveryConfig.disabled(), closeableRegistry, tmp.newFolder(), UUID.randomUUID(), materializedSstFiles, rocksDBStateUploader, lastCompletedCheckpointId);
}