Examples with StateMetaInfoSnapshot org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot used on opensource projects

Example 6 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class RocksDBIncrementalRestoreOperation method createColumnFamilyDescriptors.

/**
 * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state
 * meta data snapshot.
 */
private List<ColumnFamilyDescriptor> createColumnFamilyDescriptors(List<StateMetaInfoSnapshot> stateMetaInfoSnapshots, boolean registerTtlCompactFilter) {
    List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(stateMetaInfoSnapshots.size());
    for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
        RegisteredStateMetaInfoBase metaInfoBase = RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
        ColumnFamilyDescriptor columnFamilyDescriptor = RocksDBOperationUtils.createColumnFamilyDescriptor(metaInfoBase, this.rocksHandle.getColumnFamilyOptionsFactory(), registerTtlCompactFilter ? this.rocksHandle.getTtlCompactFiltersManager() : null, this.rocksHandle.getWriteBufferManagerCapacity());
        columnFamilyDescriptors.add(columnFamilyDescriptor);
    }
    return columnFamilyDescriptors;
}

Example 7 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class RocksDBHeapTimersFullRestoreOperation method applyRestoreResult.

private void applyRestoreResult(SavepointRestoreResult savepointRestoreResult) throws IOException, RocksDBException, StateMigrationException {
    List<StateMetaInfoSnapshot> restoredMetaInfos = savepointRestoreResult.getStateMetaInfoSnapshots();
    Map<Integer, ColumnFamilyHandle> columnFamilyHandles = new HashMap<>();
    Map<Integer, HeapPriorityQueueSnapshotRestoreWrapper<?>> restoredPQStates = new HashMap<>();
    for (int i = 0; i < restoredMetaInfos.size(); i++) {
        StateMetaInfoSnapshot restoredMetaInfo = restoredMetaInfos.get(i);
        if (restoredMetaInfo.getBackendStateType() == BackendStateType.PRIORITY_QUEUE) {
            String stateName = restoredMetaInfo.getName();
            HeapPriorityQueueSnapshotRestoreWrapper<?> queueWrapper = registeredPQStates.computeIfAbsent(stateName, key -> createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(restoredMetaInfo)));
            restoredPQStates.put(i, queueWrapper);
        } else {
            RocksDbKvStateInfo registeredStateCFHandle = this.rocksHandle.getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
            columnFamilyHandles.put(i, registeredStateCFHandle.columnFamilyHandle);
        }
    }
    try (ThrowingIterator<KeyGroup> keyGroups = savepointRestoreResult.getRestoredKeyGroups()) {
        restoreKVStateData(keyGroups, columnFamilyHandles, restoredPQStates);
    }
}

Example 8 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class RocksDBFullRestoreOperation method applyRestoreResult.

private void applyRestoreResult(SavepointRestoreResult savepointRestoreResult) throws IOException, RocksDBException, StateMigrationException {
    List<StateMetaInfoSnapshot> restoredMetaInfos = savepointRestoreResult.getStateMetaInfoSnapshots();
    Map<Integer, ColumnFamilyHandle> columnFamilyHandles = new HashMap<>();
    for (int i = 0; i < restoredMetaInfos.size(); i++) {
        StateMetaInfoSnapshot restoredMetaInfo = restoredMetaInfos.get(i);
        RocksDbKvStateInfo registeredStateCFHandle = this.rocksHandle.getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
        columnFamilyHandles.put(i, registeredStateCFHandle.columnFamilyHandle);
    }
    try (ThrowingIterator<KeyGroup> keyGroups = savepointRestoreResult.getRestoredKeyGroups()) {
        restoreKVStateData(keyGroups, columnFamilyHandles);
    }
}

Example 9 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class SerializationProxiesTest method testKeyedStateMetaInfoSerialization.

@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {
    String name = "test";
    TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        StateMetaInfoSnapshotReadersWriters.getWriter().writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
        metaInfo = reader.readStateMetaInfoSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
    }
    Assert.assertEquals(name, metaInfo.getName());
}

Example 10 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class RocksDBFullSnapshotResources method create.

public static <K> RocksDBFullSnapshotResources<K> create(final LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation, // TODO: was it important that this is a LinkedHashMap
final Map<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates, final RocksDB db, final ResourceGuard rocksDBResourceGuard, final KeyGroupRange keyGroupRange, final TypeSerializer<K> keySerializer, final int keyGroupPrefixBytes, final StreamCompressionDecorator keyGroupCompressionDecorator) throws IOException {
    final List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = new ArrayList<>(kvStateInformation.size());
    final List<RocksDBKeyedStateBackend.RocksDbKvStateInfo> metaDataCopy = new ArrayList<>(kvStateInformation.size());
    for (RocksDBKeyedStateBackend.RocksDbKvStateInfo stateInfo : kvStateInformation.values()) {
        // snapshot meta info
        stateMetaInfoSnapshots.add(stateInfo.metaInfo.snapshot());
        metaDataCopy.add(stateInfo);
    }
    List<HeapPriorityQueueStateSnapshot<?>> heapPriorityQueuesSnapshots = new ArrayList<>(registeredPQStates.size());
    for (HeapPriorityQueueSnapshotRestoreWrapper<?> stateInfo : registeredPQStates.values()) {
        stateMetaInfoSnapshots.add(stateInfo.getMetaInfo().snapshot());
        heapPriorityQueuesSnapshots.add(stateInfo.stateSnapshot());
    }
    final ResourceGuard.Lease lease = rocksDBResourceGuard.acquireResource();
    final Snapshot snapshot = db.getSnapshot();
    return new RocksDBFullSnapshotResources<>(lease, snapshot, metaDataCopy, heapPriorityQueuesSnapshots, stateMetaInfoSnapshots, db, keyGroupPrefixBytes, keyGroupRange, keySerializer, keyGroupCompressionDecorator);
}