Examples with StateMetaInfoSnapshot org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot used on opensource projects

Example 16 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class SerializationProxiesTest method testOperatorStateMetaInfoSerialization.

@Test
public void testOperatorStateMetaInfoSerialization() throws Exception {
    String name = "test";
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    StateMetaInfoSnapshot snapshot = new RegisteredOperatorStateBackendMetaInfo<>(name, stateSerializer, OperatorStateHandle.Mode.UNION).snapshot();
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        StateMetaInfoSnapshotReadersWriters.getWriter().writeStateMetaInfoSnapshot(snapshot, new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.OPERATOR_STATE);
        snapshot = reader.readStateMetaInfoSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
    }
    RegisteredOperatorStateBackendMetaInfo<?> restoredMetaInfo = new RegisteredOperatorStateBackendMetaInfo<>(snapshot);
    Assert.assertEquals(name, restoredMetaInfo.getName());
    Assert.assertEquals(OperatorStateHandle.Mode.UNION, restoredMetaInfo.getAssignmentMode());
    Assert.assertEquals(stateSerializer, restoredMetaInfo.getPartitionStateSerializer());
}

Example 17 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class SerializationProxiesTest method testKeyedBackendSerializationProxyRoundtrip.

@Test
public void testKeyedBackendSerializationProxyRoundtrip() throws Exception {
    TypeSerializer<?> keySerializer = IntSerializer.INSTANCE;
    TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    List<StateMetaInfoSnapshot> stateMetaInfoList = new ArrayList<>();
    stateMetaInfoList.add(new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "a", namespaceSerializer, stateSerializer).snapshot());
    stateMetaInfoList.add(new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "b", namespaceSerializer, stateSerializer).snapshot());
    stateMetaInfoList.add(new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "c", namespaceSerializer, stateSerializer).snapshot());
    KeyedBackendSerializationProxy<?> serializationProxy = new KeyedBackendSerializationProxy<>(keySerializer, stateMetaInfoList, true);
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        serializationProxy.write(new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    serializationProxy = new KeyedBackendSerializationProxy<>(Thread.currentThread().getContextClassLoader());
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        serializationProxy.read(new DataInputViewStreamWrapper(in));
    }
    Assert.assertTrue(serializationProxy.isUsingKeyGroupCompression());
    Assert.assertTrue(serializationProxy.getKeySerializerSnapshot() instanceof IntSerializer.IntSerializerSnapshot);
    assertEqualStateMetaInfoSnapshotsLists(stateMetaInfoList, serializationProxy.getStateMetaInfoSnapshots());
}

Example 18 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class SerializationProxiesTest method testBroadcastStateMetaInfoSerialization.

@Test
public void testBroadcastStateMetaInfoSerialization() throws Exception {
    String name = "test";
    TypeSerializer<?> keySerializer = DoubleSerializer.INSTANCE;
    TypeSerializer<?> valueSerializer = StringSerializer.INSTANCE;
    StateMetaInfoSnapshot snapshot = new RegisteredBroadcastStateBackendMetaInfo<>(name, OperatorStateHandle.Mode.BROADCAST, keySerializer, valueSerializer).snapshot();
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        StateMetaInfoSnapshotReadersWriters.getWriter().writeStateMetaInfoSnapshot(snapshot, new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.OPERATOR_STATE);
        snapshot = reader.readStateMetaInfoSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
    }
    RegisteredBroadcastStateBackendMetaInfo<?, ?> restoredMetaInfo = new RegisteredBroadcastStateBackendMetaInfo<>(snapshot);
    Assert.assertEquals(name, restoredMetaInfo.getName());
    Assert.assertEquals(OperatorStateHandle.Mode.BROADCAST, restoredMetaInfo.getAssignmentMode());
    Assert.assertEquals(keySerializer, restoredMetaInfo.getKeySerializer());
    Assert.assertEquals(valueSerializer, restoredMetaInfo.getValueSerializer());
}

Example 19 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class SerializationProxiesTest method testOperatorBackendSerializationProxyRoundtrip.

@Test
public void testOperatorBackendSerializationProxyRoundtrip() throws Exception {
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    TypeSerializer<?> keySerializer = DoubleSerializer.INSTANCE;
    TypeSerializer<?> valueSerializer = StringSerializer.INSTANCE;
    List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = new ArrayList<>();
    stateMetaInfoSnapshots.add(new RegisteredOperatorStateBackendMetaInfo<>("a", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE).snapshot());
    stateMetaInfoSnapshots.add(new RegisteredOperatorStateBackendMetaInfo<>("b", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE).snapshot());
    stateMetaInfoSnapshots.add(new RegisteredOperatorStateBackendMetaInfo<>("c", stateSerializer, OperatorStateHandle.Mode.UNION).snapshot());
    List<StateMetaInfoSnapshot> broadcastStateMetaInfoSnapshots = new ArrayList<>();
    broadcastStateMetaInfoSnapshots.add(new RegisteredBroadcastStateBackendMetaInfo<>("d", OperatorStateHandle.Mode.BROADCAST, keySerializer, valueSerializer).snapshot());
    broadcastStateMetaInfoSnapshots.add(new RegisteredBroadcastStateBackendMetaInfo<>("e", OperatorStateHandle.Mode.BROADCAST, valueSerializer, keySerializer).snapshot());
    OperatorBackendSerializationProxy serializationProxy = new OperatorBackendSerializationProxy(stateMetaInfoSnapshots, broadcastStateMetaInfoSnapshots);
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        serializationProxy.write(new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    serializationProxy = new OperatorBackendSerializationProxy(Thread.currentThread().getContextClassLoader());
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        serializationProxy.read(new DataInputViewStreamWrapper(in));
    }
    assertEqualStateMetaInfoSnapshotsLists(stateMetaInfoSnapshots, serializationProxy.getOperatorStateMetaInfoSnapshots());
    assertEqualStateMetaInfoSnapshotsLists(broadcastStateMetaInfoSnapshots, serializationProxy.getBroadcastStateMetaInfoSnapshots());
}

Example 20 with StateMetaInfoSnapshot

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot in project flink by apache.

the class RocksDBIncrementalRestoreOperation method restoreDBInstanceFromStateHandle.

private RestoredDBInstance restoreDBInstanceFromStateHandle(IncrementalRemoteKeyedStateHandle restoreStateHandle, Path temporaryRestoreInstancePath) throws Exception {
    try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(numberOfTransferringThreads)) {
        rocksDBStateDownloader.transferAllStateDataToDirectory(restoreStateHandle, temporaryRestoreInstancePath, cancelStreamRegistry);
    }
    KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
    // read meta data
    List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
    List<ColumnFamilyDescriptor> columnFamilyDescriptors = createColumnFamilyDescriptors(stateMetaInfoSnapshots, false);
    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(stateMetaInfoSnapshots.size() + 1);
    RocksDB restoreDb = RocksDBOperationUtils.openDB(temporaryRestoreInstancePath.toString(), columnFamilyDescriptors, columnFamilyHandles, RocksDBOperationUtils.createColumnFamilyOptions(this.rocksHandle.getColumnFamilyOptionsFactory(), "default"), this.rocksHandle.getDbOptions());
    return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots);
}