Examples with StateMetaInfoReader org.apache.flink.runtime.state.metainfo.StateMetaInfoReader used on opensource projects

Example 1 with StateMetaInfoReader

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoReader in project flink by apache.

the class SerializationProxiesTest method testKeyedStateMetaInfoSerialization.

@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {
    String name = "test";
    TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        StateMetaInfoSnapshotReadersWriters.getWriter().writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
        metaInfo = reader.readStateMetaInfoSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
    }
    Assert.assertEquals(name, metaInfo.getName());
}

Example 2 with StateMetaInfoReader

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoReader in project flink by apache.

the class SerializationProxiesTest method testOperatorStateMetaInfoSerialization.

@Test
public void testOperatorStateMetaInfoSerialization() throws Exception {
    String name = "test";
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    StateMetaInfoSnapshot snapshot = new RegisteredOperatorStateBackendMetaInfo<>(name, stateSerializer, OperatorStateHandle.Mode.UNION).snapshot();
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        StateMetaInfoSnapshotReadersWriters.getWriter().writeStateMetaInfoSnapshot(snapshot, new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.OPERATOR_STATE);
        snapshot = reader.readStateMetaInfoSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
    }
    RegisteredOperatorStateBackendMetaInfo<?> restoredMetaInfo = new RegisteredOperatorStateBackendMetaInfo<>(snapshot);
    Assert.assertEquals(name, restoredMetaInfo.getName());
    Assert.assertEquals(OperatorStateHandle.Mode.UNION, restoredMetaInfo.getAssignmentMode());
    Assert.assertEquals(stateSerializer, restoredMetaInfo.getPartitionStateSerializer());
}

Example 3 with StateMetaInfoReader

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoReader in project flink by apache.

the class SerializationProxiesTest method testBroadcastStateMetaInfoSerialization.

@Test
public void testBroadcastStateMetaInfoSerialization() throws Exception {
    String name = "test";
    TypeSerializer<?> keySerializer = DoubleSerializer.INSTANCE;
    TypeSerializer<?> valueSerializer = StringSerializer.INSTANCE;
    StateMetaInfoSnapshot snapshot = new RegisteredBroadcastStateBackendMetaInfo<>(name, OperatorStateHandle.Mode.BROADCAST, keySerializer, valueSerializer).snapshot();
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        StateMetaInfoSnapshotReadersWriters.getWriter().writeStateMetaInfoSnapshot(snapshot, new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.OPERATOR_STATE);
        snapshot = reader.readStateMetaInfoSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
    }
    RegisteredBroadcastStateBackendMetaInfo<?, ?> restoredMetaInfo = new RegisteredBroadcastStateBackendMetaInfo<>(snapshot);
    Assert.assertEquals(name, restoredMetaInfo.getName());
    Assert.assertEquals(OperatorStateHandle.Mode.BROADCAST, restoredMetaInfo.getAssignmentMode());
    Assert.assertEquals(keySerializer, restoredMetaInfo.getKeySerializer());
    Assert.assertEquals(valueSerializer, restoredMetaInfo.getValueSerializer());
}

Example 4 with StateMetaInfoReader

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoReader in project flink by apache.

the class KeyedBackendSerializationProxy method read.

@SuppressWarnings("unchecked")
@Override
public void read(DataInputView in) throws IOException {
    super.read(in);
    final int readVersion = getReadVersion();
    if (readVersion >= 4) {
        usingKeyGroupCompression = in.readBoolean();
    } else {
        usingKeyGroupCompression = false;
    }
    // only starting from version 3, we have the key serializer and its config snapshot written
    if (readVersion >= 6) {
        this.keySerializerSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(in, userCodeClassLoader, null);
    } else if (readVersion >= 3) {
        Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>> keySerializerAndConfig = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader).get(0);
        this.keySerializerSnapshot = (TypeSerializerSnapshot<K>) keySerializerAndConfig.f1;
    } else {
        this.keySerializerSnapshot = new BackwardsCompatibleSerializerSnapshot<>(TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true));
    }
    this.keySerializer = null;
    Integer metaInfoSnapshotVersion = META_INFO_SNAPSHOT_FORMAT_VERSION_MAPPER.get(readVersion);
    if (metaInfoSnapshotVersion == null) {
        // this should not happen; guard for the future
        throw new IOException("Cannot determine corresponding meta info snapshot version for keyed backend serialization readVersion=" + readVersion);
    }
    final StateMetaInfoReader stateMetaInfoReader = StateMetaInfoSnapshotReadersWriters.getReader(metaInfoSnapshotVersion, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
    int numKvStates = in.readShort();
    stateMetaInfoSnapshots = new ArrayList<>(numKvStates);
    for (int i = 0; i < numKvStates; i++) {
        StateMetaInfoSnapshot snapshot = stateMetaInfoReader.readStateMetaInfoSnapshot(in, userCodeClassLoader);
        stateMetaInfoSnapshots.add(snapshot);
    }
}

Example 5 with StateMetaInfoReader

use of org.apache.flink.runtime.state.metainfo.StateMetaInfoReader in project flink by apache.

the class OperatorBackendSerializationProxy method read.

@Override
public void read(DataInputView in) throws IOException {
    super.read(in);
    final int proxyReadVersion = getReadVersion();
    final Integer metaInfoSnapshotVersion = META_INFO_SNAPSHOT_FORMAT_VERSION_MAPPER.get(proxyReadVersion);
    if (metaInfoSnapshotVersion == null) {
        // this should not happen; guard for the future
        throw new IOException("Cannot determine corresponding meta info snapshot version for operator backend serialization readVersion=" + proxyReadVersion);
    }
    final StateMetaInfoReader stateMetaInfoReader = StateMetaInfoSnapshotReadersWriters.getReader(metaInfoSnapshotVersion, StateMetaInfoSnapshotReadersWriters.StateTypeHint.OPERATOR_STATE);
    int numOperatorStates = in.readShort();
    operatorStateMetaInfoSnapshots = new ArrayList<>(numOperatorStates);
    for (int i = 0; i < numOperatorStates; i++) {
        operatorStateMetaInfoSnapshots.add(stateMetaInfoReader.readStateMetaInfoSnapshot(in, userCodeClassLoader));
    }
    if (proxyReadVersion >= 3) {
        // broadcast states did not exist prior to version 3
        int numBroadcastStates = in.readShort();
        broadcastStateMetaInfoSnapshots = new ArrayList<>(numBroadcastStates);
        for (int i = 0; i < numBroadcastStates; i++) {
            broadcastStateMetaInfoSnapshots.add(stateMetaInfoReader.readStateMetaInfoSnapshot(in, userCodeClassLoader));
        }
    } else {
        broadcastStateMetaInfoSnapshots = new ArrayList<>();
    }
}