Examples with DataInputViewStreamWrapper org.apache.flink.core.memory.DataInputViewStreamWrapper used on opensource projects

Example 61 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class SerializationProxiesTest method testKeyedBackendSerializationProxyRoundtrip.

@Test
public void testKeyedBackendSerializationProxyRoundtrip() throws Exception {
    TypeSerializer<?> keySerializer = IntSerializer.INSTANCE;
    TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfoList = new ArrayList<>();
    stateMetaInfoList.add(new KeyedBackendSerializationProxy.StateMetaInfo<>(StateDescriptor.Type.VALUE, "a", namespaceSerializer, stateSerializer));
    stateMetaInfoList.add(new KeyedBackendSerializationProxy.StateMetaInfo<>(StateDescriptor.Type.VALUE, "b", namespaceSerializer, stateSerializer));
    stateMetaInfoList.add(new KeyedBackendSerializationProxy.StateMetaInfo<>(StateDescriptor.Type.VALUE, "c", namespaceSerializer, stateSerializer));
    KeyedBackendSerializationProxy serializationProxy = new KeyedBackendSerializationProxy(keySerializer, stateMetaInfoList);
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        serializationProxy.write(new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    serializationProxy = new KeyedBackendSerializationProxy(Thread.currentThread().getContextClassLoader());
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        serializationProxy.read(new DataInputViewStreamWrapper(in));
    }
    Assert.assertEquals(keySerializer, serializationProxy.getKeySerializerProxy().getTypeSerializer());
    Assert.assertEquals(stateMetaInfoList, serializationProxy.getNamedStateSerializationProxies());
}

Example 62 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class KeyedStateCheckpointOutputStreamTest method testReadWriteMissingKeyGroups.

@Test
public void testReadWriteMissingKeyGroups() throws Exception {
    final KeyGroupRange keyRange = new KeyGroupRange(0, 2);
    KeyedStateCheckpointOutputStream stream = createStream(keyRange);
    DataOutputView dov = new DataOutputViewStreamWrapper(stream);
    stream.startNewKeyGroup(1);
    dov.writeInt(1);
    KeyGroupsStateHandle fullHandle = stream.closeAndGetHandle();
    int count = 0;
    try (FSDataInputStream in = fullHandle.openInputStream()) {
        DataInputView div = new DataInputViewStreamWrapper(in);
        for (int kg : fullHandle.keyGroups()) {
            long off = fullHandle.getOffsetForKeyGroup(kg);
            if (off >= 0) {
                in.seek(off);
                Assert.assertEquals(1, div.readInt());
                ++count;
            }
        }
    }
    Assert.assertEquals(1, count);
}

Example 63 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class StateTableSnapshotCompatibilityTest method restoreStateTableFromSnapshot.

private static <K, N, S> void restoreStateTableFromSnapshot(StateTable<K, N, S> stateTable, StateTableSnapshot snapshot, KeyGroupRange keyGroupRange) throws IOException {
    final ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos(1024 * 1024);
    final DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out);
    for (Integer keyGroup : keyGroupRange) {
        snapshot.writeMappingsInKeyGroup(dov, keyGroup);
    }
    final ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(out.getBuf());
    final DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(in);
    final StateTableByKeyGroupReader keyGroupReader = StateTableByKeyGroupReaders.readerForVersion(stateTable, KeyedBackendSerializationProxy.VERSION);
    for (Integer keyGroup : keyGroupRange) {
        keyGroupReader.readMappingsInKeyGroup(div, keyGroup);
    }
}

Example 64 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class FoldApplyProcessWindowFunction method open.

@Override
public void open(Configuration configuration) throws Exception {
    FunctionUtils.openFunction(this.windowFunction, configuration);
    if (serializedInitialValue == null) {
        throw new RuntimeException("No initial value was serialized for the fold " + "window function. Probably the setOutputType method was not called.");
    }
    ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
    DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
    initialValue = accSerializer.deserialize(in);
}

Example 65 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class AbstractStreamOperator method initializeState.

/**
	 * Stream operators with state which can be restored need to override this hook method.
	 *
	 * @param context context that allows to register different states.
	 */
public void initializeState(StateInitializationContext context) throws Exception {
    if (getKeyedStateBackend() != null) {
        KeyGroupsList localKeyGroupRange = getKeyedStateBackend().getKeyGroupRange();
        // and then initialize the timer services
        for (KeyGroupStatePartitionStreamProvider streamProvider : context.getRawKeyedStateInputs()) {
            int keyGroupIdx = streamProvider.getKeyGroupId();
            checkArgument(localKeyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range.");
            timeServiceManager.restoreStateForKeyGroup(new DataInputViewStreamWrapper(streamProvider.getStream()), keyGroupIdx, getUserCodeClassloader());
        }
    }
}