Examples with DataInputViewStreamWrapper org.apache.flink.core.memory.DataInputViewStreamWrapper used on opensource projects

Example 56 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class TaskConfig method getOutputDataDistribution.

public DataDistribution getOutputDataDistribution(int outputNum, final ClassLoader cl) throws ClassNotFoundException {
    final String className = this.config.getString(OUTPUT_DATA_DISTRIBUTION_CLASS, null);
    if (className == null) {
        return null;
    }
    final Class<? extends DataDistribution> clazz;
    try {
        clazz = Class.forName(className, true, cl).asSubclass(DataDistribution.class);
    } catch (ClassCastException ccex) {
        throw new CorruptConfigurationException("The class noted in the configuration as the data distribution " + "is no subclass of DataDistribution.");
    }
    final DataDistribution distribution = InstantiationUtil.instantiate(clazz, DataDistribution.class);
    final byte[] stateEncoded = this.config.getBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, null);
    if (stateEncoded == null) {
        throw new CorruptConfigurationException("The configuration contained the data distribution type, but no serialized state.");
    }
    final ByteArrayInputStream bais = new ByteArrayInputStream(stateEncoded);
    final DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
    try {
        distribution.read(in);
        return distribution;
    } catch (Exception ex) {
        throw new RuntimeException("The deserialization of the encoded data distribution state caused an error" + (ex.getMessage() == null ? "." : ": " + ex.getMessage()), ex);
    }
}

Example 57 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project beam by apache.

the class DedupingOperator method initializeState.

@Override
public void initializeState(StateInitializationContext context) throws Exception {
    if (getKeyedStateBackend() != null) {
        KeyGroupsList localKeyGroupRange = getKeyedStateBackend().getKeyGroupRange();
        for (KeyGroupStatePartitionStreamProvider streamProvider : context.getRawKeyedStateInputs()) {
            DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(streamProvider.getStream());
            int keyGroupIdx = streamProvider.getKeyGroupId();
            checkArgument(localKeyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range.");
            // if (this instanceof KeyGroupRestoringOperator)
            restoreKeyGroupState(keyGroupIdx, div);
        }
    }
}

Example 58 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class OperatorStateOutputCheckpointStreamTest method verifyRead.

private static void verifyRead(OperatorStateHandle fullHandle, int numPartitions) throws IOException {
    int count = 0;
    try (FSDataInputStream in = fullHandle.openInputStream()) {
        OperatorStateHandle.StateMetaInfo metaInfo = fullHandle.getStateNameToPartitionOffsets().get(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);
        long[] offsets = metaInfo.getOffsets();
        Assert.assertNotNull(offsets);
        DataInputView div = new DataInputViewStreamWrapper(in);
        for (int i = 0; i < numPartitions; ++i) {
            in.seek(offsets[i]);
            Assert.assertEquals(i, div.readInt());
            ++count;
        }
    }
    Assert.assertEquals(numPartitions, count);
}

Example 59 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class SerializationProxiesTest method testOperatorStateMetaInfoSerialization.

@Test
public void testOperatorStateMetaInfoSerialization() throws Exception {
    String name = "test";
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    OperatorBackendSerializationProxy.StateMetaInfo<?> metaInfo = new OperatorBackendSerializationProxy.StateMetaInfo<>(name, stateSerializer, OperatorStateHandle.Mode.BROADCAST);
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        metaInfo.write(new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    metaInfo = new OperatorBackendSerializationProxy.StateMetaInfo<>(Thread.currentThread().getContextClassLoader());
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        metaInfo.read(new DataInputViewStreamWrapper(in));
    }
    Assert.assertEquals(name, metaInfo.getName());
}

Example 60 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class SerializationProxiesTest method testOperatorBackendSerializationProxyRoundtrip.

@Test
public void testOperatorBackendSerializationProxyRoundtrip() throws Exception {
    TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
    List<OperatorBackendSerializationProxy.StateMetaInfo<?>> stateMetaInfoList = new ArrayList<>();
    stateMetaInfoList.add(new OperatorBackendSerializationProxy.StateMetaInfo<>("a", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
    stateMetaInfoList.add(new OperatorBackendSerializationProxy.StateMetaInfo<>("b", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
    stateMetaInfoList.add(new OperatorBackendSerializationProxy.StateMetaInfo<>("c", stateSerializer, OperatorStateHandle.Mode.BROADCAST));
    OperatorBackendSerializationProxy serializationProxy = new OperatorBackendSerializationProxy(stateMetaInfoList);
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        serializationProxy.write(new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    serializationProxy = new OperatorBackendSerializationProxy(Thread.currentThread().getContextClassLoader());
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        serializationProxy.read(new DataInputViewStreamWrapper(in));
    }
    Assert.assertEquals(stateMetaInfoList, serializationProxy.getNamedStateSerializationProxies());
}