Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 31 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SerializerTestBase method testSnapshotConfigurationAndReconfigure.

@Test
public void testSnapshotConfigurationAndReconfigure() throws Exception {
    final TypeSerializer<T> serializer = getSerializer();
    final TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), configSnapshot, serializer);
        serializedConfig = out.toByteArray();
    }
    TypeSerializerSnapshot<T> restoredConfig;
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), getSerializer());
    }
    TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(getSerializer());
    final TypeSerializer<T> restoreSerializer;
    if (strategy.isCompatibleAsIs()) {
        restoreSerializer = restoredConfig.restoreSerializer();
    } else if (strategy.isCompatibleWithReconfiguredSerializer()) {
        restoreSerializer = strategy.getReconfiguredSerializer();
    } else {
        throw new AssertionError("Unable to restore serializer with " + strategy);
    }
    assertEquals(serializer.getClass(), restoreSerializer.getClass());
}

Example 32 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class EnumSerializerCompatibilityTest method checkCompatibility.

@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB) throws IOException, ClassNotFoundException {
    ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);
    EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));
    TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
    byte[] snapshotBytes;
    try (ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
        DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(outputViewStreamWrapper, snapshot, enumSerializer);
        snapshotBytes = outBuffer.toByteArray();
    }
    ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);
    TypeSerializerSnapshot restoredSnapshot;
    try (ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
        DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {
        restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(inputViewStreamWrapper, classLoader2, enumSerializer);
    }
    EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
    return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}

Example 33 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class FullSnapshotAsyncWriter method writeKVStateData.

private void writeKVStateData(final KeyValueStateIterator mergeIterator, final CheckpointStreamWithResultProvider checkpointStreamWithResultProvider, final KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {
    byte[] previousKey = null;
    byte[] previousValue = null;
    DataOutputView kgOutView = null;
    OutputStream kgOutStream = null;
    CheckpointStateOutputStream checkpointOutputStream = checkpointStreamWithResultProvider.getCheckpointOutputStream();
    try {
        // preamble: setup with first key-group as our lookahead
        if (mergeIterator.isValid()) {
            // begin first key-group by recording the offset
            keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), checkpointOutputStream.getPos());
            // write the k/v-state id as metadata
            kgOutStream = snapshotResources.getStreamCompressionDecorator().decorateWithCompression(checkpointOutputStream);
            kgOutView = new DataOutputViewStreamWrapper(kgOutStream);
            // TODO this could be aware of keyGroupPrefixBytes and write only one byte
            // if possible
            kgOutView.writeShort(mergeIterator.kvStateId());
            previousKey = mergeIterator.key();
            previousValue = mergeIterator.value();
            mergeIterator.next();
        }
        // key-group offsets.
        while (mergeIterator.isValid()) {
            assert (!hasMetaDataFollowsFlag(previousKey));
            // after this k/v pair
            if (mergeIterator.isNewKeyGroup() || mergeIterator.isNewKeyValueState()) {
                // be cooperative and check for interruption from time to time in the
                // hot loop
                checkInterrupted();
                setMetaDataFollowsFlagInKey(previousKey);
            }
            writeKeyValuePair(previousKey, previousValue, kgOutView);
            // write meta data if we have to
            if (mergeIterator.isNewKeyGroup()) {
                // TODO this could be aware of keyGroupPrefixBytes and write only one
                // byte if possible
                kgOutView.writeShort(END_OF_KEY_GROUP_MARK);
                // this will just close the outer stream
                kgOutStream.close();
                // begin new key-group
                keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), checkpointOutputStream.getPos());
                // write the kev-state
                // TODO this could be aware of keyGroupPrefixBytes and write only one
                // byte if possible
                kgOutStream = snapshotResources.getStreamCompressionDecorator().decorateWithCompression(checkpointOutputStream);
                kgOutView = new DataOutputViewStreamWrapper(kgOutStream);
                kgOutView.writeShort(mergeIterator.kvStateId());
            } else if (mergeIterator.isNewKeyValueState()) {
                // write the k/v-state
                // TODO this could be aware of keyGroupPrefixBytes and write only one
                // byte if possible
                kgOutView.writeShort(mergeIterator.kvStateId());
            }
            // request next k/v pair
            previousKey = mergeIterator.key();
            previousValue = mergeIterator.value();
            mergeIterator.next();
        }
        // epilogue: write last key-group
        if (previousKey != null) {
            assert (!hasMetaDataFollowsFlag(previousKey));
            setMetaDataFollowsFlagInKey(previousKey);
            writeKeyValuePair(previousKey, previousValue, kgOutView);
            // TODO this could be aware of keyGroupPrefixBytes and write only one byte if
            // possible
            kgOutView.writeShort(END_OF_KEY_GROUP_MARK);
            // this will just close the outer stream
            kgOutStream.close();
            kgOutStream = null;
        }
    } finally {
        // this will just close the outer stream
        IOUtils.closeQuietly(kgOutStream);
    }
}

Example 34 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class PartitionableListState method write.

public long[] write(FSDataOutputStream out) throws IOException {
    long[] partitionOffsets = new long[internalList.size()];
    DataOutputView dov = new DataOutputViewStreamWrapper(out);
    for (int i = 0; i < internalList.size(); ++i) {
        S element = internalList.get(i);
        partitionOffsets[i] = out.getPos();
        getStateMetaInfo().getPartitionStateSerializer().serialize(element, dov);
    }
    return partitionOffsets;
}

Example 35 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class StringValueSerializationTest method testSerialization.

public static void testSerialization(String[] values) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    DataOutputViewStreamWrapper serializer = new DataOutputViewStreamWrapper(baos);
    for (String value : values) {
        StringValue sv = new StringValue(value);
        sv.write(serializer);
    }
    serializer.close();
    baos.close();
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInputViewStreamWrapper deserializer = new DataInputViewStreamWrapper(bais);
    int num = 0;
    while (bais.available() > 0) {
        StringValue deser = new StringValue();
        deser.read(deserializer);
        assertEquals("DeserializedString differs from original string.", values[num], deser.getValue());
        num++;
    }
    assertEquals("Wrong number of deserialized values", values.length, num);
}