Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 36 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class StringValueSerializationTest method testCopy.

public static void testCopy(String[] values) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    DataOutputViewStreamWrapper serializer = new DataOutputViewStreamWrapper(baos);
    StringValue sValue = new StringValue();
    for (String value : values) {
        sValue.setValue(value);
        sValue.write(serializer);
    }
    serializer.close();
    baos.close();
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInputViewStreamWrapper source = new DataInputViewStreamWrapper(bais);
    ByteArrayOutputStream targetOutput = new ByteArrayOutputStream(4096);
    DataOutputViewStreamWrapper target = new DataOutputViewStreamWrapper(targetOutput);
    for (String value : values) {
        sValue.copy(source, target);
    }
    ByteArrayInputStream validateInput = new ByteArrayInputStream(targetOutput.toByteArray());
    DataInputViewStreamWrapper validate = new DataInputViewStreamWrapper(validateInput);
    int num = 0;
    while (validateInput.available() > 0) {
        sValue.read(validate);
        assertEquals("DeserializedString differs from original string.", values[num], sValue.getValue());
        num++;
    }
    assertEquals("Wrong number of deserialized values", values.length, num);
}

Example 37 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SequentialFormatTestBase method calcRawDataSize.

/**
 * Count how many bytes would be written if all records were directly serialized.
 */
@Before
public void calcRawDataSize() throws IOException {
    int recordIndex = 0;
    for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
        ByteCounter byteCounter = new ByteCounter();
        for (int fileCount = 0; fileCount < this.getNumberOfTuplesPerFile(fileIndex); fileCount++, recordIndex++) {
            writeRecord(this.getRecord(recordIndex), new DataOutputViewStreamWrapper(byteCounter));
        }
        this.rawDataSizes[fileIndex] = byteCounter.getLength();
    }
}

Example 38 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class TypeSerializerSerializationUtil method writeSerializersAndConfigsWithResilience.

/**
 * Write a list of serializers and their corresponding config snapshots to the provided data
 * output view. This method writes in a fault tolerant way, so that when read again using {@link
 * #readSerializersAndConfigsWithResilience(DataInputView, ClassLoader)}, if deserialization of
 * the serializer fails, its configuration snapshot will remain intact.
 *
 * <p>Specifically, all written serializers and their config snapshots are indexed by their
 * offset positions within the serialized bytes. The serialization format is as follows:
 *
 * <ul>
 *   <li>1. number of serializer and configuration snapshot pairs.
 *   <li>2. offsets of each serializer and configuration snapshot, in order.
 *   <li>3. total number of bytes for the serialized serializers and the config snapshots.
 *   <li>4. serialized serializers and the config snapshots.
 * </ul>
 *
 * @param out the data output view.
 * @param serializersAndConfigs serializer and configuration snapshot pairs
 * @throws IOException
 */
public static void writeSerializersAndConfigsWithResilience(DataOutputView out, List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs) throws IOException {
    try (ByteArrayOutputStreamWithPos bufferWithPos = new ByteArrayOutputStreamWithPos();
        DataOutputViewStreamWrapper bufferWrapper = new DataOutputViewStreamWrapper(bufferWithPos)) {
        out.writeInt(serializersAndConfigs.size());
        for (Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>> serAndConfSnapshot : serializersAndConfigs) {
            out.writeInt(bufferWithPos.getPosition());
            writeSerializer(bufferWrapper, serAndConfSnapshot.f0);
            out.writeInt(bufferWithPos.getPosition());
            TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(bufferWrapper, (TypeSerializerSnapshot) serAndConfSnapshot.f1, serAndConfSnapshot.f0);
        }
        out.writeInt(bufferWithPos.getPosition());
        out.write(bufferWithPos.getBuf(), 0, bufferWithPos.getPosition());
    }
}

Example 39 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class StateDescriptor method writeObject.

// ------------------------------------------------------------------------
// Serialization
// ------------------------------------------------------------------------
private void writeObject(final ObjectOutputStream out) throws IOException {
    // write all the non-transient fields
    out.defaultWriteObject();
    // write the non-serializable default value field
    if (defaultValue == null) {
        // we don't have a default value
        out.writeBoolean(false);
    } else {
        TypeSerializer<T> serializer = serializerAtomicReference.get();
        checkNotNull(serializer, "Serializer not initialized.");
        // we have a default value
        out.writeBoolean(true);
        byte[] serializedDefaultValue;
        try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
            DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {
            TypeSerializer<T> duplicateSerializer = serializer.duplicate();
            duplicateSerializer.serialize(defaultValue, outView);
            outView.flush();
            serializedDefaultValue = baos.toByteArray();
        } catch (Exception e) {
            throw new IOException("Unable to serialize default value of type " + defaultValue.getClass().getSimpleName() + ".", e);
        }
        out.writeInt(serializedDefaultValue.length);
        out.write(serializedDefaultValue);
    }
}

Example 40 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class InstantiationUtil method createCopyWritable.

/**
 * Clones the given writable using the {@link IOReadableWritable serialization}.
 *
 * @param original Object to clone
 * @param <T> Type of the object to clone
 * @return Cloned object
 * @throws IOException Thrown is the serialization fails.
 */
public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException {
    if (original == null) {
        return null;
    }
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
        original.write(out);
    }
    final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {
        @SuppressWarnings("unchecked") T copy = (T) instantiate(original.getClass());
        copy.read(in);
        return copy;
    }
}