Examples with DataOutputSerializer org.apache.flink.core.memory.DataOutputSerializer used on opensource projects

Example 16 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class TypeInformationKeyValueSerializationSchema method serializeKey.

@Override
public byte[] serializeKey(Tuple2<K, V> element) {
    if (element.f0 == null) {
        return null;
    } else {
        // key is not null. serialize it:
        if (keyOutputSerializer == null) {
            keyOutputSerializer = new DataOutputSerializer(16);
        }
        try {
            keySerializer.serialize(element.f0, keyOutputSerializer);
        } catch (IOException e) {
            throw new RuntimeException("Unable to serialize record", e);
        }
        // check if key byte array size changed
        byte[] res = keyOutputSerializer.getByteArray();
        if (res.length != keyOutputSerializer.length()) {
            byte[] n = new byte[keyOutputSerializer.length()];
            System.arraycopy(res, 0, n, 0, keyOutputSerializer.length());
            res = n;
        }
        keyOutputSerializer.clear();
        return res;
    }
}

Example 17 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class KvStateSerializer method serializeMap.

/**
 * Serializes all values of the Iterable with the given serializer.
 *
 * @param entries Key-value pairs to serialize
 * @param keySerializer Serializer for UK
 * @param valueSerializer Serializer for UV
 * @param <UK> Type of the keys
 * @param <UV> Type of the values
 * @return Serialized values or <code>null</code> if values <code>null</code> or empty
 * @throws IOException On failure during serialization
 */
public static <UK, UV> byte[] serializeMap(Iterable<Map.Entry<UK, UV>> entries, TypeSerializer<UK> keySerializer, TypeSerializer<UV> valueSerializer) throws IOException {
    if (entries != null) {
        // Serialize
        DataOutputSerializer dos = new DataOutputSerializer(32);
        for (Map.Entry<UK, UV> entry : entries) {
            keySerializer.serialize(entry.getKey(), dos);
            if (entry.getValue() == null) {
                dos.writeBoolean(true);
            } else {
                dos.writeBoolean(false);
                valueSerializer.serialize(entry.getValue(), dos);
            }
        }
        return dos.getCopyOfBuffer();
    } else {
        return null;
    }
}

Example 18 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class SerializerComparatorTestData method getOrderedStringTestData.

@SuppressWarnings("unchecked")
static Tuple2<byte[], StreamRecord<String>>[] getOrderedStringTestData() {
    StringSerializer stringSerializer = new StringSerializer();
    DataOutputSerializer outputSerializer = new DataOutputSerializer(64);
    return Stream.of(new String(new byte[] { -1, 0 }), new String(new byte[] { 0, 1 }), "A", "AB", "ABC", "ABCD", "ABCDE", "ABCDEF", "ABCDEFG", "ABCDEFGH").map(str -> {
        try {
            stringSerializer.serialize(str, outputSerializer);
            byte[] copyOfBuffer = outputSerializer.getCopyOfBuffer();
            outputSerializer.clear();
            return Tuple2.of(copyOfBuffer, new StreamRecord<>(str, 0));
        } catch (IOException e) {
            throw new AssertionError(e);
        }
    }).sorted((o1, o2) -> {
        byte[] key0 = o1.f0;
        byte[] key1 = o2.f0;
        int firstLength = key0.length;
        int secondLength = key1.length;
        int minLength = Math.min(firstLength, secondLength);
        for (int i = 0; i < minLength; i++) {
            int cmp = Byte.compare(key0[i], key1[i]);
            if (cmp != 0) {
                return cmp;
            }
        }
        int lengthCmp = Integer.compare(firstLength, secondLength);
        if (lengthCmp != 0) {
            return lengthCmp;
        }
        return Long.compare(o1.f1.getTimestamp(), o2.f1.getTimestamp());
    }).toArray(Tuple2[]::new);
}

Example 19 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class StreamTaskNetworkInputTest method serializeRecord.

private void serializeRecord(long value, BufferBuilder bufferBuilder) throws IOException {
    DataOutputSerializer serializer = new DataOutputSerializer(128);
    SerializationDelegate<StreamElement> serializationDelegate = new SerializationDelegate<>(new StreamElementSerializer<>(LongSerializer.INSTANCE));
    serializationDelegate.setInstance(new StreamRecord<>(value));
    ByteBuffer serializedRecord = RecordWriter.serializeRecord(serializer, serializationDelegate);
    bufferBuilder.appendAndCommit(serializedRecord);
    assertFalse(bufferBuilder.isFull());
}

Example 20 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class SerializerComparatorTestData method getOrderedIntTestData.

@SuppressWarnings("unchecked")
static Tuple2<byte[], StreamRecord<Integer>>[] getOrderedIntTestData() {
    IntSerializer intSerializer = new IntSerializer();
    DataOutputSerializer outputSerializer = new DataOutputSerializer(intSerializer.getLength());
    return IntStream.range(-10, 10).mapToObj(idx -> {
        try {
            intSerializer.serialize(idx, outputSerializer);
            byte[] copyOfBuffer = outputSerializer.getCopyOfBuffer();
            outputSerializer.clear();
            return Tuple2.of(copyOfBuffer, new StreamRecord<>(idx, idx));
        } catch (IOException e) {
            throw new AssertionError(e);
        }
    }).toArray(Tuple2[]::new);
}