Examples with DataOutputSerializer org.apache.flink.core.memory.DataOutputSerializer used on opensource projects

Example 41 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class AvroSerializerSnapshotTest method serialize.

private static <T> ByteBuffer serialize(TypeSerializer<T> serializer, T record) throws IOException {
    DataOutputSerializer out = new DataOutputSerializer(1024);
    serializer.serialize(record, out);
    return out.wrapAsByteBuffer();
}

Example 42 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class AvroSerializerSnapshotTest method roundTrip.

// ---------------------------------------------------------------------------------------------------------------
// Utils
// ---------------------------------------------------------------------------------------------------------------
/**
 * Serialize an (avro)TypeSerializerSnapshot and deserialize it.
 */
private static <T> AvroSerializerSnapshot<T> roundTrip(TypeSerializerSnapshot<T> original) throws IOException {
    // writeSnapshot();
    DataOutputSerializer out = new DataOutputSerializer(1024);
    original.writeSnapshot(out);
    // init
    AvroSerializerSnapshot<T> restored = new AvroSerializerSnapshot<>();
    // readSnapshot();
    DataInputView in = new DataInputDeserializer(out.wrapAsByteBuffer());
    restored.readSnapshot(restored.getCurrentVersion(), in, original.getClass().getClassLoader());
    return restored;
}

Example 43 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class BucketStateSerializer method serialize.

@Override
public byte[] serialize(BucketState<BucketID> state) throws IOException {
    DataOutputSerializer out = new DataOutputSerializer(256);
    out.writeInt(MAGIC_NUMBER);
    serializeV2(state, out);
    return out.getCopyOfBuffer();
}

Example 44 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class CommittableMessageSerializer method serialize.

@Override
public byte[] serialize(CommittableMessage<CommT> obj) throws IOException {
    DataOutputSerializer out = new DataOutputSerializer(256);
    if (obj instanceof CommittableWithLineage) {
        out.writeByte(COMMITTABLE);
        SimpleVersionedSerialization.writeVersionAndSerialize(committableSerializer, ((CommittableWithLineage<CommT>) obj).getCommittable(), out);
        writeCheckpointId(out, obj);
        out.writeInt(obj.getSubtaskId());
    } else if (obj instanceof CommittableSummary) {
        out.writeByte(SUMMARY);
        out.writeInt(obj.getSubtaskId());
        CommittableSummary<?> committableSummary = (CommittableSummary<?>) obj;
        out.writeInt(committableSummary.getNumberOfSubtasks());
        writeCheckpointId(out, obj);
        out.writeInt(committableSummary.getNumberOfCommittables());
        out.writeInt(committableSummary.getNumberOfPendingCommittables());
        out.writeInt(committableSummary.getNumberOfFailedCommittables());
    } else {
        throw new IllegalArgumentException("Unknown message: " + obj.getClass());
    }
    return out.getCopyOfBuffer();
}

Example 45 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class IteratorWrappingTestSingleInputGate method wrapIterator.

private IteratorWrappingTestSingleInputGate<T> wrapIterator(MutableObjectIterator<T> iterator) throws IOException, InterruptedException {
    inputIterator = iterator;
    serializer = new DataOutputSerializer(128);
    // The input iterator can produce an infinite stream. That's why we have to serialize each
    // record on demand and cannot do it upfront.
    final BufferAndAvailabilityProvider answer = new BufferAndAvailabilityProvider() {

        private boolean hasData = inputIterator.next(reuse) != null;

        @Override
        public Optional<BufferAndAvailability> getBufferAvailability() throws IOException {
            if (hasData) {
                ByteBuffer serializedRecord = RecordWriter.serializeRecord(serializer, reuse);
                BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
                BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
                bufferBuilder.appendAndCommit(serializedRecord);
                hasData = inputIterator.next(reuse) != null;
                // Call getCurrentBuffer to ensure size is set
                final Buffer.DataType nextDataType = hasData ? Buffer.DataType.DATA_BUFFER : Buffer.DataType.EVENT_BUFFER;
                return Optional.of(new BufferAndAvailability(bufferConsumer.build(), nextDataType, 0, 0));
            } else {
                inputChannel.setReleased();
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), Buffer.DataType.NONE, 0, 0));
            }
        }
    };
    inputChannel.addBufferAndAvailability(answer);
    inputGate.setInputChannels(inputChannel);
    return this;
}