Examples with DataOutputSerializer org.apache.flink.core.memory.DataOutputSerializer used on opensource projects

Example 26 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class StreamTestSingleInputGate method setupInputChannels.

private TestInputChannel[] setupInputChannels() {
    TestInputChannel[] inputChannels = new TestInputChannel[numInputChannels];
    for (int i = 0; i < numInputChannels; i++) {
        final int channelIndex = i;
        final DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128);
        final SerializationDelegate<StreamElement> delegate = new SerializationDelegate<>(new StreamElementSerializer<T>(serializer));
        inputQueues[channelIndex] = new ConcurrentLinkedQueue<>();
        inputChannels[channelIndex] = new TestInputChannel(inputGate, i);
        final BufferAndAvailabilityProvider answer = () -> {
            ConcurrentLinkedQueue<InputValue<Object>> inputQueue = inputQueues[channelIndex];
            InputValue<Object> input;
            Buffer.DataType nextType;
            synchronized (inputQueue) {
                input = inputQueue.poll();
                nextType = !inputQueue.isEmpty() ? Buffer.DataType.DATA_BUFFER : Buffer.DataType.NONE;
            }
            if (input != null && input.isStreamEnd()) {
                inputChannels[channelIndex].setReleased();
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), nextType, 0, 0));
            } else if (input != null && input.isDataEnd()) {
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(new EndOfData(StopMode.DRAIN), false), nextType, 0, 0));
            } else if (input != null && input.isStreamRecord()) {
                StreamElement inputElement = input.getStreamRecord();
                delegate.setInstance(inputElement);
                ByteBuffer serializedRecord = RecordWriter.serializeRecord(dataOutputSerializer, delegate);
                BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
                BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
                bufferBuilder.appendAndCommit(serializedRecord);
                bufferBuilder.finish();
                bufferBuilder.close();
                // Call getCurrentBuffer to ensure size is set
                return Optional.of(new BufferAndAvailability(bufferConsumer.build(), nextType, 0, 0));
            } else if (input != null && input.isEvent()) {
                AbstractEvent event = input.getEvent();
                if (event instanceof EndOfPartitionEvent) {
                    inputChannels[channelIndex].setReleased();
                }
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(event, false), nextType, 0, 0));
            } else {
                return Optional.empty();
            }
        };
        inputChannels[channelIndex].addBufferAndAvailability(answer);
    }
    return inputChannels;
}

Example 27 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class TypeSerializerUpgradeTestBase method readAndThenWriteData.

private static <T> DataInputView readAndThenWriteData(DataInputView originalDataInput, TypeSerializer<T> readSerializer, TypeSerializer<T> writeSerializer, Matcher<T> testDataMatcher) throws IOException {
    T data = readSerializer.deserialize(originalDataInput);
    assertThat(data, testDataMatcher);
    DataOutputSerializer out = new DataOutputSerializer(INITIAL_OUTPUT_BUFFER_SIZE);
    writeSerializer.serialize(data, out);
    return new DataInputDeserializer(out.wrapAsByteBuffer());
}

Example 28 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class TypeSerializerUpgradeTestBase method writeAndThenReadSerializerSnapshot.

private static <T> TypeSerializerSnapshot<T> writeAndThenReadSerializerSnapshot(TypeSerializer<T> serializer) throws IOException {
    DataOutputSerializer out = new DataOutputSerializer(INITIAL_OUTPUT_BUFFER_SIZE);
    writeSerializerSnapshotCurrentFormat(out, serializer);
    DataInputDeserializer in = new DataInputDeserializer(out.wrapAsByteBuffer());
    return readSerializerSnapshotCurrentFormat(in, Thread.currentThread().getContextClassLoader());
}

Example 29 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class TypeSerializerSnapshotTest method testSerializeConfigWhenSerializerMissing.

@Test
public void testSerializeConfigWhenSerializerMissing() throws Exception {
    TestSerializer ser = new TestSerializer();
    TypeSerializerConfigSnapshot<Object> snap = (TypeSerializerConfigSnapshot<Object>) ser.snapshotConfiguration();
    try {
        TypeSerializerSnapshot.writeVersionedSnapshot(new DataOutputSerializer(64), snap);
        fail("exception expected");
    } catch (IllegalStateException e) {
    // expected
    }
}

Example 30 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class KvStateSerializer method serializeKeyAndNamespace.

// ------------------------------------------------------------------------
// Generic serialization utils
// ------------------------------------------------------------------------
/**
 * Serializes the key and namespace into a {@link ByteBuffer}.
 *
 * <p>The serialized format matches the RocksDB state backend key format, i.e. the key and
 * namespace don't have to be deserialized for RocksDB lookups.
 *
 * @param key Key to serialize
 * @param keySerializer Serializer for the key
 * @param namespace Namespace to serialize
 * @param namespaceSerializer Serializer for the namespace
 * @param <K> Key type
 * @param <N> Namespace type
 * @return Buffer holding the serialized key and namespace
 * @throws IOException Serialization errors are forwarded
 */
public static <K, N> byte[] serializeKeyAndNamespace(K key, TypeSerializer<K> keySerializer, N namespace, TypeSerializer<N> namespaceSerializer) throws IOException {
    DataOutputSerializer dos = new DataOutputSerializer(32);
    keySerializer.serialize(key, dos);
    dos.writeByte(MAGIC_NUMBER);
    namespaceSerializer.serialize(namespace, dos);
    return dos.getCopyOfBuffer();
}