Examples with AdaptiveSpanningRecordDeserializer org.apache.flink.runtime.io.network.api.serialization.AdaptiveSpanningRecordDeserializer used on opensource projects

Example 1 with AdaptiveSpanningRecordDeserializer

use of org.apache.flink.runtime.io.network.api.serialization.AdaptiveSpanningRecordDeserializer in project flink by apache.

the class MockEnvironment method addOutput.

public void addOutput(final List<Record> outputList) {
    try {
        // The record-oriented writers wrap the buffer writer. We mock it
        // to collect the returned buffers and deserialize the content to
        // the output list
        BufferProvider mockBufferProvider = mock(BufferProvider.class);
        when(mockBufferProvider.requestBufferBlocking()).thenAnswer(new Answer<Buffer>() {

            @Override
            public Buffer answer(InvocationOnMock invocationOnMock) throws Throwable {
                return new Buffer(MemorySegmentFactory.allocateUnpooledSegment(bufferSize), mock(BufferRecycler.class));
            }
        });
        ResultPartitionWriter mockWriter = mock(ResultPartitionWriter.class);
        when(mockWriter.getNumberOfOutputChannels()).thenReturn(1);
        when(mockWriter.getBufferProvider()).thenReturn(mockBufferProvider);
        final Record record = new Record();
        final RecordDeserializer<Record> deserializer = new AdaptiveSpanningRecordDeserializer<Record>();
        // Add records from the buffer to the output list
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
                Buffer buffer = (Buffer) invocationOnMock.getArguments()[0];
                deserializer.setNextBuffer(buffer);
                while (deserializer.hasUnfinishedData()) {
                    RecordDeserializer.DeserializationResult result = deserializer.getNextRecord(record);
                    if (result.isFullRecord()) {
                        outputList.add(record.createCopy());
                    }
                    if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER || result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
                        break;
                    }
                }
                return null;
            }
        }).when(mockWriter).writeBuffer(any(Buffer.class), anyInt());
        outputs.add(mockWriter);
    } catch (Throwable t) {
        t.printStackTrace();
        fail(t.getMessage());
    }
}

Example 2 with AdaptiveSpanningRecordDeserializer

use of org.apache.flink.runtime.io.network.api.serialization.AdaptiveSpanningRecordDeserializer in project flink by apache.

the class LargeRecordsTest method testHandleMixedLargeRecords.

@Test
public void testHandleMixedLargeRecords() {
    try {
        final int NUM_RECORDS = 99;
        final int SEGMENT_SIZE = 32 * 1024;
        final RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
        final RecordDeserializer<SerializationTestType> deserializer = new AdaptiveSpanningRecordDeserializer<SerializationTestType>();
        final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(SEGMENT_SIZE), mock(BufferRecycler.class));
        List<SerializationTestType> originalRecords = new ArrayList<SerializationTestType>((NUM_RECORDS + 1) / 2);
        List<SerializationTestType> deserializedRecords = new ArrayList<SerializationTestType>((NUM_RECORDS + 1) / 2);
        LargeObjectType genLarge = new LargeObjectType();
        Random rnd = new Random();
        for (int i = 0; i < NUM_RECORDS; i++) {
            if (i % 2 == 0) {
                originalRecords.add(new IntType(42));
                deserializedRecords.add(new IntType());
            } else {
                originalRecords.add(genLarge.getRandom(rnd));
                deserializedRecords.add(new LargeObjectType());
            }
        }
        // -------------------------------------------------------------------------------------------------------------
        serializer.setNextBuffer(buffer);
        int numRecordsDeserialized = 0;
        for (SerializationTestType record : originalRecords) {
            // serialize record
            if (serializer.addRecord(record).isFullBuffer()) {
                // buffer is full => move to deserializer
                deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), SEGMENT_SIZE);
                // deserialize records, as many complete as there are
                while (numRecordsDeserialized < deserializedRecords.size()) {
                    SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
                    if (deserializer.getNextRecord(next).isFullRecord()) {
                        assertEquals(originalRecords.get(numRecordsDeserialized), next);
                        numRecordsDeserialized++;
                    } else {
                        break;
                    }
                }
                // move buffers as long as necessary (for long records)
                while (serializer.setNextBuffer(buffer).isFullBuffer()) {
                    deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), SEGMENT_SIZE);
                }
                // deserialize records, as many as there are in the last buffer
                while (numRecordsDeserialized < deserializedRecords.size()) {
                    SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
                    if (deserializer.getNextRecord(next).isFullRecord()) {
                        assertEquals(originalRecords.get(numRecordsDeserialized), next);
                        numRecordsDeserialized++;
                    } else {
                        break;
                    }
                }
            }
        }
        // move the last (incomplete buffer)
        Buffer last = serializer.getCurrentBuffer();
        deserializer.setNextMemorySegment(last.getMemorySegment(), last.getSize());
        serializer.clear();
        // deserialize records, as many as there are in the last buffer
        while (numRecordsDeserialized < deserializedRecords.size()) {
            SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
            assertTrue(deserializer.getNextRecord(next).isFullRecord());
            assertEquals(originalRecords.get(numRecordsDeserialized), next);
            numRecordsDeserialized++;
        }
        // might be that the last big records has not yet been fully moved, and a small one is missing
        assertFalse(serializer.hasData());
        assertFalse(deserializer.hasUnfinishedData());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with AdaptiveSpanningRecordDeserializer

use of org.apache.flink.runtime.io.network.api.serialization.AdaptiveSpanningRecordDeserializer in project flink by apache.

the class StreamMockEnvironment method addOutput.

public <T> void addOutput(final Queue<Object> outputList, final TypeSerializer<T> serializer) {
    try {
        // The record-oriented writers wrap the buffer writer. We mock it
        // to collect the returned buffers and deserialize the content to
        // the output list
        BufferProvider mockBufferProvider = mock(BufferProvider.class);
        when(mockBufferProvider.requestBufferBlocking()).thenAnswer(new Answer<Buffer>() {

            @Override
            public Buffer answer(InvocationOnMock invocationOnMock) throws Throwable {
                return new Buffer(MemorySegmentFactory.allocateUnpooledSegment(bufferSize), mock(BufferRecycler.class));
            }
        });
        ResultPartitionWriter mockWriter = mock(ResultPartitionWriter.class);
        when(mockWriter.getNumberOfOutputChannels()).thenReturn(1);
        when(mockWriter.getBufferProvider()).thenReturn(mockBufferProvider);
        final RecordDeserializer<DeserializationDelegate<T>> recordDeserializer = new AdaptiveSpanningRecordDeserializer<DeserializationDelegate<T>>();
        final NonReusingDeserializationDelegate<T> delegate = new NonReusingDeserializationDelegate<T>(serializer);
        // Add records and events from the buffer to the output list
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
                Buffer buffer = (Buffer) invocationOnMock.getArguments()[0];
                addBufferToOutputList(recordDeserializer, delegate, buffer, outputList);
                return null;
            }
        }).when(mockWriter).writeBuffer(any(Buffer.class), anyInt());
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
                Buffer buffer = (Buffer) invocationOnMock.getArguments()[0];
                addBufferToOutputList(recordDeserializer, delegate, buffer, outputList);
                return null;
            }
        }).when(mockWriter).writeBufferToAllChannels(any(Buffer.class));
        outputs.add(mockWriter);
    } catch (Throwable t) {
        t.printStackTrace();
        fail(t.getMessage());
    }
}