Examples with SerializationTestType org.apache.flink.runtime.io.network.api.serialization.types.SerializationTestType used on opensource projects

Example 1 with SerializationTestType

use of org.apache.flink.runtime.io.network.api.serialization.types.SerializationTestType in project flink by apache.

the class LargeRecordsTest method testHandleMixedLargeRecords.

@Test
public void testHandleMixedLargeRecords() {
    try {
        final int NUM_RECORDS = 99;
        final int SEGMENT_SIZE = 32 * 1024;
        final RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
        final RecordDeserializer<SerializationTestType> deserializer = new AdaptiveSpanningRecordDeserializer<SerializationTestType>();
        final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(SEGMENT_SIZE), mock(BufferRecycler.class));
        List<SerializationTestType> originalRecords = new ArrayList<SerializationTestType>((NUM_RECORDS + 1) / 2);
        List<SerializationTestType> deserializedRecords = new ArrayList<SerializationTestType>((NUM_RECORDS + 1) / 2);
        LargeObjectType genLarge = new LargeObjectType();
        Random rnd = new Random();
        for (int i = 0; i < NUM_RECORDS; i++) {
            if (i % 2 == 0) {
                originalRecords.add(new IntType(42));
                deserializedRecords.add(new IntType());
            } else {
                originalRecords.add(genLarge.getRandom(rnd));
                deserializedRecords.add(new LargeObjectType());
            }
        }
        // -------------------------------------------------------------------------------------------------------------
        serializer.setNextBuffer(buffer);
        int numRecordsDeserialized = 0;
        for (SerializationTestType record : originalRecords) {
            // serialize record
            if (serializer.addRecord(record).isFullBuffer()) {
                // buffer is full => move to deserializer
                deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), SEGMENT_SIZE);
                // deserialize records, as many complete as there are
                while (numRecordsDeserialized < deserializedRecords.size()) {
                    SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
                    if (deserializer.getNextRecord(next).isFullRecord()) {
                        assertEquals(originalRecords.get(numRecordsDeserialized), next);
                        numRecordsDeserialized++;
                    } else {
                        break;
                    }
                }
                // move buffers as long as necessary (for long records)
                while (serializer.setNextBuffer(buffer).isFullBuffer()) {
                    deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), SEGMENT_SIZE);
                }
                // deserialize records, as many as there are in the last buffer
                while (numRecordsDeserialized < deserializedRecords.size()) {
                    SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
                    if (deserializer.getNextRecord(next).isFullRecord()) {
                        assertEquals(originalRecords.get(numRecordsDeserialized), next);
                        numRecordsDeserialized++;
                    } else {
                        break;
                    }
                }
            }
        }
        // move the last (incomplete buffer)
        Buffer last = serializer.getCurrentBuffer();
        deserializer.setNextMemorySegment(last.getMemorySegment(), last.getSize());
        serializer.clear();
        // deserialize records, as many as there are in the last buffer
        while (numRecordsDeserialized < deserializedRecords.size()) {
            SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
            assertTrue(deserializer.getNextRecord(next).isFullRecord());
            assertEquals(originalRecords.get(numRecordsDeserialized), next);
            numRecordsDeserialized++;
        }
        // might be that the last big records has not yet been fully moved, and a small one is missing
        assertFalse(serializer.hasData());
        assertFalse(deserializer.hasUnfinishedData());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with SerializationTestType

use of org.apache.flink.runtime.io.network.api.serialization.types.SerializationTestType in project flink by apache.

the class PagedViewsTest method testSequenceOfTypes.

private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {
    List<SerializationTestType> elements = new ArrayList<SerializationTestType>(512);
    TestOutputView outView = new TestOutputView(segmentSize);
    // write
    for (SerializationTestType type : sequence) {
        // serialize the record
        type.write(outView);
        elements.add(type);
    }
    outView.close();
    // check the records
    TestInputView inView = new TestInputView(outView.segments);
    for (SerializationTestType reference : elements) {
        SerializationTestType result = reference.getClass().newInstance();
        result.read(inView);
        assertEquals(reference, result);
    }
}

Example 3 with SerializationTestType

use of org.apache.flink.runtime.io.network.api.serialization.types.SerializationTestType in project flink by apache.

the class SpanningRecordSerializationTest method test.

/**
	 * Iterates over the provided records and tests whether {@link SpanningRecordSerializer} and {@link AdaptiveSpanningRecordDeserializer}
	 * interact as expected.
	 * <p>
	 * Only a single {@link MemorySegment} will be allocated.
	 *
	 * @param records records to test
	 * @param segmentSize size for the {@link MemorySegment}
	 */
private void test(Util.MockRecords records, int segmentSize, RecordSerializer<SerializationTestType> serializer, RecordDeserializer<SerializationTestType> deserializer) throws Exception {
    // length encoding
    final int SERIALIZATION_OVERHEAD = 4;
    final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(segmentSize), mock(BufferRecycler.class));
    final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<SerializationTestType>();
    // -------------------------------------------------------------------------------------------------------------
    serializer.setNextBuffer(buffer);
    int numBytes = 0;
    int numRecords = 0;
    for (SerializationTestType record : records) {
        serializedRecords.add(record);
        numRecords++;
        numBytes += record.length() + SERIALIZATION_OVERHEAD;
        // serialize record
        if (serializer.addRecord(record).isFullBuffer()) {
            // buffer is full => start deserializing
            deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), segmentSize);
            while (!serializedRecords.isEmpty()) {
                SerializationTestType expected = serializedRecords.poll();
                SerializationTestType actual = expected.getClass().newInstance();
                if (deserializer.getNextRecord(actual).isFullRecord()) {
                    Assert.assertEquals(expected, actual);
                    numRecords--;
                } else {
                    serializedRecords.addFirst(expected);
                    break;
                }
            }
            while (serializer.setNextBuffer(buffer).isFullBuffer()) {
                deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), segmentSize);
            }
        }
    }
    // deserialize left over records
    deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), (numBytes % segmentSize));
    serializer.clear();
    while (!serializedRecords.isEmpty()) {
        SerializationTestType expected = serializedRecords.poll();
        SerializationTestType actual = expected.getClass().newInstance();
        RecordDeserializer.DeserializationResult result = deserializer.getNextRecord(actual);
        Assert.assertTrue(result.isFullRecord());
        Assert.assertEquals(expected, actual);
        numRecords--;
    }
    // assert that all records have been serialized and deserialized
    Assert.assertEquals(0, numRecords);
    Assert.assertFalse(serializer.hasData());
    Assert.assertFalse(deserializer.hasUnfinishedData());
}

Example 4 with SerializationTestType

use of org.apache.flink.runtime.io.network.api.serialization.types.SerializationTestType in project flink by apache.

the class SpanningRecordSerializerTest method test.

// -----------------------------------------------------------------------------------------------------------------
/**
	 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
	 * {@link RecordSerializer.SerializationResult} values.
	 * <p>
	 * Only a single {@link MemorySegment} will be allocated.
	 *
	 * @param records records to test
	 * @param segmentSize size for the {@link MemorySegment}
	 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
    // length encoding
    final int SERIALIZATION_OVERHEAD = 4;
    final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
    final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(segmentSize), mock(BufferRecycler.class));
    // -------------------------------------------------------------------------------------------------------------
    serializer.setNextBuffer(buffer);
    int numBytes = 0;
    for (SerializationTestType record : records) {
        RecordSerializer.SerializationResult result = serializer.addRecord(record);
        numBytes += record.length() + SERIALIZATION_OVERHEAD;
        if (numBytes < segmentSize) {
            Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
        } else if (numBytes == segmentSize) {
            Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
            serializer.setNextBuffer(buffer);
            numBytes = 0;
        } else {
            Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);
            while (result.isFullBuffer()) {
                numBytes -= segmentSize;
                result = serializer.setNextBuffer(buffer);
            }
        }
    }
}

Example 5 with SerializationTestType

use of org.apache.flink.runtime.io.network.api.serialization.types.SerializationTestType in project flink by apache.

the class DataInputOutputSerializerTest method testWrapAsByteBuffer.

@Test
public void testWrapAsByteBuffer() {
    SerializationTestType randomInt = Util.randomRecord(SerializationTestTypeFactory.INT);
    DataOutputSerializer serializer = new DataOutputSerializer(randomInt.length());
    MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(randomInt.length());
    try {
        // empty buffer, read buffer should be empty
        ByteBuffer wrapper = serializer.wrapAsByteBuffer();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(0, wrapper.limit());
        // write to data output, read buffer should still be empty
        randomInt.write(serializer);
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(0, wrapper.limit());
        // get updated read buffer, read buffer should contain written data
        wrapper = serializer.wrapAsByteBuffer();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(randomInt.length(), wrapper.limit());
        // clear data output, read buffer should still contain written data
        serializer.clear();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(randomInt.length(), wrapper.limit());
        // get updated read buffer, should be empty
        wrapper = serializer.wrapAsByteBuffer();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(0, wrapper.limit());
        // write to data output and read back to memory
        randomInt.write(serializer);
        wrapper = serializer.wrapAsByteBuffer();
        segment.put(0, wrapper, randomInt.length());
        Assert.assertEquals(randomInt.length(), wrapper.position());
        Assert.assertEquals(randomInt.length(), wrapper.limit());
    } catch (IOException e) {
        e.printStackTrace();
        Assert.fail("Test encountered an unexpected exception.");
    }
}