Examples with SerializationTestType org.apache.flink.testutils.serialization.types.SerializationTestType used on opensource projects

Example 6 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class DataInputOutputSerializerTest method testRandomValuesWriteRead.

@Test
public void testRandomValuesWriteRead() {
    final int numElements = 100000;
    final ArrayDeque<SerializationTestType> reference = new ArrayDeque<>();
    DataOutputSerializer serializer = new DataOutputSerializer(1);
    for (SerializationTestType value : Util.randomRecords(numElements)) {
        reference.add(value);
        try {
            value.write(serializer);
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail("Test encountered an unexpected exception.");
        }
    }
    DataInputDeserializer deserializer = new DataInputDeserializer(serializer.wrapAsByteBuffer());
    for (SerializationTestType expected : reference) {
        try {
            SerializationTestType actual = expected.getClass().newInstance();
            actual.read(deserializer);
            Assert.assertEquals(expected, actual);
        } catch (Exception e) {
            e.printStackTrace();
            Assert.fail("Test encountered an unexpected exception.");
        }
    }
    reference.clear();
}

Example 7 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class DeserializationUtils method deserializeRecords.

/**
 * Iterates over the provided records to deserialize, verifies the results and stats the number
 * of full records.
 *
 * @param records records to be deserialized
 * @param deserializer the record deserializer
 * @return the number of full deserialized records
 */
public static int deserializeRecords(ArrayDeque<SerializationTestType> records, RecordDeserializer<SerializationTestType> deserializer) throws Exception {
    int deserializedRecords = 0;
    while (!records.isEmpty()) {
        SerializationTestType expected = records.poll();
        SerializationTestType actual = expected.getClass().newInstance();
        final DeserializationResult result = deserializer.getNextRecord(actual);
        if (result.isFullRecord()) {
            Assert.assertEquals(expected, actual);
            deserializedRecords++;
        } else {
            records.addFirst(expected);
        }
        if (result.isBufferConsumed()) {
            break;
        }
    }
    return deserializedRecords;
}

Example 8 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class RecordWriterTest method testBroadcastEmitRecord.

/**
 * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastEmitRecord() throws Exception {
    final int numberOfChannels = 4;
    final int bufferSize = 32;
    final int numValues = 8;
    final int serializationLength = 4;
    final ResultPartition partition = createResultPartition(bufferSize, numberOfChannels);
    final RecordWriter<SerializationTestType> writer = createRecordWriter(partition);
    final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(new String[] { tempFolder.getRoot().getAbsolutePath() });
    final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
    final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
    for (SerializationTestType record : records) {
        serializedRecords.add(record);
        writer.broadcastEmit(record);
    }
    final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength));
    if (isBroadcastWriter) {
        assertEquals(numRequiredBuffers, partition.getBufferPool().bestEffortGetNumOfUsedBuffers());
    } else {
        assertEquals(numRequiredBuffers * numberOfChannels, partition.getBufferPool().bestEffortGetNumOfUsedBuffers());
    }
    for (int i = 0; i < numberOfChannels; i++) {
        assertEquals(numRequiredBuffers, partition.getNumberOfQueuedBuffers(i));
        ResultSubpartitionView view = partition.createSubpartitionView(i, new NoOpBufferAvailablityListener());
        verifyDeserializationResults(view, deserializer, serializedRecords.clone(), numRequiredBuffers, numValues);
    }
}

Example 9 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class SpanningRecordSerializationTest method testSerializationRoundTrip.

/**
 * Iterates over the provided records and tests whether {@link RecordWriter#serializeRecord} and
 * {@link RecordDeserializer} interact as expected.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private static void testSerializationRoundTrip(Iterable<SerializationTestType> records, int segmentSize, RecordDeserializer<SerializationTestType> deserializer) throws Exception {
    final DataOutputSerializer serializer = new DataOutputSerializer(128);
    final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
    // -------------------------------------------------------------------------------------------------------------
    BufferAndSerializerResult serializationResult = setNextBufferForSerializer(serializer.wrapAsByteBuffer(), segmentSize);
    int numRecords = 0;
    for (SerializationTestType record : records) {
        serializedRecords.add(record);
        numRecords++;
        // serialize record
        serializer.clear();
        ByteBuffer serializedRecord = RecordWriter.serializeRecord(serializer, record);
        serializationResult.getBufferBuilder().appendAndCommit(serializedRecord);
        if (serializationResult.getBufferBuilder().isFull()) {
            // buffer is full => start deserializing
            deserializer.setNextBuffer(serializationResult.buildBuffer());
            numRecords -= DeserializationUtils.deserializeRecords(serializedRecords, deserializer);
            // move buffers as long as necessary (for long records)
            while ((serializationResult = setNextBufferForSerializer(serializedRecord, segmentSize)).isFullBuffer()) {
                deserializer.setNextBuffer(serializationResult.buildBuffer());
            }
        }
        Assert.assertFalse(serializedRecord.hasRemaining());
    }
    // deserialize left over records
    deserializer.setNextBuffer(serializationResult.buildBuffer());
    while (!serializedRecords.isEmpty()) {
        SerializationTestType expected = serializedRecords.poll();
        SerializationTestType actual = expected.getClass().newInstance();
        RecordDeserializer.DeserializationResult result = deserializer.getNextRecord(actual);
        Assert.assertTrue(result.isFullRecord());
        Assert.assertEquals(expected, actual);
        numRecords--;
    }
    // assert that all records have been serialized and deserialized
    Assert.assertEquals(0, numRecords);
}