Examples with SerializationTestType org.apache.flink.testutils.serialization.types.SerializationTestType used on opensource projects

Example 1 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class DataInputOutputSerializerTest method testWrapAsByteBuffer.

@Test
public void testWrapAsByteBuffer() {
    SerializationTestType randomInt = Util.randomRecord(SerializationTestTypeFactory.INT);
    DataOutputSerializer serializer = new DataOutputSerializer(randomInt.length());
    MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(randomInt.length());
    try {
        // empty buffer, read buffer should be empty
        ByteBuffer wrapper = serializer.wrapAsByteBuffer();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(0, wrapper.limit());
        // write to data output, read buffer should still be empty
        randomInt.write(serializer);
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(0, wrapper.limit());
        // get updated read buffer, read buffer should contain written data
        wrapper = serializer.wrapAsByteBuffer();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(randomInt.length(), wrapper.limit());
        // clear data output, read buffer should still contain written data
        serializer.clear();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(randomInt.length(), wrapper.limit());
        // get updated read buffer, should be empty
        wrapper = serializer.wrapAsByteBuffer();
        Assert.assertEquals(0, wrapper.position());
        Assert.assertEquals(0, wrapper.limit());
        // write to data output and read back to memory
        randomInt.write(serializer);
        wrapper = serializer.wrapAsByteBuffer();
        segment.put(0, wrapper, randomInt.length());
        Assert.assertEquals(randomInt.length(), wrapper.position());
        Assert.assertEquals(randomInt.length(), wrapper.limit());
    } catch (IOException e) {
        e.printStackTrace();
        Assert.fail("Test encountered an unexpected exception.");
    }
}

Example 2 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class BroadcastRecordWriterTest method testBroadcastMixedRandomEmitRecord.

/**
 * Tests the number of requested buffers and results are correct in the case of switching modes
 * between {@link BroadcastRecordWriter#broadcastEmit(IOReadableWritable)} and {@link
 * BroadcastRecordWriter#randomEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastMixedRandomEmitRecord() throws Exception {
    final int numberOfChannels = 8;
    final int numberOfRecords = 8;
    final int bufferSize = 32;
    final ResultPartition partition = createResultPartition(bufferSize, numberOfChannels);
    final BroadcastRecordWriter<SerializationTestType> writer = new BroadcastRecordWriter<>(partition, -1, "test");
    final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(new String[] { tempFolder.getRoot().getAbsolutePath() });
    // generate the configured number of int values as global record set
    final Iterable<SerializationTestType> records = Util.randomRecords(numberOfRecords, SerializationTestTypeFactory.INT);
    // restore the corresponding record set for every input channel
    final Map<Integer, ArrayDeque<SerializationTestType>> serializedRecords = new HashMap<>();
    for (int i = 0; i < numberOfChannels; i++) {
        serializedRecords.put(i, new ArrayDeque<>());
    }
    // every record in global set would both emit into one random channel and broadcast to all
    // the channels
    int index = 0;
    for (SerializationTestType record : records) {
        int randomChannel = index++ % numberOfChannels;
        writer.emit(record, randomChannel);
        serializedRecords.get(randomChannel).add(record);
        writer.broadcastEmit(record);
        for (int i = 0; i < numberOfChannels; i++) {
            serializedRecords.get(i).add(record);
        }
    }
    final int numberOfCreatedBuffers = partition.getBufferPool().bestEffortGetNumOfUsedBuffers();
    // verify the expected number of requested buffers, and it would always request a new buffer
    // while random emitting
    assertEquals(2 * numberOfRecords, numberOfCreatedBuffers);
    for (int i = 0; i < numberOfChannels; i++) {
        // every channel would queue the number of above crated buffers
        assertEquals(numberOfRecords + 1, partition.getNumberOfQueuedBuffers(i));
        final int excessRandomRecords = i < numberOfRecords % numberOfChannels ? 1 : 0;
        final int numberOfRandomRecords = numberOfRecords / numberOfChannels + excessRandomRecords;
        final int numberOfTotalRecords = numberOfRecords + numberOfRandomRecords;
        // verify the data correctness in every channel queue
        verifyDeserializationResults(partition.createSubpartitionView(i, new NoOpBufferAvailablityListener()), deserializer, serializedRecords.get(i), numberOfRecords + 1, numberOfTotalRecords);
    }
}

Example 3 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class BroadcastRecordWriterTest method testRandomEmitAndBufferRecycling.

/**
 * FLINK-17780: Tests that a shared buffer(or memory segment) of a buffer builder is only freed
 * when all consumers are closed.
 */
@Test
public void testRandomEmitAndBufferRecycling() throws Exception {
    int recordSize = 8;
    int numberOfChannels = 2;
    ResultPartition partition = createResultPartition(2 * recordSize, numberOfChannels);
    BufferPool bufferPool = partition.getBufferPool();
    BroadcastRecordWriter<SerializationTestType> writer = new BroadcastRecordWriter<>(partition, -1, "test");
    // force materialization of both buffers for easier availability tests
    List<Buffer> buffers = Arrays.asList(bufferPool.requestBuffer(), bufferPool.requestBuffer());
    buffers.forEach(Buffer::recycleBuffer);
    assertEquals(3, bufferPool.getNumberOfAvailableMemorySegments());
    // fill first buffer
    writer.broadcastEmit(new IntType(1));
    writer.broadcastEmit(new IntType(2));
    assertEquals(2, bufferPool.getNumberOfAvailableMemorySegments());
    // simulate consumption of first buffer consumer; this should not free buffers
    assertEquals(1, partition.getNumberOfQueuedBuffers(0));
    ResultSubpartitionView view0 = partition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
    closeConsumer(view0, 2 * recordSize);
    assertEquals(2, bufferPool.getNumberOfAvailableMemorySegments());
    // use second buffer
    writer.emit(new IntType(3), 0);
    assertEquals(1, bufferPool.getNumberOfAvailableMemorySegments());
    // fully free first buffer
    assertEquals(1, partition.getNumberOfQueuedBuffers(1));
    ResultSubpartitionView view1 = partition.createSubpartitionView(1, new NoOpBufferAvailablityListener());
    closeConsumer(view1, 2 * recordSize);
    assertEquals(2, bufferPool.getNumberOfAvailableMemorySegments());
}

Example 4 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class SpanningRecordSerializationTest method testHandleMixedLargeRecords.

@Test
public void testHandleMixedLargeRecords() throws Exception {
    final int numValues = 99;
    final int segmentSize = 32 * 1024;
    List<SerializationTestType> originalRecords = new ArrayList<>((numValues + 1) / 2);
    LargeObjectType genLarge = new LargeObjectType();
    Random rnd = new Random();
    for (int i = 0; i < numValues; i++) {
        if (i % 2 == 0) {
            originalRecords.add(new IntType(42));
        } else {
            originalRecords.add(genLarge.getRandom(rnd));
        }
    }
    testSerializationRoundTrip(originalRecords, segmentSize);
}

Example 5 with SerializationTestType

use of org.apache.flink.testutils.serialization.types.SerializationTestType in project flink by apache.

the class PagedViewsTest method testSequenceOfTypes.

private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {
    List<SerializationTestType> elements = new ArrayList<>(512);
    TestOutputView outView = new TestOutputView(segmentSize);
    // write
    for (SerializationTestType type : sequence) {
        // serialize the record
        type.write(outView);
        elements.add(type);
    }
    outView.close();
    // check the records
    TestInputView inView = new TestInputView(outView.segments);
    for (SerializationTestType reference : elements) {
        SerializationTestType result = reference.getClass().newInstance();
        result.read(inView);
        assertEquals(reference, result);
    }
}