Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 91 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class NettyMessageSerializationTest method testEncodeDecode.

@Test
public void testEncodeDecode() {
    {
        Buffer buffer = spy(new Buffer(MemorySegmentFactory.allocateUnpooledSegment(1024), mock(BufferRecycler.class)));
        ByteBuffer nioBuffer = buffer.getNioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            nioBuffer.putInt(i);
        }
        NettyMessage.BufferResponse expected = new NettyMessage.BufferResponse(buffer, random.nextInt(), new InputChannelID());
        NettyMessage.BufferResponse actual = encodeAndDecode(expected);
        // Verify recycle has been called on buffer instance
        verify(buffer, times(1)).recycle();
        final ByteBuf retainedSlice = actual.getNettyBuffer();
        // Ensure not recycled and same size as original buffer
        assertEquals(1, retainedSlice.refCnt());
        assertEquals(1024, retainedSlice.readableBytes());
        nioBuffer = retainedSlice.nioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            assertEquals(i, nioBuffer.getInt());
        }
        // Release the retained slice
        actual.releaseBuffer();
        assertEquals(0, retainedSlice.refCnt());
        assertEquals(expected.sequenceNumber, actual.sequenceNumber);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        {
            IllegalStateException expectedError = new IllegalStateException();
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertNull(actual.receiverId);
            assertTrue(actual.isFatalError());
        }
    }
    {
        NettyMessage.PartitionRequest expected = new NettyMessage.PartitionRequest(new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), random.nextInt(), new InputChannelID());
        NettyMessage.PartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.queueIndex, actual.queueIndex);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.TaskEventRequest expected = new NettyMessage.TaskEventRequest(new IntegerTaskEvent(random.nextInt()), new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), new InputChannelID());
        NettyMessage.TaskEventRequest actual = encodeAndDecode(expected);
        assertEquals(expected.event, actual.event);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CancelPartitionRequest expected = new NettyMessage.CancelPartitionRequest(new InputChannelID());
        NettyMessage.CancelPartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CloseRequest expected = new NettyMessage.CloseRequest();
        NettyMessage.CloseRequest actual = encodeAndDecode(expected);
        assertEquals(expected.getClass(), actual.getClass());
    }
}

Example 92 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PartitionRequestClientHandlerTest method testAutoReadAfterUnsuccessfulStagedMessage.

/**
	 * Tests that an unsuccessful message decode call for a staged message
	 * does not leave the channel with auto read set to false.
	 */
@Test
@SuppressWarnings("unchecked")
public void testAutoReadAfterUnsuccessfulStagedMessage() throws Exception {
    PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    final AtomicReference<EventListener<Buffer>> listener = new AtomicReference<>();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.addListener(any(EventListener.class))).thenAnswer(new Answer<Boolean>() {

        @Override
        @SuppressWarnings("unchecked")
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            listener.set((EventListener<Buffer>) invocation.getArguments()[0]);
            return true;
        }
    });
    when(bufferProvider.requestBuffer()).thenReturn(null);
    InputChannelID channelId = new InputChannelID(0, 0);
    RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(channelId);
    // The 3rd staged msg has a null buffer provider
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider, bufferProvider, null);
    handler.addInputChannel(inputChannel);
    BufferResponse msg = createBufferResponse(createBuffer(true), 0, channelId);
    // Write 1st buffer msg. No buffer is available, therefore the buffer
    // should be staged and auto read should be set to false.
    assertTrue(channel.config().isAutoRead());
    channel.writeInbound(msg);
    // No buffer available, auto read false
    assertFalse(channel.config().isAutoRead());
    // Write more buffers... all staged.
    msg = createBufferResponse(createBuffer(true), 1, channelId);
    channel.writeInbound(msg);
    msg = createBufferResponse(createBuffer(true), 2, channelId);
    channel.writeInbound(msg);
    // Notify about buffer => handle 1st msg
    Buffer availableBuffer = createBuffer(false);
    listener.get().onEvent(availableBuffer);
    // Start processing of staged buffers (in run pending tasks). Make
    // sure that the buffer provider acts like it's destroyed.
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    // Execute all tasks that are scheduled in the event loop. Further
    // eventLoop().execute() calls are directly executed, if they are
    // called in the scope of this call.
    channel.runPendingTasks();
    assertTrue(channel.config().isAutoRead());
}

Example 93 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpanningRecordSerializerTest method testHasData.

@Test
public void testHasData() {
    final int SEGMENT_SIZE = 16;
    final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
    final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(SEGMENT_SIZE), mock(BufferRecycler.class));
    final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);
    Assert.assertFalse(serializer.hasData());
    try {
        serializer.addRecord(randomIntRecord);
        Assert.assertTrue(serializer.hasData());
        serializer.setNextBuffer(buffer);
        Assert.assertTrue(serializer.hasData());
        serializer.clear();
        Assert.assertFalse(serializer.hasData());
        serializer.setNextBuffer(buffer);
        serializer.addRecord(randomIntRecord);
        Assert.assertTrue(serializer.hasData());
        serializer.addRecord(randomIntRecord);
        Assert.assertTrue(serializer.hasData());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 94 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpanningRecordSerializerTest method testEmptyRecords.

@Test
public void testEmptyRecords() {
    final int SEGMENT_SIZE = 11;
    final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
    final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(SEGMENT_SIZE), mock(BufferRecycler.class));
    try {
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, serializer.setNextBuffer(buffer));
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        SerializationTestType emptyRecord = new SerializationTestType() {

            @Override
            public SerializationTestType getRandom(Random rnd) {
                throw new UnsupportedOperationException();
            }

            @Override
            public int length() {
                throw new UnsupportedOperationException();
            }

            @Override
            public void write(DataOutputView out) {
            }

            @Override
            public void read(DataInputView in) {
            }

            @Override
            public int hashCode() {
                throw new UnsupportedOperationException();
            }

            @Override
            public boolean equals(Object obj) {
                throw new UnsupportedOperationException();
            }
        };
        RecordSerializer.SerializationResult result = serializer.addRecord(emptyRecord);
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
        result = serializer.addRecord(emptyRecord);
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
        result = serializer.addRecord(emptyRecord);
        Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);
        result = serializer.setNextBuffer(buffer);
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 95 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class EventSerializerTest method testIsEventPeakOnly.

/**
	 * Tests {@link EventSerializer#isEvent(Buffer, Class, ClassLoader)}
	 * whether it peaks into the buffer only, i.e. after the call, the buffer
	 * is still de-serializable.
	 *
	 * @throws Exception
	 */
@Test
public void testIsEventPeakOnly() throws Exception {
    final Buffer serializedEvent = EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE);
    try {
        final ClassLoader cl = getClass().getClassLoader();
        assertTrue(EventSerializer.isEvent(serializedEvent, EndOfPartitionEvent.class, cl));
        EndOfPartitionEvent event = (EndOfPartitionEvent) EventSerializer.fromBuffer(serializedEvent, cl);
        assertEquals(EndOfPartitionEvent.INSTANCE, event);
    } finally {
        serializedEvent.recycle();
    }
}