Examples with BufferAvailabilityListener org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener used on opensource projects

Example 1 with BufferAvailabilityListener

use of org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener in project flink by apache.

the class PartitionRequestQueueTest method testNotifyNewBufferSize.

@Test
public void testNotifyNewBufferSize() throws Exception {
    // given: Result partition and the reader for subpartition 0.
    ResultPartition parent = createResultPartition();
    BufferAvailabilityListener bufferAvailabilityListener = new NoOpBufferAvailablityListener();
    ResultSubpartitionView view = parent.createSubpartitionView(0, bufferAvailabilityListener);
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    InputChannelID receiverId = new InputChannelID();
    PartitionRequestQueue queue = new PartitionRequestQueue();
    CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    // when: New buffer size received.
    queue.notifyNewBufferSize(receiverId, 65);
    // and: New records emit.
    parent.emitRecord(ByteBuffer.allocate(128), 0);
    parent.emitRecord(ByteBuffer.allocate(10), 0);
    parent.emitRecord(ByteBuffer.allocate(60), 0);
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    // then: Buffers of received size will be in outbound channel.
    Object data1 = channel.readOutbound();
    // The size can not be less than the first record in buffer.
    assertEquals(128, ((NettyMessage.BufferResponse) data1).buffer.getSize());
    Object data2 = channel.readOutbound();
    // The size should shrink up to notified buffer size.
    assertEquals(65, ((NettyMessage.BufferResponse) data2).buffer.getSize());
}

Example 2 with BufferAvailabilityListener

use of org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener in project flink by apache.

the class CancelPartitionRequestTest method testDuplicateCancel.

@Test
public void testDuplicateCancel() throws Exception {
    NettyServerAndClient serverAndClient = null;
    try {
        final TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
        ResultPartitionManager partitions = mock(ResultPartitionManager.class);
        ResultPartitionID pid = new ResultPartitionID();
        final CountDownLatch sync = new CountDownLatch(1);
        final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
        // Return infinite subpartition
        when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

            @Override
            public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
                BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
                listener.notifyDataAvailable();
                return view;
            }
        });
        NettyProtocol protocol = new NettyProtocol(partitions, mock(TaskEventDispatcher.class));
        serverAndClient = initServerAndClient(protocol);
        Channel ch = connect(serverAndClient);
        // Request for non-existing input channel => results in cancel request
        InputChannelID inputChannelId = new InputChannelID();
        ch.writeAndFlush(new PartitionRequest(pid, 0, inputChannelId, Integer.MAX_VALUE)).await();
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about cancelled partition.");
        }
        ch.writeAndFlush(new CancelPartitionRequest(inputChannelId)).await();
        ch.close();
        NettyTestUtil.awaitClose(ch);
        verify(view, times(1)).releaseAllResources();
    } finally {
        shutdown(serverAndClient);
    }
}

Example 3 with BufferAvailabilityListener

use of org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener in project flink by apache.

the class CancelPartitionRequestTest method testCancelPartitionRequest.

/**
 * Verifies that requests for non-existing (failed/cancelled) input channels are properly
 * cancelled. The receiver receives data, but there is no input channel to receive the data.
 * This should cancel the request.
 */
@Test
public void testCancelPartitionRequest() throws Exception {
    NettyServerAndClient serverAndClient = null;
    try {
        TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
        ResultPartitionManager partitions = mock(ResultPartitionManager.class);
        ResultPartitionID pid = new ResultPartitionID();
        CountDownLatch sync = new CountDownLatch(1);
        final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
        // Return infinite subpartition
        when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

            @Override
            public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
                BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
                listener.notifyDataAvailable();
                return view;
            }
        });
        NettyProtocol protocol = new NettyProtocol(partitions, mock(TaskEventDispatcher.class));
        serverAndClient = initServerAndClient(protocol);
        Channel ch = connect(serverAndClient);
        // Request for non-existing input channel => results in cancel request
        ch.writeAndFlush(new PartitionRequest(pid, 0, new InputChannelID(), Integer.MAX_VALUE)).await();
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about cancelled partition.");
        }
        verify(view, times(1)).releaseAllResources();
    } finally {
        shutdown(serverAndClient);
    }
}

Example 4 with BufferAvailabilityListener

use of org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener in project flink by apache.

the class PartitionRequestQueueTest method testEnqueueReaderByResumingConsumption.

/**
 * Tests {@link PartitionRequestQueue#enqueueAvailableReader(NetworkSequenceViewReader)},
 * verifying the reader would be enqueued in the pipeline after resuming data consumption if
 * there are credit and data available.
 */
@Test
public void testEnqueueReaderByResumingConsumption() throws Exception {
    PipelinedSubpartition subpartition = PipelinedSubpartitionTest.createPipelinedSubpartition();
    Buffer.DataType dataType1 = Buffer.DataType.ALIGNED_CHECKPOINT_BARRIER;
    Buffer.DataType dataType2 = Buffer.DataType.DATA_BUFFER;
    subpartition.add(createEventBufferConsumer(4096, dataType1));
    subpartition.add(createEventBufferConsumer(4096, dataType2));
    BufferAvailabilityListener bufferAvailabilityListener = new NoOpBufferAvailablityListener();
    PipelinedSubpartitionView view = subpartition.createReadView(bufferAvailabilityListener);
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    InputChannelID receiverId = new InputChannelID();
    PartitionRequestQueue queue = new PartitionRequestQueue();
    CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    assertTrue(reader.getAvailabilityAndBacklog().isAvailable());
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    assertFalse(reader.getAvailabilityAndBacklog().isAvailable());
    assertEquals(1, subpartition.unsynchronizedGetNumberOfQueuedBuffers());
    queue.addCreditOrResumeConsumption(receiverId, NetworkSequenceViewReader::resumeConsumption);
    assertFalse(reader.getAvailabilityAndBacklog().isAvailable());
    assertEquals(0, subpartition.unsynchronizedGetNumberOfQueuedBuffers());
    Object data1 = channel.readOutbound();
    assertEquals(dataType1, ((NettyMessage.BufferResponse) data1).buffer.getDataType());
    Object data2 = channel.readOutbound();
    assertEquals(dataType2, ((NettyMessage.BufferResponse) data2).buffer.getDataType());
}

Example 5 with BufferAvailabilityListener

use of org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener in project flink by apache.

the class ServerTransportErrorHandlingTest method testRemoteClose.

/**
 * Verifies remote closes trigger the release of all resources.
 */
@Test
public void testRemoteClose() throws Exception {
    final TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
    final CountDownLatch sync = new CountDownLatch(1);
    final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

        @Override
        public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
            BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
            listener.notifyDataAvailable();
            return new CancelPartitionRequestTest.InfiniteSubpartitionView(outboundBuffers, sync);
        }
    });
    NettyProtocol protocol = new NettyProtocol(partitionManager, mock(TaskEventDispatcher.class)) {

        @Override
        public ChannelHandler[] getClientChannelHandlers() {
            return new ChannelHandler[] { new NettyMessage.NettyMessageEncoder(), // Close on read
            new ChannelInboundHandlerAdapter() {

                @Override
                public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                    ctx.channel().close();
                }
            } };
        }
    };
    NettyTestUtil.NettyServerAndClient serverAndClient = null;
    try {
        for (int retry = 0; retry < NETTY_INIT_MAX_RETRY_TIMES; retry++) {
            try {
                serverAndClient = initServerAndClient(protocol, createConfig());
                break;
            } catch (Exception e) {
                if (retry >= NETTY_INIT_MAX_RETRY_TIMES - 1) {
                    throw new RuntimeException("Failed to initialize netty server and client, retried " + retry + " times.", e);
                }
                if (e instanceof BindException || ExceptionUtils.findThrowableWithMessage(e, "Address already in use").isPresent()) {
                    continue;
                }
                throw e;
            }
        }
        Channel ch = connect(serverAndClient);
        // Write something to trigger close by server
        ch.writeAndFlush(new NettyMessage.PartitionRequest(new ResultPartitionID(), 0, new InputChannelID(), Integer.MAX_VALUE));
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about released partition.");
        }
    } finally {
        shutdown(serverAndClient);
    }
}