Examples with PartitionRequest org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest used on opensource projects

Example 6 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testNotifyCreditAvailableAfterReleased.

/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline, but {@link AddCredit}
 * message is not sent actually when this input channel is released.
 */
@Test
public void testNotifyCreditAvailableAfterReleased() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // This should send the partition request
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
        // Trigger request floating buffers via buffer response to notify credits available
        final BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), 1, new NetworkBufferAllocator(handler));
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);
        assertEquals(2, inputChannel.getUnannouncedCredit());
        // Release the input channel
        inputGate.close();
        // it should send a close request after releasing the input channel,
        // but will not notify credits for a released input channel.
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(CloseRequest.class));
        channel.runPendingTasks();
        assertNull(channel.readOutbound());
    } finally {
        releaseResource(inputGate, networkBufferPool);
        channel.close();
    }
}

Example 7 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class NettyPartitionRequestClientTest method testDoublePartitionRequest.

@Test
public void testDoublePartitionRequest() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final int numExclusiveBuffers = 2;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // The input channel should only send one partition request
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 8 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class NettyPartitionRequestClientTest method testRetriggerPartitionRequest.

@Test
public void testRetriggerPartitionRequest() throws Exception {
    // 30 secs
    final long deadline = System.currentTimeMillis() + 30_000L;
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final int numExclusiveBuffers = 2;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder().setConnectionManager(mockConnectionManagerWithPartitionRequestClient(client)).setInitialBackoff(1).setMaxBackoff(2).buildRemoteChannel(inputGate);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        // first subpartition request
        inputChannel.requestSubpartition();
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        // retrigger subpartition request, e.g. due to failures
        inputGate.retriggerPartitionRequest(inputChannel.getPartitionId().getPartitionId(), inputChannel.getConsumedSubpartitionIndex());
        runAllScheduledPendingTasks(channel, deadline);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        // retrigger subpartition request once again, e.g. due to failures
        inputGate.retriggerPartitionRequest(inputChannel.getPartitionId().getPartitionId(), inputChannel.getConsumedSubpartitionIndex());
        runAllScheduledPendingTasks(channel, deadline);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 9 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class NettyPartitionRequestClient method requestSubpartition.

/**
 * Requests a remote intermediate result partition queue.
 *
 * <p>The request goes to the remote producer, for which this partition request client instance
 * has been created.
 */
@Override
public void requestSubpartition(final ResultPartitionID partitionId, final int subpartitionIndex, final RemoteInputChannel inputChannel, int delayMs) throws IOException {
    checkNotClosed();
    LOG.debug("Requesting subpartition {} of partition {} with {} ms delay.", subpartitionIndex, partitionId, delayMs);
    clientHandler.addInputChannel(inputChannel);
    final PartitionRequest request = new PartitionRequest(partitionId, subpartitionIndex, inputChannel.getInputChannelId(), inputChannel.getInitialCredit());
    final ChannelFutureListener listener = new ChannelFutureListener() {

        @Override
        public void operationComplete(ChannelFuture future) throws Exception {
            if (!future.isSuccess()) {
                clientHandler.removeInputChannel(inputChannel);
                inputChannel.onError(new LocalTransportException(String.format("Sending the partition request to '%s (#%d)' failed.", connectionId.getAddress(), connectionId.getConnectionIndex()), future.channel().localAddress(), future.cause()));
                sendToChannel(new ConnectionErrorMessage(future.cause() == null ? new RuntimeException("Cannot send partition request.") : future.cause()));
            }
        }
    };
    if (delayMs == 0) {
        ChannelFuture f = tcpChannel.writeAndFlush(request);
        f.addListener(listener);
    } else {
        final ChannelFuture[] f = new ChannelFuture[1];
        tcpChannel.eventLoop().schedule(new Runnable() {

            @Override
            public void run() {
                f[0] = tcpChannel.writeAndFlush(request);
                f[0].addListener(listener);
            }
        }, delayMs, TimeUnit.MILLISECONDS);
    }
}

Example 10 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class CancelPartitionRequestTest method testCancelPartitionRequest.

/**
 * Verifies that requests for non-existing (failed/cancelled) input channels are properly
 * cancelled. The receiver receives data, but there is no input channel to receive the data.
 * This should cancel the request.
 */
@Test
public void testCancelPartitionRequest() throws Exception {
    NettyServerAndClient serverAndClient = null;
    try {
        TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
        ResultPartitionManager partitions = mock(ResultPartitionManager.class);
        ResultPartitionID pid = new ResultPartitionID();
        CountDownLatch sync = new CountDownLatch(1);
        final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
        // Return infinite subpartition
        when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

            @Override
            public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
                BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
                listener.notifyDataAvailable();
                return view;
            }
        });
        NettyProtocol protocol = new NettyProtocol(partitions, mock(TaskEventDispatcher.class));
        serverAndClient = initServerAndClient(protocol);
        Channel ch = connect(serverAndClient);
        // Request for non-existing input channel => results in cancel request
        ch.writeAndFlush(new PartitionRequest(pid, 0, new InputChannelID(), Integer.MAX_VALUE)).await();
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about cancelled partition.");
        }
        verify(view, times(1)).releaseAllResources();
    } finally {
        shutdown(serverAndClient);
    }
}