Examples with PartitionRequest org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest used on opensource projects

Example 1 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class PartitionRequestClient method requestSubpartition.

/**
	 * Requests a remote intermediate result partition queue.
	 * <p>
	 * The request goes to the remote producer, for which this partition
	 * request client instance has been created.
	 */
public ChannelFuture requestSubpartition(final ResultPartitionID partitionId, final int subpartitionIndex, final RemoteInputChannel inputChannel, int delayMs) throws IOException {
    checkNotClosed();
    LOG.debug("Requesting subpartition {} of partition {} with {} ms delay.", subpartitionIndex, partitionId, delayMs);
    partitionRequestHandler.addInputChannel(inputChannel);
    final PartitionRequest request = new PartitionRequest(partitionId, subpartitionIndex, inputChannel.getInputChannelId());
    final ChannelFutureListener listener = new ChannelFutureListener() {

        @Override
        public void operationComplete(ChannelFuture future) throws Exception {
            if (!future.isSuccess()) {
                partitionRequestHandler.removeInputChannel(inputChannel);
                inputChannel.onError(new LocalTransportException("Sending the partition request failed.", future.channel().localAddress(), future.cause()));
            }
        }
    };
    if (delayMs == 0) {
        ChannelFuture f = tcpChannel.writeAndFlush(request);
        f.addListener(listener);
        return f;
    } else {
        final ChannelFuture[] f = new ChannelFuture[1];
        tcpChannel.eventLoop().schedule(new Runnable() {

            @Override
            public void run() {
                f[0] = tcpChannel.writeAndFlush(request);
                f[0].addListener(listener);
            }
        }, delayMs, TimeUnit.MILLISECONDS);
        return f[0];
    }
}

Example 2 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class PartitionRequestServerHandler method channelRead0.

@Override
protected void channelRead0(ChannelHandlerContext ctx, NettyMessage msg) throws Exception {
    try {
        Class<?> msgClazz = msg.getClass();
        // ----------------------------------------------------------------
        if (msgClazz == PartitionRequest.class) {
            PartitionRequest request = (PartitionRequest) msg;
            LOG.debug("Read channel on {}: {}.", ctx.channel().localAddress(), request);
            try {
                NetworkSequenceViewReader reader;
                reader = new CreditBasedSequenceNumberingViewReader(request.receiverId, request.credit, outboundQueue);
                reader.requestSubpartitionView(partitionProvider, request.partitionId, request.queueIndex);
                outboundQueue.notifyReaderCreated(reader);
            } catch (PartitionNotFoundException notFound) {
                respondWithError(ctx, notFound, request.receiverId);
            }
        } else // ----------------------------------------------------------------
        if (msgClazz == TaskEventRequest.class) {
            TaskEventRequest request = (TaskEventRequest) msg;
            if (!taskEventPublisher.publish(request.partitionId, request.event)) {
                respondWithError(ctx, new IllegalArgumentException("Task event receiver not found."), request.receiverId);
            }
        } else if (msgClazz == CancelPartitionRequest.class) {
            CancelPartitionRequest request = (CancelPartitionRequest) msg;
            outboundQueue.cancel(request.receiverId);
        } else if (msgClazz == CloseRequest.class) {
            outboundQueue.close();
        } else if (msgClazz == AddCredit.class) {
            AddCredit request = (AddCredit) msg;
            outboundQueue.addCreditOrResumeConsumption(request.receiverId, reader -> reader.addCredit(request.credit));
        } else if (msgClazz == ResumeConsumption.class) {
            ResumeConsumption request = (ResumeConsumption) msg;
            outboundQueue.addCreditOrResumeConsumption(request.receiverId, NetworkSequenceViewReader::resumeConsumption);
        } else if (msgClazz == AckAllUserRecordsProcessed.class) {
            AckAllUserRecordsProcessed request = (AckAllUserRecordsProcessed) msg;
            outboundQueue.acknowledgeAllRecordsProcessed(request.receiverId);
        } else if (msgClazz == NewBufferSize.class) {
            NewBufferSize request = (NewBufferSize) msg;
            outboundQueue.notifyNewBufferSize(request.receiverId, request.bufferSize);
        } else {
            LOG.warn("Received unexpected client request: {}", msg);
        }
    } catch (Throwable t) {
        respondWithError(ctx, t);
    }
}

Example 3 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class PartitionRequestServerHandlerTest method testResponsePartitionNotFoundException.

/**
 * Tests that {@link PartitionRequestServerHandler} responds {@link ErrorResponse} with wrapped
 * {@link PartitionNotFoundException} after receiving invalid {@link PartitionRequest}.
 */
@Test
public void testResponsePartitionNotFoundException() {
    final PartitionRequestServerHandler serverHandler = new PartitionRequestServerHandler(new ResultPartitionManager(), new TaskEventDispatcher(), new PartitionRequestQueue());
    final EmbeddedChannel channel = new EmbeddedChannel(serverHandler);
    final ResultPartitionID partitionId = new ResultPartitionID();
    // Write the message of partition request to server
    channel.writeInbound(new PartitionRequest(partitionId, 0, new InputChannelID(), 2));
    channel.runPendingTasks();
    // Read the response message after handling partition request
    final Object msg = channel.readOutbound();
    assertThat(msg, instanceOf(ErrorResponse.class));
    final ErrorResponse err = (ErrorResponse) msg;
    assertThat(err.cause, instanceOf(PartitionNotFoundException.class));
    final ResultPartitionID actualPartitionId = ((PartitionNotFoundException) err.cause).getPartitionId();
    assertThat(partitionId, is(actualPartitionId));
}

Example 4 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class CancelPartitionRequestTest method testDuplicateCancel.

@Test
public void testDuplicateCancel() throws Exception {
    NettyServerAndClient serverAndClient = null;
    try {
        final TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
        ResultPartitionManager partitions = mock(ResultPartitionManager.class);
        ResultPartitionID pid = new ResultPartitionID();
        final CountDownLatch sync = new CountDownLatch(1);
        final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
        // Return infinite subpartition
        when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

            @Override
            public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
                BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
                listener.notifyDataAvailable();
                return view;
            }
        });
        NettyProtocol protocol = new NettyProtocol(partitions, mock(TaskEventDispatcher.class));
        serverAndClient = initServerAndClient(protocol);
        Channel ch = connect(serverAndClient);
        // Request for non-existing input channel => results in cancel request
        InputChannelID inputChannelId = new InputChannelID();
        ch.writeAndFlush(new PartitionRequest(pid, 0, inputChannelId, Integer.MAX_VALUE)).await();
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about cancelled partition.");
        }
        ch.writeAndFlush(new CancelPartitionRequest(inputChannelId)).await();
        ch.close();
        NettyTestUtil.awaitClose(ch);
        verify(view, times(1)).releaseAllResources();
    } finally {
        shutdown(serverAndClient);
    }
}

Example 5 with PartitionRequest

use of org.apache.flink.runtime.io.network.netty.NettyMessage.PartitionRequest in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testNotifyCreditAvailable.

/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline for notifying credits,
 * and verifies the behaviour of credit notification by triggering channel's writability
 * changed.
 */
@Test
public void testNotifyCreditAvailable() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final NetworkBufferAllocator allocator = new NetworkBufferAllocator(handler);
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(2, networkBufferPool);
    final RemoteInputChannel[] inputChannels = new RemoteInputChannel[2];
    inputChannels[0] = createRemoteInputChannel(inputGate, client);
    inputChannels[1] = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannels);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannels[0].requestSubpartition();
        inputChannels[1].requestSubpartition();
        // The two input channels should send partition requests
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannels[0].getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannels[1].getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
        // The buffer response will take one available buffer from input channel, and it will
        // trigger
        // requesting (backlog + numExclusiveBuffers - numAvailableBuffers) floating buffers
        final BufferResponse bufferResponse1 = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannels[0].getInputChannelId(), 1, allocator);
        final BufferResponse bufferResponse2 = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannels[1].getInputChannelId(), 1, allocator);
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse1);
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse2);
        assertEquals(2, inputChannels[0].getUnannouncedCredit());
        assertEquals(2, inputChannels[1].getUnannouncedCredit());
        channel.runPendingTasks();
        // The two input channels should notify credits availability via the writable channel
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(AddCredit.class));
        assertEquals(inputChannels[0].getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
        assertEquals(2, ((AddCredit) readFromOutbound).credit);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(AddCredit.class));
        assertEquals(inputChannels[1].getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
        assertEquals(2, ((AddCredit) readFromOutbound).credit);
        assertNull(channel.readOutbound());
        ByteBuf channelBlockingBuffer = blockChannel(channel);
        // Trigger notify credits availability via buffer response on the condition of an
        // un-writable channel
        final BufferResponse bufferResponse3 = createBufferResponse(TestBufferFactory.createBuffer(32), 1, inputChannels[0].getInputChannelId(), 1, allocator);
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse3);
        assertEquals(1, inputChannels[0].getUnannouncedCredit());
        assertEquals(0, inputChannels[1].getUnannouncedCredit());
        channel.runPendingTasks();
        // The input channel will not notify credits via un-writable channel
        assertFalse(channel.isWritable());
        assertNull(channel.readOutbound());
        // Flush the buffer to make the channel writable again
        channel.flush();
        assertSame(channelBlockingBuffer, channel.readOutbound());
        // The input channel should notify credits via channel's writability changed event
        assertTrue(channel.isWritable());
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(AddCredit.class));
        assertEquals(1, ((AddCredit) readFromOutbound).credit);
        assertEquals(0, inputChannels[0].getUnannouncedCredit());
        assertEquals(0, inputChannels[1].getUnannouncedCredit());
        // no more messages
        assertNull(channel.readOutbound());
    } finally {
        releaseResource(inputGate, networkBufferPool);
        channel.close();
    }
}