Examples with RemoteInputChannel org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel used on opensource projects

Example 6 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithRemoteChannels.

@Test
public void testConsumptionWithRemoteChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ConnectionManager connManager = createDummyConnectionManager();
    final Source[] sources = new Source[numChannels];
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        sources[i] = new RemoteChannelSource(channel);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 7 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannels.

@Test
public void testFairConsumptionRemoteChannels() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numChannels];
    final int[] channelSequenceNums = new int[numChannels];
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        channels[i] = channel;
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    channels[11].onBuffer(mockBuffer, 0);
    channelSequenceNums[11]++;
    // read all the buffers and the EOF event
    for (int i = 0; i < numChannels * buffersPerChannel; i++) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
        if (i % (2 * numChannels) == 0) {
            // add three buffers to each channel, in random order
            fillRandom(channels, channelSequenceNums, 3, mockBuffer);
        }
    }
}

Example 8 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannelsPreFilled.

@Test
public void testFairConsumptionRemoteChannelsPreFilled() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numChannels];
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        channels[i] = channel;
        for (int p = 0; p < buffersPerChannel; p++) {
            channel.onBuffer(mockBuffer, p);
        }
        channel.onBuffer(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE), buffersPerChannel);
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    // read all the buffers and the EOF event
    for (int i = numChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
    }
    assertNull(gate.getNextBufferOrEvent());
}

Example 9 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class PartitionRequestClientHandlerTest method testCancelBeforeActive.

@Test
public void testCancelBeforeActive() throws Exception {
    final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
    final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
    client.addInputChannel(inputChannel);
    // Don't throw NPE
    client.cancelRequestFor(null);
    // Don't throw NPE, because channel is not active yet
    client.cancelRequestFor(inputChannel.getInputChannelId());
}

Example 10 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class PartitionRequestClientHandler method decodeMsg.

private boolean decodeMsg(Object msg, boolean isStagedBuffer) throws Throwable {
    final Class<?> msgClazz = msg.getClass();
    // ---- Buffer --------------------------------------------------------
    if (msgClazz == NettyMessage.BufferResponse.class) {
        NettyMessage.BufferResponse bufferOrEvent = (NettyMessage.BufferResponse) msg;
        RemoteInputChannel inputChannel = inputChannels.get(bufferOrEvent.receiverId);
        if (inputChannel == null) {
            bufferOrEvent.releaseBuffer();
            cancelRequestFor(bufferOrEvent.receiverId);
            return true;
        }
        return decodeBufferOrEvent(inputChannel, bufferOrEvent, isStagedBuffer);
    } else // ---- Error ---------------------------------------------------------
    if (msgClazz == NettyMessage.ErrorResponse.class) {
        NettyMessage.ErrorResponse error = (NettyMessage.ErrorResponse) msg;
        SocketAddress remoteAddr = ctx.channel().remoteAddress();
        if (error.isFatalError()) {
            notifyAllChannelsOfErrorAndClose(new RemoteTransportException("Fatal error at remote task manager '" + remoteAddr + "'.", remoteAddr, error.cause));
        } else {
            RemoteInputChannel inputChannel = inputChannels.get(error.receiverId);
            if (inputChannel != null) {
                if (error.cause.getClass() == PartitionNotFoundException.class) {
                    inputChannel.onFailedPartitionRequest();
                } else {
                    inputChannel.onError(new RemoteTransportException("Error at remote task manager '" + remoteAddr + "'.", remoteAddr, error.cause));
                }
            }
        }
    } else {
        throw new IllegalStateException("Received unknown message from producer: " + msg.getClass());
    }
    return true;
}