Examples with InputChannel org.apache.flink.runtime.io.network.partition.consumer.InputChannel used on opensource projects

Example 11 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class InputGateFairnessTest method testFairConsumptionLocalChannelsPreFilled.

@Test
public void testFairConsumptionLocalChannelsPreFilled() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
    final BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42);
    // ----- create some source channels and fill them with buffers -----
    final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
    for (final PipelinedSubpartition subpartition : sources) {
        for (int p = 0; p < buffersPerChannel; p++) {
            subpartition.add(bufferConsumer.copy());
        }
        subpartition.finish();
    }
    for (ResultPartition rp : resultPartitions) {
        rp.setup();
    }
    // ----- create reading side -----
    final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
    final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
    setupInputGate(gate, inputChannels);
    // read all the buffers and the EOF event
    for (int i = numberOfChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNext());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (PipelinedSubpartition source : sources) {
            int size = source.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == (min + 1));
    }
    assertFalse(gate.getNext().isPresent());
}

Example 12 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class NettyShuffleEnvironmentTest method testRegisterTaskWithLimitedBuffers.

private void testRegisterTaskWithLimitedBuffers(int bufferPoolSize) throws Exception {
    final NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(bufferPoolSize).build();
    final ConnectionManager connManager = createDummyConnectionManager();
    int channels = 2;
    int rp4Channels = 4;
    int floatingBuffers = network.getConfiguration().floatingNetworkBuffersPerGate();
    int exclusiveBuffers = network.getConfiguration().networkBuffersPerChannel();
    int expectedBuffers = channels * exclusiveBuffers + floatingBuffers;
    int expectedRp4Buffers = rp4Channels * exclusiveBuffers + floatingBuffers;
    // result partitions
    ResultPartition rp1 = createPartition(network, ResultPartitionType.PIPELINED, channels);
    ResultPartition rp2 = createPartition(network, fileChannelManager, ResultPartitionType.BLOCKING, channels);
    ResultPartition rp3 = createPartition(network, ResultPartitionType.PIPELINED_BOUNDED, channels);
    ResultPartition rp4 = createPartition(network, ResultPartitionType.PIPELINED_BOUNDED, rp4Channels);
    final ResultPartition[] resultPartitions = new ResultPartition[] { rp1, rp2, rp3, rp4 };
    // input gates
    SingleInputGate ig1 = createSingleInputGate(network, ResultPartitionType.PIPELINED, channels);
    SingleInputGate ig2 = createSingleInputGate(network, ResultPartitionType.BLOCKING, channels);
    SingleInputGate ig3 = createSingleInputGate(network, ResultPartitionType.PIPELINED_BOUNDED, channels);
    SingleInputGate ig4 = createSingleInputGate(network, ResultPartitionType.PIPELINED_BOUNDED, rp4Channels);
    InputChannel[] ic1 = new InputChannel[channels];
    InputChannel[] ic2 = new InputChannel[channels];
    InputChannel[] ic3 = new InputChannel[channels];
    InputChannel[] ic4 = new InputChannel[rp4Channels];
    final SingleInputGate[] inputGates = new SingleInputGate[] { ig1, ig2, ig3, ig4 };
    ic4[0] = createRemoteInputChannel(ig4, 0, rp1, connManager);
    ic4[1] = createRemoteInputChannel(ig4, 0, rp2, connManager);
    ic4[2] = createRemoteInputChannel(ig4, 0, rp3, connManager);
    ic4[3] = createRemoteInputChannel(ig4, 0, rp4, connManager);
    ig4.setInputChannels(ic4);
    ic1[0] = createRemoteInputChannel(ig1, 1, rp1, connManager);
    ic1[1] = createRemoteInputChannel(ig1, 1, rp4, connManager);
    ig1.setInputChannels(ic1);
    ic2[0] = createRemoteInputChannel(ig2, 1, rp2, connManager);
    ic2[1] = createRemoteInputChannel(ig2, 2, rp4, connManager);
    ig2.setInputChannels(ic2);
    ic3[0] = createRemoteInputChannel(ig3, 1, rp3, connManager);
    ic3[1] = createRemoteInputChannel(ig3, 3, rp4, connManager);
    ig3.setInputChannels(ic3);
    Task.setupPartitionsAndGates(resultPartitions, inputGates);
    // verify buffer pools for the result partitions
    assertEquals(Integer.MAX_VALUE, rp1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(Integer.MAX_VALUE, rp2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(expectedBuffers, rp3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(expectedRp4Buffers, rp4.getBufferPool().getMaxNumberOfMemorySegments());
    for (ResultPartition rp : resultPartitions) {
        assertEquals(rp.getNumberOfSubpartitions() + 1, rp.getBufferPool().getNumberOfRequiredMemorySegments());
        assertEquals(rp.getNumberOfSubpartitions() + 1, rp.getBufferPool().getNumBuffers());
    }
    // verify buffer pools for the input gates (NOTE: credit-based uses minimum required buffers
    // for exclusive buffers not managed by the buffer pool)
    assertEquals(1, ig1.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(1, ig2.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(1, ig3.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(1, ig4.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(floatingBuffers, ig1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(floatingBuffers, ig2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(floatingBuffers, ig3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(floatingBuffers, ig4.getBufferPool().getMaxNumberOfMemorySegments());
    verify(ig1, times(1)).setupChannels();
    verify(ig2, times(1)).setupChannels();
    verify(ig3, times(1)).setupChannels();
    verify(ig4, times(1)).setupChannels();
    for (ResultPartition rp : resultPartitions) {
        rp.release();
    }
    for (SingleInputGate ig : inputGates) {
        ig.close();
    }
    network.close();
}

Example 13 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class NettyShuffleUtilsTest method calculateBuffersConsumption.

private int calculateBuffersConsumption(SingleInputGate inputGate) throws Exception {
    inputGate.setChannelStateWriter(ChannelStateWriter.NO_OP);
    inputGate.finishReadRecoveredState();
    while (!inputGate.getStateConsumedFuture().isDone()) {
        inputGate.pollNext();
    }
    inputGate.convertRecoveredInputChannels();
    int ret = 0;
    for (InputChannel ch : inputGate.getInputChannels().values()) {
        RemoteInputChannel rChannel = (RemoteInputChannel) ch;
        ret += rChannel.getNumberOfAvailableBuffers();
    }
    ret += inputGate.getBufferPool().getMaxNumberOfMemorySegments();
    return ret;
}

Example 14 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class AlternatingCheckpointsTest method testBarrierHandling.

private void testBarrierHandling(SnapshotType checkpointType) throws Exception {
    final long barrierId = 123L;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel fast = new TestInputChannel(gate, 0, false, true);
    TestInputChannel slow = new TestInputChannel(gate, 1, false, true);
    gate.setInputChannels(fast, slow);
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(gate);
    CheckpointedInputGate checkpointedGate = new CheckpointedInputGate(gate, barrierHandler, new SyncMailboxExecutor());
    if (checkpointType.isSavepoint()) {
        fast.setBlocked(true);
        slow.setBlocked(true);
    }
    CheckpointOptions options = checkpointType.isSavepoint() ? alignedNoTimeout(checkpointType, getDefault()) : unaligned(CheckpointType.CHECKPOINT, getDefault());
    Buffer barrier = barrier(barrierId, 1, options);
    send(barrier.retainBuffer(), fast, checkpointedGate);
    assertEquals(checkpointType.isSavepoint(), target.triggeredCheckpoints.isEmpty());
    send(barrier.retainBuffer(), slow, checkpointedGate);
    assertEquals(singletonList(barrierId), target.triggeredCheckpoints);
    if (checkpointType.isSavepoint()) {
        for (InputChannel channel : gate.getInputChannels().values()) {
            assertFalse(String.format("channel %d should be resumed", channel.getChannelIndex()), ((TestInputChannel) channel).isBlocked());
        }
    }
}

Example 15 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class InputGateMetrics method refreshAndGetAvg.

/**
 * Iterates over all input channels and collects the average number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return average number of queued buffers per channel
 */
float refreshAndGetAvg() {
    long total = 0;
    int count = 0;
    for (InputChannel channel : inputGate.getInputChannels().values()) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            total += size;
            ++count;
        }
    }
    return count == 0 ? 0 : total / (float) count;
}