Examples with InputChannel org.apache.flink.runtime.io.network.partition.consumer.InputChannel used on opensource projects

Example 6 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.

the class InputGateMetrics method refreshAndGetAvg.

/**
 * Iterates over all input channels and collects the average number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return average number of queued buffers per channel
 */
float refreshAndGetAvg() {
    long total = 0;
    int count = 0;
    for (InputChannel channel : inputGate.getInputChannels().values()) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            total += size;
            ++count;
        }
    }
    return count == 0 ? 0 : total / (float) count;
}

Example 7 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.

the class InputGateMetrics method refreshAndGetMax.

/**
 * Iterates over all input channels and collects the maximum number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return maximum number of queued buffers per channel
 */
int refreshAndGetMax() {
    int max = 0;
    for (InputChannel channel : inputGate.getInputChannels().values()) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            max = Math.max(max, size);
        }
    }
    return max;
}

Example 8 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.

the class InputGateFairnessTest method testFairConsumptionLocalChannels.

@Test
public void testFairConsumptionLocalChannels() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
    try (BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42)) {
        // ----- create some source channels and fill them with one buffer each -----
        final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
        // ----- create reading side -----
        final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
        final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
        for (ResultPartition rp : resultPartitions) {
            rp.setup();
        }
        // seed one initial buffer
        sources[12].add(bufferConsumer.copy());
        setupInputGate(gate, inputChannels);
        // read all the buffers and the EOF event
        for (int i = 0; i < numberOfChannels * buffersPerChannel; i++) {
            assertNotNull(gate.getNext());
            int min = Integer.MAX_VALUE;
            int max = 0;
            for (PipelinedSubpartition source : sources) {
                int size = source.getNumberOfQueuedBuffers();
                min = Math.min(min, size);
                max = Math.max(max, size);
            }
            assertTrue(max == min || max == min + 1);
            if (i % (2 * numberOfChannels) == 0) {
                // add three buffers to each channel, in random order
                fillRandom(sources, 3, bufferConsumer);
            }
        }
    // there is still more in the queues
    }
}

Example 9 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.

the class TestCheckpointedInputGateBuilder method buildMixedGate.

private SingleInputGate buildMixedGate(Integer... testChannelIds) throws IOException {
    Set<Integer> testChannelIdSet = new HashSet<>(Arrays.asList(testChannelIds));
    SingleInputGate gate = buildRemoteGate();
    InputChannel[] channels = new InputChannel[numChannels];
    for (int i = 0; i < numChannels; i++) {
        if (testChannelIdSet.contains(i)) {
            channels[i] = new TestInputChannel(gate, i, false, true);
        } else {
            channels[i] = gate.getChannel(i);
        }
    }
    gate.setInputChannels(channels);
    return gate;
}

Example 10 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class FloatingBuffersUsageGauge method calculateUsedBuffers.

@Override
public int calculateUsedBuffers(SingleInputGate inputGate) {
    int availableFloatingBuffers = 0;
    BufferPool bufferPool = inputGate.getBufferPool();
    if (bufferPool != null) {
        int requestedFloatingBuffers = bufferPool.bestEffortGetNumOfUsedBuffers();
        for (InputChannel ic : inputGate.getInputChannels().values()) {
            if (ic instanceof RemoteInputChannel) {
                availableFloatingBuffers += ((RemoteInputChannel) ic).unsynchronizedGetFloatingBuffersAvailable();
            }
        }
        return Math.max(0, requestedFloatingBuffers - availableFloatingBuffers);
    }
    return 0;
}