Examples with InputChannel org.apache.flink.runtime.io.network.partition.consumer.InputChannel used on opensource projects

Example 26 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class InputGateMetrics method refreshAndGetAvg.

/**
 * Iterates over all input channels and collects the average number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return average number of queued buffers per channel
 */
float refreshAndGetAvg() {
    long total = 0;
    int count = 0;
    for (InputChannel channel : inputGate.getInputChannels().values()) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            total += size;
            ++count;
        }
    }
    return count == 0 ? 0 : total / (float) count;
}

Example 27 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class InputGateMetrics method refreshAndGetMin.

/**
 * Iterates over all input channels and collects the minimum number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return minimum number of queued buffers per channel (<tt>0</tt> if no channels exist)
 */
int refreshAndGetMin() {
    int min = Integer.MAX_VALUE;
    Collection<InputChannel> channels = inputGate.getInputChannels().values();
    for (InputChannel channel : channels) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            min = Math.min(min, size);
        }
    }
    if (min == Integer.MAX_VALUE) {
        // was empty
        return 0;
    }
    return min;
}

Example 28 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class InputGateMetrics method refreshAndGetMax.

/**
 * Iterates over all input channels and collects the maximum number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return maximum number of queued buffers per channel
 */
int refreshAndGetMax() {
    int max = 0;
    for (InputChannel channel : inputGate.getInputChannels().values()) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            max = Math.max(max, size);
        }
    }
    return max;
}

Example 29 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class NettyShuffleUtilsTest method calculateBuffersConsumption.

private int calculateBuffersConsumption(SingleInputGate inputGate) throws Exception {
    inputGate.setChannelStateWriter(ChannelStateWriter.NO_OP);
    inputGate.finishReadRecoveredState();
    while (!inputGate.getStateConsumedFuture().isDone()) {
        inputGate.pollNext();
    }
    inputGate.convertRecoveredInputChannels();
    int ret = 0;
    for (InputChannel ch : inputGate.getInputChannels().values()) {
        RemoteInputChannel rChannel = (RemoteInputChannel) ch;
        ret += rChannel.getNumberOfAvailableBuffers();
    }
    ret += inputGate.getBufferPool().getMaxNumberOfMemorySegments();
    return ret;
}

Example 30 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class AlternatingCheckpointsTest method testBarrierHandling.

private void testBarrierHandling(SnapshotType checkpointType) throws Exception {
    final long barrierId = 123L;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel fast = new TestInputChannel(gate, 0, false, true);
    TestInputChannel slow = new TestInputChannel(gate, 1, false, true);
    gate.setInputChannels(fast, slow);
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(gate);
    CheckpointedInputGate checkpointedGate = new CheckpointedInputGate(gate, barrierHandler, new SyncMailboxExecutor());
    if (checkpointType.isSavepoint()) {
        fast.setBlocked(true);
        slow.setBlocked(true);
    }
    CheckpointOptions options = checkpointType.isSavepoint() ? alignedNoTimeout(checkpointType, getDefault()) : unaligned(CheckpointType.CHECKPOINT, getDefault());
    Buffer barrier = barrier(barrierId, 1, options);
    send(barrier.retainBuffer(), fast, checkpointedGate);
    assertEquals(checkpointType.isSavepoint(), target.triggeredCheckpoints.isEmpty());
    send(barrier.retainBuffer(), slow, checkpointedGate);
    assertEquals(singletonList(barrierId), target.triggeredCheckpoints);
    if (checkpointType.isSavepoint()) {
        for (InputChannel channel : gate.getInputChannels().values()) {
            assertFalse(String.format("channel %d should be resumed", channel.getChannelIndex()), ((TestInputChannel) channel).isBlocked());
        }
    }
}