Search in sources :

Example 1 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class FloatingBuffersUsageGauge method calculateUsedBuffers.

@Override
public int calculateUsedBuffers(SingleInputGate inputGate) {
    int availableFloatingBuffers = 0;
    BufferPool bufferPool = inputGate.getBufferPool();
    if (bufferPool != null) {
        int requestedFloatingBuffers = bufferPool.bestEffortGetNumOfUsedBuffers();
        for (InputChannel ic : inputGate.getInputChannels().values()) {
            if (ic instanceof RemoteInputChannel) {
                availableFloatingBuffers += ((RemoteInputChannel) ic).unsynchronizedGetFloatingBuffersAvailable();
            }
        }
        return Math.max(0, requestedFloatingBuffers - availableFloatingBuffers);
    }
    return 0;
}
Also used : BufferPool(org.apache.flink.runtime.io.network.buffer.BufferPool) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Example 2 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class InputGateMetrics method refreshAndGetMin.

/**
 * Iterates over all input channels and collects the minimum number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return minimum number of queued buffers per channel (<tt>0</tt> if no channels exist)
 */
int refreshAndGetMin() {
    int min = Integer.MAX_VALUE;
    Collection<InputChannel> channels = inputGate.getInputChannels().values();
    for (InputChannel channel : channels) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            min = Math.min(min, size);
        }
    }
    if (min == Integer.MAX_VALUE) {
        // was empty
        return 0;
    }
    return min;
}
Also used : InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Example 3 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class InputGateMetrics method refreshAndGetMax.

/**
 * Iterates over all input channels and collects the maximum number of queued buffers in a
 * channel in a best-effort way.
 *
 * @return maximum number of queued buffers per channel
 */
int refreshAndGetMax() {
    int max = 0;
    for (InputChannel channel : inputGate.getInputChannels().values()) {
        if (channel instanceof RemoteInputChannel) {
            RemoteInputChannel rc = (RemoteInputChannel) channel;
            int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
            max = Math.max(max, size);
        }
    }
    return max;
}
Also used : InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Example 4 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionLocalChannelsPreFilled.

@Test
public void testFairConsumptionLocalChannelsPreFilled() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
    final BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42);
    // ----- create some source channels and fill them with buffers -----
    final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
    for (final PipelinedSubpartition subpartition : sources) {
        for (int p = 0; p < buffersPerChannel; p++) {
            subpartition.add(bufferConsumer.copy());
        }
        subpartition.finish();
    }
    for (ResultPartition rp : resultPartitions) {
        rp.setup();
    }
    // ----- create reading side -----
    final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
    final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
    setupInputGate(gate, inputChannels);
    // read all the buffers and the EOF event
    for (int i = numberOfChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNext());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (PipelinedSubpartition source : sources) {
            int size = source.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == (min + 1));
    }
    assertFalse(gate.getNext().isPresent());
}
Also used : IntStream(java.util.stream.IntStream) Arrays(java.util.Arrays) TestBufferFactory(org.apache.flink.runtime.io.network.util.TestBufferFactory) SystemClock(org.apache.flink.util.clock.SystemClock) ThroughputCalculator(org.apache.flink.runtime.throughput.ThroughputCalculator) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) BufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferConsumer) SubpartitionIndexRange(org.apache.flink.runtime.deployment.SubpartitionIndexRange) InputChannelTestUtils.createDummyConnectionManager(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createDummyConnectionManager) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) Assert.fail(org.junit.Assert.fail) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) EventSerializer(org.apache.flink.runtime.io.network.api.serialization.EventSerializer) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) Assert.assertNotNull(org.junit.Assert.assertNotNull) BufferPool(org.apache.flink.runtime.io.network.buffer.BufferPool) Collection(java.util.Collection) InputChannelBuilder(org.apache.flink.runtime.io.network.partition.consumer.InputChannelBuilder) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) BufferBuilderTestUtils.createFilledFinishedBufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) NoOpBufferPool(org.apache.flink.runtime.io.network.buffer.NoOpBufferPool) UnpooledMemorySegmentProvider(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.UnpooledMemorySegmentProvider) Collections(java.util.Collections) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) SupplierWithException(org.apache.flink.util.function.SupplierWithException) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) BufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferConsumer) BufferBuilderTestUtils.createFilledFinishedBufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) Test(org.junit.Test)

Example 5 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.

the class TestCheckpointedInputGateBuilder method buildMixedGate.

private SingleInputGate buildMixedGate(Integer... testChannelIds) throws IOException {
    Set<Integer> testChannelIdSet = new HashSet<>(Arrays.asList(testChannelIds));
    SingleInputGate gate = buildRemoteGate();
    InputChannel[] channels = new InputChannel[numChannels];
    for (int i = 0; i < numChannels; i++) {
        if (testChannelIdSet.contains(i)) {
            channels[i] = new TestInputChannel(gate, i, false, true);
        } else {
            channels[i] = gate.getChannel(i);
        }
    }
    gate.setInputChannels(channels);
    return gate;
}
Also used : TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) HashSet(java.util.HashSet)

Aggregations

InputChannel (org.apache.flink.runtime.io.network.partition.consumer.InputChannel)11 RemoteInputChannel (org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)11 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)5 HashSet (java.util.HashSet)3 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)3 BufferPool (org.apache.flink.runtime.io.network.buffer.BufferPool)3 InputChannelTestUtils.createDummyConnectionManager (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createDummyConnectionManager)3 SingleInputGateBuilder (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 Optional (java.util.Optional)2 IntStream (java.util.stream.IntStream)2 SubpartitionIndexRange (org.apache.flink.runtime.deployment.SubpartitionIndexRange)2 ConnectionManager (org.apache.flink.runtime.io.network.ConnectionManager)2 EndOfPartitionEvent (org.apache.flink.runtime.io.network.api.EndOfPartitionEvent)2 EventSerializer (org.apache.flink.runtime.io.network.api.serialization.EventSerializer)2 BufferBuilderTestUtils.createFilledFinishedBufferConsumer (org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer)2