use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class FloatingBuffersUsageGauge method calculateUsedBuffers.
@Override
public int calculateUsedBuffers(SingleInputGate inputGate) {
int availableFloatingBuffers = 0;
BufferPool bufferPool = inputGate.getBufferPool();
if (bufferPool != null) {
int requestedFloatingBuffers = bufferPool.bestEffortGetNumOfUsedBuffers();
for (InputChannel ic : inputGate.getInputChannels().values()) {
if (ic instanceof RemoteInputChannel) {
availableFloatingBuffers += ((RemoteInputChannel) ic).unsynchronizedGetFloatingBuffersAvailable();
}
}
return Math.max(0, requestedFloatingBuffers - availableFloatingBuffers);
}
return 0;
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class InputGateMetrics method refreshAndGetMin.
/**
* Iterates over all input channels and collects the minimum number of queued buffers in a
* channel in a best-effort way.
*
* @return minimum number of queued buffers per channel (<tt>0</tt> if no channels exist)
*/
int refreshAndGetMin() {
int min = Integer.MAX_VALUE;
Collection<InputChannel> channels = inputGate.getInputChannels().values();
for (InputChannel channel : channels) {
if (channel instanceof RemoteInputChannel) {
RemoteInputChannel rc = (RemoteInputChannel) channel;
int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
min = Math.min(min, size);
}
}
if (min == Integer.MAX_VALUE) {
// was empty
return 0;
}
return min;
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class InputGateMetrics method refreshAndGetMax.
/**
* Iterates over all input channels and collects the maximum number of queued buffers in a
* channel in a best-effort way.
*
* @return maximum number of queued buffers per channel
*/
int refreshAndGetMax() {
int max = 0;
for (InputChannel channel : inputGate.getInputChannels().values()) {
if (channel instanceof RemoteInputChannel) {
RemoteInputChannel rc = (RemoteInputChannel) channel;
int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
max = Math.max(max, size);
}
}
return max;
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class InputGateFairnessTest method testFairConsumptionLocalChannelsPreFilled.
@Test
public void testFairConsumptionLocalChannelsPreFilled() throws Exception {
final int numberOfChannels = 37;
final int buffersPerChannel = 27;
PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
final BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42);
// ----- create some source channels and fill them with buffers -----
final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
for (final PipelinedSubpartition subpartition : sources) {
for (int p = 0; p < buffersPerChannel; p++) {
subpartition.add(bufferConsumer.copy());
}
subpartition.finish();
}
for (ResultPartition rp : resultPartitions) {
rp.setup();
}
// ----- create reading side -----
final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
setupInputGate(gate, inputChannels);
// read all the buffers and the EOF event
for (int i = numberOfChannels * (buffersPerChannel + 1); i > 0; --i) {
assertNotNull(gate.getNext());
int min = Integer.MAX_VALUE;
int max = 0;
for (PipelinedSubpartition source : sources) {
int size = source.getNumberOfQueuedBuffers();
min = Math.min(min, size);
max = Math.max(max, size);
}
assertTrue(max == min || max == (min + 1));
}
assertFalse(gate.getNext().isPresent());
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class TestCheckpointedInputGateBuilder method buildMixedGate.
private SingleInputGate buildMixedGate(Integer... testChannelIds) throws IOException {
Set<Integer> testChannelIdSet = new HashSet<>(Arrays.asList(testChannelIds));
SingleInputGate gate = buildRemoteGate();
InputChannel[] channels = new InputChannel[numChannels];
for (int i = 0; i < numChannels; i++) {
if (testChannelIdSet.contains(i)) {
channels[i] = new TestInputChannel(gate, i, false, true);
} else {
channels[i] = gate.getChannel(i);
}
}
gate.setInputChannels(channels);
return gate;
}
Aggregations