use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class InputGateMetrics method refreshAndGetTotal.
// ------------------------------------------------------------------------
// these methods are package private to make access from the nested classes faster
/**
* Iterates over all input channels and collects the total number of queued buffers in a
* best-effort way.
*
* @return total number of queued buffers
*/
long refreshAndGetTotal() {
long total = 0;
for (InputChannel channel : inputGate.getInputChannels().values()) {
if (channel instanceof RemoteInputChannel) {
RemoteInputChannel rc = (RemoteInputChannel) channel;
total += rc.unsynchronizedGetNumberOfQueuedBuffers();
}
}
return total;
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class NettyShuffleEnvironmentTest method testRegisterTaskWithLimitedBuffers.
private void testRegisterTaskWithLimitedBuffers(int bufferPoolSize) throws Exception {
final NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(bufferPoolSize).build();
final ConnectionManager connManager = createDummyConnectionManager();
int channels = 2;
int rp4Channels = 4;
int floatingBuffers = network.getConfiguration().floatingNetworkBuffersPerGate();
int exclusiveBuffers = network.getConfiguration().networkBuffersPerChannel();
int expectedBuffers = channels * exclusiveBuffers + floatingBuffers;
int expectedRp4Buffers = rp4Channels * exclusiveBuffers + floatingBuffers;
// result partitions
ResultPartition rp1 = createPartition(network, ResultPartitionType.PIPELINED, channels);
ResultPartition rp2 = createPartition(network, fileChannelManager, ResultPartitionType.BLOCKING, channels);
ResultPartition rp3 = createPartition(network, ResultPartitionType.PIPELINED_BOUNDED, channels);
ResultPartition rp4 = createPartition(network, ResultPartitionType.PIPELINED_BOUNDED, rp4Channels);
final ResultPartition[] resultPartitions = new ResultPartition[] { rp1, rp2, rp3, rp4 };
// input gates
SingleInputGate ig1 = createSingleInputGate(network, ResultPartitionType.PIPELINED, channels);
SingleInputGate ig2 = createSingleInputGate(network, ResultPartitionType.BLOCKING, channels);
SingleInputGate ig3 = createSingleInputGate(network, ResultPartitionType.PIPELINED_BOUNDED, channels);
SingleInputGate ig4 = createSingleInputGate(network, ResultPartitionType.PIPELINED_BOUNDED, rp4Channels);
InputChannel[] ic1 = new InputChannel[channels];
InputChannel[] ic2 = new InputChannel[channels];
InputChannel[] ic3 = new InputChannel[channels];
InputChannel[] ic4 = new InputChannel[rp4Channels];
final SingleInputGate[] inputGates = new SingleInputGate[] { ig1, ig2, ig3, ig4 };
ic4[0] = createRemoteInputChannel(ig4, 0, rp1, connManager);
ic4[1] = createRemoteInputChannel(ig4, 0, rp2, connManager);
ic4[2] = createRemoteInputChannel(ig4, 0, rp3, connManager);
ic4[3] = createRemoteInputChannel(ig4, 0, rp4, connManager);
ig4.setInputChannels(ic4);
ic1[0] = createRemoteInputChannel(ig1, 1, rp1, connManager);
ic1[1] = createRemoteInputChannel(ig1, 1, rp4, connManager);
ig1.setInputChannels(ic1);
ic2[0] = createRemoteInputChannel(ig2, 1, rp2, connManager);
ic2[1] = createRemoteInputChannel(ig2, 2, rp4, connManager);
ig2.setInputChannels(ic2);
ic3[0] = createRemoteInputChannel(ig3, 1, rp3, connManager);
ic3[1] = createRemoteInputChannel(ig3, 3, rp4, connManager);
ig3.setInputChannels(ic3);
Task.setupPartitionsAndGates(resultPartitions, inputGates);
// verify buffer pools for the result partitions
assertEquals(Integer.MAX_VALUE, rp1.getBufferPool().getMaxNumberOfMemorySegments());
assertEquals(Integer.MAX_VALUE, rp2.getBufferPool().getMaxNumberOfMemorySegments());
assertEquals(expectedBuffers, rp3.getBufferPool().getMaxNumberOfMemorySegments());
assertEquals(expectedRp4Buffers, rp4.getBufferPool().getMaxNumberOfMemorySegments());
for (ResultPartition rp : resultPartitions) {
assertEquals(rp.getNumberOfSubpartitions() + 1, rp.getBufferPool().getNumberOfRequiredMemorySegments());
assertEquals(rp.getNumberOfSubpartitions() + 1, rp.getBufferPool().getNumBuffers());
}
// verify buffer pools for the input gates (NOTE: credit-based uses minimum required buffers
// for exclusive buffers not managed by the buffer pool)
assertEquals(1, ig1.getBufferPool().getNumberOfRequiredMemorySegments());
assertEquals(1, ig2.getBufferPool().getNumberOfRequiredMemorySegments());
assertEquals(1, ig3.getBufferPool().getNumberOfRequiredMemorySegments());
assertEquals(1, ig4.getBufferPool().getNumberOfRequiredMemorySegments());
assertEquals(floatingBuffers, ig1.getBufferPool().getMaxNumberOfMemorySegments());
assertEquals(floatingBuffers, ig2.getBufferPool().getMaxNumberOfMemorySegments());
assertEquals(floatingBuffers, ig3.getBufferPool().getMaxNumberOfMemorySegments());
assertEquals(floatingBuffers, ig4.getBufferPool().getMaxNumberOfMemorySegments());
verify(ig1, times(1)).setupChannels();
verify(ig2, times(1)).setupChannels();
verify(ig3, times(1)).setupChannels();
verify(ig4, times(1)).setupChannels();
for (ResultPartition rp : resultPartitions) {
rp.release();
}
for (SingleInputGate ig : inputGates) {
ig.close();
}
network.close();
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by apache.
the class InputGateFairnessTest method testFairConsumptionLocalChannels.
@Test
public void testFairConsumptionLocalChannels() throws Exception {
final int numberOfChannels = 37;
final int buffersPerChannel = 27;
PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
try (BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42)) {
// ----- create some source channels and fill them with one buffer each -----
final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
// ----- create reading side -----
final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
for (ResultPartition rp : resultPartitions) {
rp.setup();
}
// seed one initial buffer
sources[12].add(bufferConsumer.copy());
setupInputGate(gate, inputChannels);
// read all the buffers and the EOF event
for (int i = 0; i < numberOfChannels * buffersPerChannel; i++) {
assertNotNull(gate.getNext());
int min = Integer.MAX_VALUE;
int max = 0;
for (PipelinedSubpartition source : sources) {
int size = source.getNumberOfQueuedBuffers();
min = Math.min(min, size);
max = Math.max(max, size);
}
assertTrue(max == min || max == min + 1);
if (i % (2 * numberOfChannels) == 0) {
// add three buffers to each channel, in random order
fillRandom(sources, 3, bufferConsumer);
}
}
// there is still more in the queues
}
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.
the class FloatingBuffersUsageGauge method calculateUsedBuffers.
@Override
public int calculateUsedBuffers(SingleInputGate inputGate) {
int availableFloatingBuffers = 0;
BufferPool bufferPool = inputGate.getBufferPool();
if (bufferPool != null) {
int requestedFloatingBuffers = bufferPool.bestEffortGetNumOfUsedBuffers();
for (InputChannel ic : inputGate.getInputChannels().values()) {
if (ic instanceof RemoteInputChannel) {
availableFloatingBuffers += ((RemoteInputChannel) ic).unsynchronizedGetFloatingBuffersAvailable();
}
}
return Math.max(0, requestedFloatingBuffers - availableFloatingBuffers);
}
return 0;
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.
the class InputGateMetrics method refreshAndGetTotal.
// ------------------------------------------------------------------------
// these methods are package private to make access from the nested classes faster
/**
* Iterates over all input channels and collects the total number of queued buffers in a
* best-effort way.
*
* @return total number of queued buffers
*/
long refreshAndGetTotal() {
long total = 0;
for (InputChannel channel : inputGate.getInputChannels().values()) {
if (channel instanceof RemoteInputChannel) {
RemoteInputChannel rc = (RemoteInputChannel) channel;
total += rc.unsynchronizedGetNumberOfQueuedBuffers();
}
}
return total;
}
Aggregations