use of org.apache.flink.runtime.io.network.metrics.CreditBasedInputBuffersUsageGauge in project flink by apache.
the class InputBuffersMetricsTest method testCalculateTotalBuffersSize.
@Test
public void testCalculateTotalBuffersSize() throws Exception {
int numberOfRemoteChannels = 2;
int numberOfLocalChannels = 0;
int numberOfBufferPerChannel = 2;
int numberOfBuffersPerGate = 8;
NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(numberOfBufferPerChannel).setFloatingNetworkBuffersPerGate(numberOfBuffersPerGate).build();
closeableRegistry.registerCloseable(network::close);
SingleInputGate inputGate1 = buildInputGate(network, numberOfRemoteChannels, numberOfLocalChannels).f0;
closeableRegistry.registerCloseable(inputGate1::close);
inputGate1.setup();
SingleInputGate[] inputGates = new SingleInputGate[] { inputGate1 };
FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
CreditBasedInputBuffersUsageGauge inputBufferPoolUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
closeableRegistry.registerCloseable(network::close);
closeableRegistry.registerCloseable(inputGate1::close);
assertEquals(numberOfBuffersPerGate, floatingBuffersUsageGauge.calculateTotalBuffers(inputGate1));
assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel, exclusiveBuffersUsageGauge.calculateTotalBuffers(inputGate1));
assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel + numberOfBuffersPerGate, inputBufferPoolUsageGauge.calculateTotalBuffers(inputGate1));
}
use of org.apache.flink.runtime.io.network.metrics.CreditBasedInputBuffersUsageGauge in project flink by apache.
the class InputBuffersMetricsTest method testExclusiveBuffersUsage.
@Test
public void testExclusiveBuffersUsage() throws Exception {
int numberOfRemoteChannelsGate1 = 2;
int numberOfLocalChannelsGate1 = 0;
int numberOfRemoteChannelsGate2 = 1;
int numberOfLocalChannelsGate2 = 1;
int totalNumberOfRemoteChannels = numberOfRemoteChannelsGate1 + numberOfRemoteChannelsGate2;
int buffersPerChannel = 2;
int extraNetworkBuffersPerGate = 8;
NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(buffersPerChannel).setFloatingNetworkBuffersPerGate(extraNetworkBuffersPerGate).build();
closeableRegistry.registerCloseable(network::close);
Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple1 = buildInputGate(network, numberOfRemoteChannelsGate1, numberOfLocalChannelsGate1);
Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple2 = buildInputGate(network, numberOfRemoteChannelsGate2, numberOfLocalChannelsGate2);
SingleInputGate inputGate1 = tuple1.f0;
SingleInputGate inputGate2 = tuple2.f0;
closeableRegistry.registerCloseable(inputGate1::close);
closeableRegistry.registerCloseable(inputGate2::close);
inputGate1.setup();
inputGate2.setup();
List<RemoteInputChannel> remoteInputChannels = tuple1.f1;
SingleInputGate[] inputGates = new SingleInputGate[] { tuple1.f0, tuple2.f0 };
FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
CreditBasedInputBuffersUsageGauge inputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
assertEquals(0.0, exclusiveBuffersUsageGauge.getValue(), 0.0);
assertEquals(0.0, inputBuffersUsageGauge.getValue(), 0.0);
int totalBuffers = extraNetworkBuffersPerGate * inputGates.length + buffersPerChannel * totalNumberOfRemoteChannels;
int channelIndex = 1;
for (RemoteInputChannel channel : remoteInputChannels) {
drainAndValidate(buffersPerChannel, buffersPerChannel * channelIndex++, channel, totalBuffers, buffersPerChannel * totalNumberOfRemoteChannels, exclusiveBuffersUsageGauge, inputBuffersUsageGauge, inputGate1);
}
}
use of org.apache.flink.runtime.io.network.metrics.CreditBasedInputBuffersUsageGauge in project flink by apache.
the class InputBuffersMetricsTest method testFloatingBuffersUsage.
@Test
public void testFloatingBuffersUsage() throws Exception {
int numberOfRemoteChannelsGate1 = 2;
int numberOfLocalChannelsGate1 = 0;
int numberOfRemoteChannelsGate2 = 1;
int numberOfLocalChannelsGate2 = 1;
int totalNumberOfRemoteChannels = numberOfRemoteChannelsGate1 + numberOfRemoteChannelsGate2;
int buffersPerChannel = 2;
int extraNetworkBuffersPerGate = 8;
NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(buffersPerChannel).setFloatingNetworkBuffersPerGate(extraNetworkBuffersPerGate).build();
closeableRegistry.registerCloseable(network::close);
Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple1 = buildInputGate(network, numberOfRemoteChannelsGate1, numberOfLocalChannelsGate1);
SingleInputGate inputGate2 = buildInputGate(network, numberOfRemoteChannelsGate2, numberOfLocalChannelsGate2).f0;
SingleInputGate inputGate1 = tuple1.f0;
closeableRegistry.registerCloseable(inputGate1::close);
closeableRegistry.registerCloseable(inputGate2::close);
inputGate1.setup();
inputGate2.setup();
RemoteInputChannel remoteInputChannel1 = tuple1.f1.get(0);
SingleInputGate[] inputGates = new SingleInputGate[] { tuple1.f0, inputGate2 };
FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
CreditBasedInputBuffersUsageGauge inputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
assertEquals(0.0, floatingBuffersUsageGauge.getValue(), 0.0);
assertEquals(0.0, inputBuffersUsageGauge.getValue(), 0.0);
// drain gate1's exclusive buffers
drainBuffer(buffersPerChannel, remoteInputChannel1);
int totalBuffers = extraNetworkBuffersPerGate * inputGates.length + buffersPerChannel * totalNumberOfRemoteChannels;
remoteInputChannel1.requestSubpartition();
int backlog = 3;
int totalRequestedBuffers = buffersPerChannel + backlog;
remoteInputChannel1.onSenderBacklog(backlog);
assertEquals(totalRequestedBuffers, remoteInputChannel1.unsynchronizedGetFloatingBuffersAvailable());
drainBuffer(totalRequestedBuffers, remoteInputChannel1);
assertEquals(0, remoteInputChannel1.unsynchronizedGetFloatingBuffersAvailable());
assertEquals((double) (buffersPerChannel + totalRequestedBuffers) / totalBuffers, inputBuffersUsageGauge.getValue(), 0.0001);
}
Aggregations