Examples with ExclusiveBuffersUsageGauge org.apache.flink.runtime.io.network.metrics.ExclusiveBuffersUsageGauge used on opensource projects

Example 1 with ExclusiveBuffersUsageGauge

use of org.apache.flink.runtime.io.network.metrics.ExclusiveBuffersUsageGauge in project flink by apache.

the class InputBuffersMetricsTest method testCalculateTotalBuffersSize.

@Test
public void testCalculateTotalBuffersSize() throws Exception {
    int numberOfRemoteChannels = 2;
    int numberOfLocalChannels = 0;
    int numberOfBufferPerChannel = 2;
    int numberOfBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(numberOfBufferPerChannel).setFloatingNetworkBuffersPerGate(numberOfBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    SingleInputGate inputGate1 = buildInputGate(network, numberOfRemoteChannels, numberOfLocalChannels).f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    inputGate1.setup();
    SingleInputGate[] inputGates = new SingleInputGate[] { inputGate1 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBufferPoolUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    closeableRegistry.registerCloseable(network::close);
    closeableRegistry.registerCloseable(inputGate1::close);
    assertEquals(numberOfBuffersPerGate, floatingBuffersUsageGauge.calculateTotalBuffers(inputGate1));
    assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel, exclusiveBuffersUsageGauge.calculateTotalBuffers(inputGate1));
    assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel + numberOfBuffersPerGate, inputBufferPoolUsageGauge.calculateTotalBuffers(inputGate1));
}

Example 2 with ExclusiveBuffersUsageGauge

use of org.apache.flink.runtime.io.network.metrics.ExclusiveBuffersUsageGauge in project flink by apache.

the class InputBuffersMetricsTest method testExclusiveBuffersUsage.

@Test
public void testExclusiveBuffersUsage() throws Exception {
    int numberOfRemoteChannelsGate1 = 2;
    int numberOfLocalChannelsGate1 = 0;
    int numberOfRemoteChannelsGate2 = 1;
    int numberOfLocalChannelsGate2 = 1;
    int totalNumberOfRemoteChannels = numberOfRemoteChannelsGate1 + numberOfRemoteChannelsGate2;
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(buffersPerChannel).setFloatingNetworkBuffersPerGate(extraNetworkBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple1 = buildInputGate(network, numberOfRemoteChannelsGate1, numberOfLocalChannelsGate1);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple2 = buildInputGate(network, numberOfRemoteChannelsGate2, numberOfLocalChannelsGate2);
    SingleInputGate inputGate1 = tuple1.f0;
    SingleInputGate inputGate2 = tuple2.f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    closeableRegistry.registerCloseable(inputGate2::close);
    inputGate1.setup();
    inputGate2.setup();
    List<RemoteInputChannel> remoteInputChannels = tuple1.f1;
    SingleInputGate[] inputGates = new SingleInputGate[] { tuple1.f0, tuple2.f0 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    assertEquals(0.0, exclusiveBuffersUsageGauge.getValue(), 0.0);
    assertEquals(0.0, inputBuffersUsageGauge.getValue(), 0.0);
    int totalBuffers = extraNetworkBuffersPerGate * inputGates.length + buffersPerChannel * totalNumberOfRemoteChannels;
    int channelIndex = 1;
    for (RemoteInputChannel channel : remoteInputChannels) {
        drainAndValidate(buffersPerChannel, buffersPerChannel * channelIndex++, channel, totalBuffers, buffersPerChannel * totalNumberOfRemoteChannels, exclusiveBuffersUsageGauge, inputBuffersUsageGauge, inputGate1);
    }
}

Example 3 with ExclusiveBuffersUsageGauge

use of org.apache.flink.runtime.io.network.metrics.ExclusiveBuffersUsageGauge in project flink by apache.

the class InputBuffersMetricsTest method testFloatingBuffersUsage.

@Test
public void testFloatingBuffersUsage() throws Exception {
    int numberOfRemoteChannelsGate1 = 2;
    int numberOfLocalChannelsGate1 = 0;
    int numberOfRemoteChannelsGate2 = 1;
    int numberOfLocalChannelsGate2 = 1;
    int totalNumberOfRemoteChannels = numberOfRemoteChannelsGate1 + numberOfRemoteChannelsGate2;
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(buffersPerChannel).setFloatingNetworkBuffersPerGate(extraNetworkBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple1 = buildInputGate(network, numberOfRemoteChannelsGate1, numberOfLocalChannelsGate1);
    SingleInputGate inputGate2 = buildInputGate(network, numberOfRemoteChannelsGate2, numberOfLocalChannelsGate2).f0;
    SingleInputGate inputGate1 = tuple1.f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    closeableRegistry.registerCloseable(inputGate2::close);
    inputGate1.setup();
    inputGate2.setup();
    RemoteInputChannel remoteInputChannel1 = tuple1.f1.get(0);
    SingleInputGate[] inputGates = new SingleInputGate[] { tuple1.f0, inputGate2 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    assertEquals(0.0, floatingBuffersUsageGauge.getValue(), 0.0);
    assertEquals(0.0, inputBuffersUsageGauge.getValue(), 0.0);
    // drain gate1's exclusive buffers
    drainBuffer(buffersPerChannel, remoteInputChannel1);
    int totalBuffers = extraNetworkBuffersPerGate * inputGates.length + buffersPerChannel * totalNumberOfRemoteChannels;
    remoteInputChannel1.requestSubpartition();
    int backlog = 3;
    int totalRequestedBuffers = buffersPerChannel + backlog;
    remoteInputChannel1.onSenderBacklog(backlog);
    assertEquals(totalRequestedBuffers, remoteInputChannel1.unsynchronizedGetFloatingBuffersAvailable());
    drainBuffer(totalRequestedBuffers, remoteInputChannel1);
    assertEquals(0, remoteInputChannel1.unsynchronizedGetFloatingBuffersAvailable());
    assertEquals((double) (buffersPerChannel + totalRequestedBuffers) / totalBuffers, inputBuffersUsageGauge.getValue(), 0.0001);
}