Examples with NettyShuffleEnvironment org.apache.flink.runtime.io.network.NettyShuffleEnvironment used on opensource projects

Example 6 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class PartitionRequestQueueTest method createFinishedPartitionWithFilledData.

private static ResultPartition createFinishedPartitionWithFilledData(ResultPartitionManager partitionManager) throws Exception {
    NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().setResultPartitionManager(partitionManager).build();
    ResultPartition partition = createPartition(environment, fileChannelManager, ResultPartitionType.BLOCKING, 1);
    partition.setup();
    partition.emitRecord(ByteBuffer.allocate(BUFFER_SIZE), 0);
    partition.finish();
    return partition;
}

Example 7 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class InputBuffersMetricsTest method testCalculateTotalBuffersSize.

@Test
public void testCalculateTotalBuffersSize() throws Exception {
    int numberOfRemoteChannels = 2;
    int numberOfLocalChannels = 0;
    int numberOfBufferPerChannel = 2;
    int numberOfBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(numberOfBufferPerChannel).setFloatingNetworkBuffersPerGate(numberOfBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    SingleInputGate inputGate1 = buildInputGate(network, numberOfRemoteChannels, numberOfLocalChannels).f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    inputGate1.setup();
    SingleInputGate[] inputGates = new SingleInputGate[] { inputGate1 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBufferPoolUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    closeableRegistry.registerCloseable(network::close);
    closeableRegistry.registerCloseable(inputGate1::close);
    assertEquals(numberOfBuffersPerGate, floatingBuffersUsageGauge.calculateTotalBuffers(inputGate1));
    assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel, exclusiveBuffersUsageGauge.calculateTotalBuffers(inputGate1));
    assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel + numberOfBuffersPerGate, inputBufferPoolUsageGauge.calculateTotalBuffers(inputGate1));
}

Example 8 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class InputBuffersMetricsTest method testExclusiveBuffersUsage.

@Test
public void testExclusiveBuffersUsage() throws Exception {
    int numberOfRemoteChannelsGate1 = 2;
    int numberOfLocalChannelsGate1 = 0;
    int numberOfRemoteChannelsGate2 = 1;
    int numberOfLocalChannelsGate2 = 1;
    int totalNumberOfRemoteChannels = numberOfRemoteChannelsGate1 + numberOfRemoteChannelsGate2;
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(buffersPerChannel).setFloatingNetworkBuffersPerGate(extraNetworkBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple1 = buildInputGate(network, numberOfRemoteChannelsGate1, numberOfLocalChannelsGate1);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple2 = buildInputGate(network, numberOfRemoteChannelsGate2, numberOfLocalChannelsGate2);
    SingleInputGate inputGate1 = tuple1.f0;
    SingleInputGate inputGate2 = tuple2.f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    closeableRegistry.registerCloseable(inputGate2::close);
    inputGate1.setup();
    inputGate2.setup();
    List<RemoteInputChannel> remoteInputChannels = tuple1.f1;
    SingleInputGate[] inputGates = new SingleInputGate[] { tuple1.f0, tuple2.f0 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    assertEquals(0.0, exclusiveBuffersUsageGauge.getValue(), 0.0);
    assertEquals(0.0, inputBuffersUsageGauge.getValue(), 0.0);
    int totalBuffers = extraNetworkBuffersPerGate * inputGates.length + buffersPerChannel * totalNumberOfRemoteChannels;
    int channelIndex = 1;
    for (RemoteInputChannel channel : remoteInputChannels) {
        drainAndValidate(buffersPerChannel, buffersPerChannel * channelIndex++, channel, totalBuffers, buffersPerChannel * totalNumberOfRemoteChannels, exclusiveBuffersUsageGauge, inputBuffersUsageGauge, inputGate1);
    }
}

Example 9 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class ResultPartitionTest method createResultPartition.

private BufferWritingResultPartition createResultPartition(ResultPartitionType partitionType) throws IOException {
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(10).setBufferSize(bufferSize).build();
    ResultPartition resultPartition = createPartition(network, fileChannelManager, partitionType, 2);
    resultPartition.setup();
    return (BufferWritingResultPartition) resultPartition;
}

Example 10 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class ResultPartitionTest method testReleaseMemoryOnPipelinedPartition.

/**
 * Tests {@link ResultPartition#close()} and {@link ResultPartition#release()} on a working
 * pipelined partition.
 */
@Test
public void testReleaseMemoryOnPipelinedPartition() throws Exception {
    final int numAllBuffers = 10;
    final NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(numAllBuffers).setBufferSize(bufferSize).build();
    final ResultPartition resultPartition = createPartition(network, ResultPartitionType.PIPELINED, 1);
    try {
        resultPartition.setup();
        // take all buffers (more than the minimum required)
        for (int i = 0; i < numAllBuffers; ++i) {
            resultPartition.emitRecord(ByteBuffer.allocate(bufferSize - 1), 0);
        }
        assertEquals(0, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());
        resultPartition.close();
        assertTrue(resultPartition.getBufferPool().isDestroyed());
        assertEquals(numAllBuffers, network.getNetworkBufferPool().getNumberOfUsedMemorySegments());
        resultPartition.release();
        assertEquals(0, network.getNetworkBufferPool().getNumberOfUsedMemorySegments());
    } finally {
        network.close();
    }
}