Examples with NettyShuffleEnvironmentBuilder org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder used on opensource projects

Example 1 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class SingleInputGateTest method testSingleInputGateWithSubpartitionIndexRange.

@Test
public void testSingleInputGateWithSubpartitionIndexRange() throws IOException, InterruptedException {
    IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
    SubpartitionIndexRange subpartitionIndexRange = new SubpartitionIndexRange(0, 1);
    final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().build();
    ResourceID localLocation = ResourceID.generate();
    SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.BLOCKING, subpartitionIndexRange, netEnv, localLocation, new TestingConnectionManager(), new TestingResultPartitionManager(new NoOpResultSubpartitionView()));
    for (InputChannel channel : gate.getInputChannels().values()) {
        if (channel instanceof ChannelStateHolder) {
            ((ChannelStateHolder) channel).setChannelStateWriter(ChannelStateWriter.NO_OP);
        }
    }
    SubpartitionInfo info1 = createSubpartitionInfo(partitionIds[0], 0);
    SubpartitionInfo info2 = createSubpartitionInfo(partitionIds[0], 1);
    SubpartitionInfo info3 = createSubpartitionInfo(partitionIds[1], 0);
    SubpartitionInfo info4 = createSubpartitionInfo(partitionIds[1], 1);
    SubpartitionInfo info5 = createSubpartitionInfo(partitionIds[2], 0);
    SubpartitionInfo info6 = createSubpartitionInfo(partitionIds[2], 1);
    assertThat(gate.getInputChannels().size(), is(6));
    assertThat(gate.getInputChannels().get(info1).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info2).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info3).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info4).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info5).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info6).getConsumedSubpartitionIndex(), is(1));
    assertChannelsType(gate, LocalRecoveredInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteRecoveredInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    // test setup
    gate.setup();
    assertNotNull(gate.getBufferPool());
    assertEquals(1, gate.getBufferPool().getNumberOfRequiredMemorySegments());
    gate.finishReadRecoveredState();
    while (!gate.getStateConsumedFuture().isDone()) {
        gate.pollNext();
    }
    // test request partitions
    gate.requestPartitions();
    gate.pollNext();
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    for (InputChannel inputChannel : gate.getInputChannels().values()) {
        if (inputChannel instanceof RemoteInputChannel) {
            assertNotNull(((RemoteInputChannel) inputChannel).getPartitionRequestClient());
            assertEquals(2, ((RemoteInputChannel) inputChannel).getInitialCredit());
        } else if (inputChannel instanceof LocalInputChannel) {
            assertNotNull(((LocalInputChannel) inputChannel).getSubpartitionView());
        }
    }
    // test update channels
    gate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(partitionIds[2], localLocation));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info5, info6));
}

Example 2 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class PartitionRequestQueueTest method createResultPartition.

private static ResultPartition createResultPartition() throws IOException {
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(10).setBufferSize(BUFFER_SIZE).build();
    ResultPartition resultPartition = createPartition(network, NoOpFileChannelManager.INSTANCE, ResultPartitionType.PIPELINED, 2);
    resultPartition.setup();
    return resultPartition;
}

Example 3 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class PartitionRequestQueueTest method createFinishedPartitionWithFilledData.

private static ResultPartition createFinishedPartitionWithFilledData(ResultPartitionManager partitionManager) throws Exception {
    NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().setResultPartitionManager(partitionManager).build();
    ResultPartition partition = createPartition(environment, fileChannelManager, ResultPartitionType.BLOCKING, 1);
    partition.setup();
    partition.emitRecord(ByteBuffer.allocate(BUFFER_SIZE), 0);
    partition.finish();
    return partition;
}

Example 4 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class InputBuffersMetricsTest method testCalculateTotalBuffersSize.

@Test
public void testCalculateTotalBuffersSize() throws Exception {
    int numberOfRemoteChannels = 2;
    int numberOfLocalChannels = 0;
    int numberOfBufferPerChannel = 2;
    int numberOfBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(numberOfBufferPerChannel).setFloatingNetworkBuffersPerGate(numberOfBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    SingleInputGate inputGate1 = buildInputGate(network, numberOfRemoteChannels, numberOfLocalChannels).f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    inputGate1.setup();
    SingleInputGate[] inputGates = new SingleInputGate[] { inputGate1 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBufferPoolUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    closeableRegistry.registerCloseable(network::close);
    closeableRegistry.registerCloseable(inputGate1::close);
    assertEquals(numberOfBuffersPerGate, floatingBuffersUsageGauge.calculateTotalBuffers(inputGate1));
    assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel, exclusiveBuffersUsageGauge.calculateTotalBuffers(inputGate1));
    assertEquals(numberOfRemoteChannels * numberOfBufferPerChannel + numberOfBuffersPerGate, inputBufferPoolUsageGauge.calculateTotalBuffers(inputGate1));
}

Example 5 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class InputBuffersMetricsTest method testExclusiveBuffersUsage.

@Test
public void testExclusiveBuffersUsage() throws Exception {
    int numberOfRemoteChannelsGate1 = 2;
    int numberOfLocalChannelsGate1 = 0;
    int numberOfRemoteChannelsGate2 = 1;
    int numberOfLocalChannelsGate2 = 1;
    int totalNumberOfRemoteChannels = numberOfRemoteChannelsGate1 + numberOfRemoteChannelsGate2;
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(buffersPerChannel).setFloatingNetworkBuffersPerGate(extraNetworkBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple1 = buildInputGate(network, numberOfRemoteChannelsGate1, numberOfLocalChannelsGate1);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple2 = buildInputGate(network, numberOfRemoteChannelsGate2, numberOfLocalChannelsGate2);
    SingleInputGate inputGate1 = tuple1.f0;
    SingleInputGate inputGate2 = tuple2.f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    closeableRegistry.registerCloseable(inputGate2::close);
    inputGate1.setup();
    inputGate2.setup();
    List<RemoteInputChannel> remoteInputChannels = tuple1.f1;
    SingleInputGate[] inputGates = new SingleInputGate[] { tuple1.f0, tuple2.f0 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    assertEquals(0.0, exclusiveBuffersUsageGauge.getValue(), 0.0);
    assertEquals(0.0, inputBuffersUsageGauge.getValue(), 0.0);
    int totalBuffers = extraNetworkBuffersPerGate * inputGates.length + buffersPerChannel * totalNumberOfRemoteChannels;
    int channelIndex = 1;
    for (RemoteInputChannel channel : remoteInputChannels) {
        drainAndValidate(buffersPerChannel, buffersPerChannel * channelIndex++, channel, totalBuffers, buffersPerChannel * totalNumberOfRemoteChannels, exclusiveBuffersUsageGauge, inputBuffersUsageGauge, inputGate1);
    }
}