Examples with NettyShuffleEnvironmentBuilder org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder used on opensource projects

Example 26 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class SingleInputGateTest method testPartitionRequestLogic.

@Test
public void testPartitionRequestLogic() throws Exception {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    final SingleInputGate gate = createInputGate(environment);
    try (Closer closer = Closer.create()) {
        closer.register(environment::close);
        closer.register(gate::close);
        gate.finishReadRecoveredState();
        while (!gate.getStateConsumedFuture().isDone()) {
            gate.pollNext();
        }
        gate.requestPartitions();
        // check channel error during above partition request
        gate.pollNext();
        final InputChannel remoteChannel = gate.getChannel(0);
        assertThat(remoteChannel, instanceOf(RemoteInputChannel.class));
        assertNotNull(((RemoteInputChannel) remoteChannel).getPartitionRequestClient());
        assertEquals(2, ((RemoteInputChannel) remoteChannel).getInitialCredit());
        final InputChannel localChannel = gate.getChannel(1);
        assertThat(localChannel, instanceOf(LocalInputChannel.class));
        assertNotNull(((LocalInputChannel) localChannel).getSubpartitionView());
        assertThat(gate.getChannel(2), instanceOf(UnknownInputChannel.class));
    }
}

Example 27 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class SingleInputGateTest method testRequestBackoffConfiguration.

/**
 * Tests request back off configuration is correctly forwarded to the channels.
 */
@Test
public void testRequestBackoffConfiguration() throws Exception {
    IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
    int initialBackoff = 137;
    int maxBackoff = 1001;
    final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().setPartitionRequestInitialBackoff(initialBackoff).setPartitionRequestMaxBackoff(maxBackoff).build();
    SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.PIPELINED, netEnv);
    gate.setChannelStateWriter(ChannelStateWriter.NO_OP);
    gate.finishReadRecoveredState();
    while (!gate.getStateConsumedFuture().isDone()) {
        gate.pollNext();
    }
    gate.convertRecoveredInputChannels();
    try (Closer closer = Closer.create()) {
        closer.register(netEnv::close);
        closer.register(gate::close);
        assertEquals(ResultPartitionType.PIPELINED, gate.getConsumedPartitionType());
        Map<SubpartitionInfo, InputChannel> channelMap = gate.getInputChannels();
        assertEquals(3, channelMap.size());
        channelMap.values().forEach(channel -> {
            try {
                channel.checkError();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        });
        InputChannel localChannel = channelMap.get(createSubpartitionInfo(partitionIds[0]));
        assertEquals(LocalInputChannel.class, localChannel.getClass());
        InputChannel remoteChannel = channelMap.get(createSubpartitionInfo(partitionIds[1]));
        assertEquals(RemoteInputChannel.class, remoteChannel.getClass());
        InputChannel unknownChannel = channelMap.get(createSubpartitionInfo(partitionIds[2]));
        assertEquals(UnknownInputChannel.class, unknownChannel.getClass());
        InputChannel[] channels = new InputChannel[] { localChannel, remoteChannel, unknownChannel };
        for (InputChannel ch : channels) {
            assertEquals(0, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff * 2, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff * 2 * 2, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(maxBackoff, ch.getCurrentBackoff());
            assertFalse(ch.increaseBackoff());
        }
    }
}

Example 28 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class TaskExecutorPartitionTrackerImplTest method createClusterPartitionReport.

@Test
public void createClusterPartitionReport() {
    final TaskExecutorPartitionTrackerImpl partitionTracker = new TaskExecutorPartitionTrackerImpl(new NettyShuffleEnvironmentBuilder().build());
    assertThat(partitionTracker.createClusterPartitionReport().getEntries(), is(empty()));
    final IntermediateDataSetID dataSetId = new IntermediateDataSetID();
    final JobID jobId = new JobID();
    final ResultPartitionID clusterPartitionId = new ResultPartitionID();
    final ResultPartitionID jobPartitionId = new ResultPartitionID();
    final int numberOfPartitions = 1;
    partitionTracker.startTrackingPartition(jobId, new TaskExecutorPartitionInfo(clusterPartitionId, dataSetId, numberOfPartitions));
    partitionTracker.startTrackingPartition(jobId, new TaskExecutorPartitionInfo(jobPartitionId, dataSetId, numberOfPartitions + 1));
    partitionTracker.promoteJobPartitions(Collections.singleton(clusterPartitionId));
    final ClusterPartitionReport clusterPartitionReport = partitionTracker.createClusterPartitionReport();
    final ClusterPartitionReport.ClusterPartitionReportEntry reportEntry = Iterables.getOnlyElement(clusterPartitionReport.getEntries());
    assertThat(reportEntry.getDataSetId(), is(dataSetId));
    assertThat(reportEntry.getNumTotalPartitions(), is(numberOfPartitions));
    assertThat(reportEntry.getHostedPartitions(), hasItems(clusterPartitionId));
}

Example 29 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class ResultPartitionTest method testIsAvailableOrNot.

/**
 * Tests {@link ResultPartition#getAvailableFuture()}.
 */
@Test
public void testIsAvailableOrNot() throws IOException {
    final int numAllBuffers = 10;
    final int bufferSize = 1024;
    final NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(numAllBuffers).setBufferSize(bufferSize).build();
    final ResultPartition resultPartition = createPartition(network, ResultPartitionType.PIPELINED, 1);
    try {
        resultPartition.setup();
        resultPartition.getBufferPool().setNumBuffers(2);
        assertTrue(resultPartition.getAvailableFuture().isDone());
        resultPartition.emitRecord(ByteBuffer.allocate(bufferSize), 0);
        resultPartition.emitRecord(ByteBuffer.allocate(bufferSize), 0);
        assertFalse(resultPartition.getAvailableFuture().isDone());
    } finally {
        resultPartition.release();
        network.close();
    }
}

Example 30 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class InputBuffersMetricsTest method testFloatingBuffersUsage.

@Test
public void testFloatingBuffersUsage() throws Exception {
    int numberOfRemoteChannelsGate1 = 2;
    int numberOfLocalChannelsGate1 = 0;
    int numberOfRemoteChannelsGate2 = 1;
    int numberOfLocalChannelsGate2 = 1;
    int totalNumberOfRemoteChannels = numberOfRemoteChannelsGate1 + numberOfRemoteChannelsGate2;
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNetworkBuffersPerChannel(buffersPerChannel).setFloatingNetworkBuffersPerGate(extraNetworkBuffersPerGate).build();
    closeableRegistry.registerCloseable(network::close);
    Tuple2<SingleInputGate, List<RemoteInputChannel>> tuple1 = buildInputGate(network, numberOfRemoteChannelsGate1, numberOfLocalChannelsGate1);
    SingleInputGate inputGate2 = buildInputGate(network, numberOfRemoteChannelsGate2, numberOfLocalChannelsGate2).f0;
    SingleInputGate inputGate1 = tuple1.f0;
    closeableRegistry.registerCloseable(inputGate1::close);
    closeableRegistry.registerCloseable(inputGate2::close);
    inputGate1.setup();
    inputGate2.setup();
    RemoteInputChannel remoteInputChannel1 = tuple1.f1.get(0);
    SingleInputGate[] inputGates = new SingleInputGate[] { tuple1.f0, inputGate2 };
    FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
    ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
    CreditBasedInputBuffersUsageGauge inputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(floatingBuffersUsageGauge, exclusiveBuffersUsageGauge, inputGates);
    assertEquals(0.0, floatingBuffersUsageGauge.getValue(), 0.0);
    assertEquals(0.0, inputBuffersUsageGauge.getValue(), 0.0);
    // drain gate1's exclusive buffers
    drainBuffer(buffersPerChannel, remoteInputChannel1);
    int totalBuffers = extraNetworkBuffersPerGate * inputGates.length + buffersPerChannel * totalNumberOfRemoteChannels;
    remoteInputChannel1.requestSubpartition();
    int backlog = 3;
    int totalRequestedBuffers = buffersPerChannel + backlog;
    remoteInputChannel1.onSenderBacklog(backlog);
    assertEquals(totalRequestedBuffers, remoteInputChannel1.unsynchronizedGetFloatingBuffersAvailable());
    drainBuffer(totalRequestedBuffers, remoteInputChannel1);
    assertEquals(0, remoteInputChannel1.unsynchronizedGetFloatingBuffersAvailable());
    assertEquals((double) (buffersPerChannel + totalRequestedBuffers) / totalBuffers, inputBuffersUsageGauge.getValue(), 0.0001);
}