Examples with NettyShuffleEnvironment org.apache.flink.runtime.io.network.NettyShuffleEnvironment used on opensource projects

Example 21 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class SingleInputGateTest method testRequestBuffersWithUnknownInputChannel.

/**
 * Tests that input gate requests and assigns network buffers when unknown input channel updates
 * to remote input channel.
 */
@Test
public void testRequestBuffersWithUnknownInputChannel() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final SingleInputGate inputGate = createInputGate(network, 1, ResultPartitionType.PIPELINED_BOUNDED);
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        final ResultPartitionID resultPartitionId = new ResultPartitionID();
        InputChannel inputChannel = buildUnknownInputChannel(network, inputGate, resultPartitionId, 0);
        inputGate.setInputChannels(inputChannel);
        inputGate.setup();
        NetworkBufferPool bufferPool = network.getNetworkBufferPool();
        assertEquals(bufferPool.getTotalNumberOfMemorySegments() - 1, bufferPool.getNumberOfAvailableMemorySegments());
        // note: exclusive buffers are not handed out into LocalBufferPool and are thus not
        // counted
        assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
        // Trigger updates to remote input channel from unknown input channel
        inputGate.updateInputChannel(ResourceID.generate(), createRemoteWithIdAndLocation(resultPartitionId.getPartitionId(), ResourceID.generate()));
        RemoteInputChannel remote = (RemoteInputChannel) inputGate.getInputChannels().get(createSubpartitionInfo(resultPartitionId.getPartitionId()));
        // only the exclusive buffers should be assigned/available now
        assertEquals(buffersPerChannel, remote.getNumberOfAvailableBuffers());
        assertEquals(bufferPool.getTotalNumberOfMemorySegments() - buffersPerChannel - 1, bufferPool.getNumberOfAvailableMemorySegments());
        // note: exclusive buffers are not handed out into LocalBufferPool and are thus not
        // counted
        assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
    }
}

Example 22 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class SingleInputGateTest method testUpdateUnknownInputChannel.

/**
 * Tests that input gate can successfully convert unknown input channels into local and remote
 * channels.
 */
@Test
public void testUpdateUnknownInputChannel() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final ResultPartition localResultPartition = new ResultPartitionBuilder().setResultPartitionManager(network.getResultPartitionManager()).setupBufferPoolFactoryFromNettyShuffleEnvironment(network).build();
    final ResultPartition remoteResultPartition = new ResultPartitionBuilder().setResultPartitionManager(network.getResultPartitionManager()).setupBufferPoolFactoryFromNettyShuffleEnvironment(network).build();
    localResultPartition.setup();
    remoteResultPartition.setup();
    final SingleInputGate inputGate = createInputGate(network, 2, ResultPartitionType.PIPELINED);
    final InputChannel[] inputChannels = new InputChannel[2];
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        final ResultPartitionID localResultPartitionId = localResultPartition.getPartitionId();
        inputChannels[0] = buildUnknownInputChannel(network, inputGate, localResultPartitionId, 0);
        final ResultPartitionID remoteResultPartitionId = remoteResultPartition.getPartitionId();
        inputChannels[1] = buildUnknownInputChannel(network, inputGate, remoteResultPartitionId, 1);
        inputGate.setInputChannels(inputChannels);
        inputGate.setup();
        assertThat(inputGate.getInputChannels().get(createSubpartitionInfo(remoteResultPartitionId.getPartitionId())), is(instanceOf((UnknownInputChannel.class))));
        assertThat(inputGate.getInputChannels().get(createSubpartitionInfo(localResultPartitionId.getPartitionId())), is(instanceOf((UnknownInputChannel.class))));
        ResourceID localLocation = ResourceID.generate();
        // Trigger updates to remote input channel from unknown input channel
        inputGate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(remoteResultPartitionId.getPartitionId(), ResourceID.generate()));
        assertThat(inputGate.getInputChannels().get(createSubpartitionInfo(remoteResultPartitionId.getPartitionId())), is(instanceOf((RemoteInputChannel.class))));
        assertThat(inputGate.getInputChannels().get(createSubpartitionInfo(localResultPartitionId.getPartitionId())), is(instanceOf((UnknownInputChannel.class))));
        // Trigger updates to local input channel from unknown input channel
        inputGate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(localResultPartitionId.getPartitionId(), localLocation));
        assertThat(inputGate.getInputChannels().get(createSubpartitionInfo(remoteResultPartitionId.getPartitionId())), is(instanceOf((RemoteInputChannel.class))));
        assertThat(inputGate.getInputChannels().get(createSubpartitionInfo(localResultPartitionId.getPartitionId())), is(instanceOf((LocalInputChannel.class))));
    }
}

Example 23 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class SingleInputGateTest method testRequestBuffersWithRemoteInputChannel.

/**
 * Tests that input gate requests and assigns network buffers for remote input channel.
 */
@Test
public void testRequestBuffersWithRemoteInputChannel() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final SingleInputGate inputGate = createInputGate(network, 1, ResultPartitionType.PIPELINED_BOUNDED);
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        RemoteInputChannel remote = InputChannelBuilder.newBuilder().setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
        inputGate.setInputChannels(remote);
        inputGate.setup();
        NetworkBufferPool bufferPool = network.getNetworkBufferPool();
        // only the exclusive buffers should be assigned/available now
        assertEquals(buffersPerChannel, remote.getNumberOfAvailableBuffers());
        assertEquals(bufferPool.getTotalNumberOfMemorySegments() - buffersPerChannel - 1, bufferPool.getNumberOfAvailableMemorySegments());
        // note: exclusive buffers are not handed out into LocalBufferPool and are thus not
        // counted
        assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
    }
}

Example 24 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class SingleInputGateTest method testRequestBackoffConfiguration.

/**
 * Tests request back off configuration is correctly forwarded to the channels.
 */
@Test
public void testRequestBackoffConfiguration() throws Exception {
    IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
    int initialBackoff = 137;
    int maxBackoff = 1001;
    final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().setPartitionRequestInitialBackoff(initialBackoff).setPartitionRequestMaxBackoff(maxBackoff).build();
    SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.PIPELINED, netEnv);
    gate.setChannelStateWriter(ChannelStateWriter.NO_OP);
    gate.finishReadRecoveredState();
    while (!gate.getStateConsumedFuture().isDone()) {
        gate.pollNext();
    }
    gate.convertRecoveredInputChannels();
    try (Closer closer = Closer.create()) {
        closer.register(netEnv::close);
        closer.register(gate::close);
        assertEquals(ResultPartitionType.PIPELINED, gate.getConsumedPartitionType());
        Map<SubpartitionInfo, InputChannel> channelMap = gate.getInputChannels();
        assertEquals(3, channelMap.size());
        channelMap.values().forEach(channel -> {
            try {
                channel.checkError();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        });
        InputChannel localChannel = channelMap.get(createSubpartitionInfo(partitionIds[0]));
        assertEquals(LocalInputChannel.class, localChannel.getClass());
        InputChannel remoteChannel = channelMap.get(createSubpartitionInfo(partitionIds[1]));
        assertEquals(RemoteInputChannel.class, remoteChannel.getClass());
        InputChannel unknownChannel = channelMap.get(createSubpartitionInfo(partitionIds[2]));
        assertEquals(UnknownInputChannel.class, unknownChannel.getClass());
        InputChannel[] channels = new InputChannel[] { localChannel, remoteChannel, unknownChannel };
        for (InputChannel ch : channels) {
            assertEquals(0, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff * 2, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff * 2 * 2, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(maxBackoff, ch.getCurrentBackoff());
            assertFalse(ch.increaseBackoff());
        }
    }
}

Example 25 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class SingleInputGateTest method testSetupLogic.

/**
 * Tests {@link InputGate#setup()} should create the respective {@link BufferPool} and assign
 * exclusive buffers for {@link RemoteInputChannel}s, but should not request partitions.
 */
@Test
public void testSetupLogic() throws Exception {
    final NettyShuffleEnvironment environment = createNettyShuffleEnvironment();
    final SingleInputGate inputGate = createInputGate(environment);
    try (Closer closer = Closer.create()) {
        closer.register(environment::close);
        closer.register(inputGate::close);
        // before setup
        assertNull(inputGate.getBufferPool());
        for (InputChannel inputChannel : inputGate.getInputChannels().values()) {
            assertTrue(inputChannel instanceof RecoveredInputChannel || inputChannel instanceof UnknownInputChannel);
            if (inputChannel instanceof RecoveredInputChannel) {
                assertEquals(0, ((RecoveredInputChannel) inputChannel).bufferManager.getNumberOfAvailableBuffers());
            }
        }
        inputGate.setup();
        // after setup
        assertNotNull(inputGate.getBufferPool());
        assertEquals(1, inputGate.getBufferPool().getNumberOfRequiredMemorySegments());
        for (InputChannel inputChannel : inputGate.getInputChannels().values()) {
            if (inputChannel instanceof RemoteRecoveredInputChannel) {
                assertEquals(0, ((RemoteRecoveredInputChannel) inputChannel).bufferManager.getNumberOfAvailableBuffers());
            } else if (inputChannel instanceof LocalRecoveredInputChannel) {
                assertEquals(0, ((LocalRecoveredInputChannel) inputChannel).bufferManager.getNumberOfAvailableBuffers());
            }
        }
        inputGate.convertRecoveredInputChannels();
        assertNotNull(inputGate.getBufferPool());
        assertEquals(1, inputGate.getBufferPool().getNumberOfRequiredMemorySegments());
        for (InputChannel inputChannel : inputGate.getInputChannels().values()) {
            if (inputChannel instanceof RemoteInputChannel) {
                assertEquals(2, ((RemoteInputChannel) inputChannel).getNumberOfAvailableBuffers());
            }
        }
    }
}