Examples with SubpartitionInfo org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate.SubpartitionInfo used on opensource projects

Example 1 with SubpartitionInfo

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate.SubpartitionInfo in project flink by apache.

the class SingleInputGateTest method testSingleInputGateWithSubpartitionIndexRange.

@Test
public void testSingleInputGateWithSubpartitionIndexRange() throws IOException, InterruptedException {
    IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
    SubpartitionIndexRange subpartitionIndexRange = new SubpartitionIndexRange(0, 1);
    final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().build();
    ResourceID localLocation = ResourceID.generate();
    SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.BLOCKING, subpartitionIndexRange, netEnv, localLocation, new TestingConnectionManager(), new TestingResultPartitionManager(new NoOpResultSubpartitionView()));
    for (InputChannel channel : gate.getInputChannels().values()) {
        if (channel instanceof ChannelStateHolder) {
            ((ChannelStateHolder) channel).setChannelStateWriter(ChannelStateWriter.NO_OP);
        }
    }
    SubpartitionInfo info1 = createSubpartitionInfo(partitionIds[0], 0);
    SubpartitionInfo info2 = createSubpartitionInfo(partitionIds[0], 1);
    SubpartitionInfo info3 = createSubpartitionInfo(partitionIds[1], 0);
    SubpartitionInfo info4 = createSubpartitionInfo(partitionIds[1], 1);
    SubpartitionInfo info5 = createSubpartitionInfo(partitionIds[2], 0);
    SubpartitionInfo info6 = createSubpartitionInfo(partitionIds[2], 1);
    assertThat(gate.getInputChannels().size(), is(6));
    assertThat(gate.getInputChannels().get(info1).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info2).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info3).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info4).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info5).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info6).getConsumedSubpartitionIndex(), is(1));
    assertChannelsType(gate, LocalRecoveredInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteRecoveredInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    // test setup
    gate.setup();
    assertNotNull(gate.getBufferPool());
    assertEquals(1, gate.getBufferPool().getNumberOfRequiredMemorySegments());
    gate.finishReadRecoveredState();
    while (!gate.getStateConsumedFuture().isDone()) {
        gate.pollNext();
    }
    // test request partitions
    gate.requestPartitions();
    gate.pollNext();
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    for (InputChannel inputChannel : gate.getInputChannels().values()) {
        if (inputChannel instanceof RemoteInputChannel) {
            assertNotNull(((RemoteInputChannel) inputChannel).getPartitionRequestClient());
            assertEquals(2, ((RemoteInputChannel) inputChannel).getInitialCredit());
        } else if (inputChannel instanceof LocalInputChannel) {
            assertNotNull(((LocalInputChannel) inputChannel).getSubpartitionView());
        }
    }
    // test update channels
    gate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(partitionIds[2], localLocation));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info5, info6));
}

Example 2 with SubpartitionInfo

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate.SubpartitionInfo in project flink by apache.

the class SingleInputGateTest method testRequestBackoffConfiguration.

/**
 * Tests request back off configuration is correctly forwarded to the channels.
 */
@Test
public void testRequestBackoffConfiguration() throws Exception {
    IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
    int initialBackoff = 137;
    int maxBackoff = 1001;
    final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().setPartitionRequestInitialBackoff(initialBackoff).setPartitionRequestMaxBackoff(maxBackoff).build();
    SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.PIPELINED, netEnv);
    gate.setChannelStateWriter(ChannelStateWriter.NO_OP);
    gate.finishReadRecoveredState();
    while (!gate.getStateConsumedFuture().isDone()) {
        gate.pollNext();
    }
    gate.convertRecoveredInputChannels();
    try (Closer closer = Closer.create()) {
        closer.register(netEnv::close);
        closer.register(gate::close);
        assertEquals(ResultPartitionType.PIPELINED, gate.getConsumedPartitionType());
        Map<SubpartitionInfo, InputChannel> channelMap = gate.getInputChannels();
        assertEquals(3, channelMap.size());
        channelMap.values().forEach(channel -> {
            try {
                channel.checkError();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        });
        InputChannel localChannel = channelMap.get(createSubpartitionInfo(partitionIds[0]));
        assertEquals(LocalInputChannel.class, localChannel.getClass());
        InputChannel remoteChannel = channelMap.get(createSubpartitionInfo(partitionIds[1]));
        assertEquals(RemoteInputChannel.class, remoteChannel.getClass());
        InputChannel unknownChannel = channelMap.get(createSubpartitionInfo(partitionIds[2]));
        assertEquals(UnknownInputChannel.class, unknownChannel.getClass());
        InputChannel[] channels = new InputChannel[] { localChannel, remoteChannel, unknownChannel };
        for (InputChannel ch : channels) {
            assertEquals(0, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff * 2, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(initialBackoff * 2 * 2, ch.getCurrentBackoff());
            assertTrue(ch.increaseBackoff());
            assertEquals(maxBackoff, ch.getCurrentBackoff());
            assertFalse(ch.increaseBackoff());
        }
    }
}