use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class SingleInputGateTest method testSingleInputGateWithSubpartitionIndexRange.
@Test
public void testSingleInputGateWithSubpartitionIndexRange() throws IOException, InterruptedException {
IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
SubpartitionIndexRange subpartitionIndexRange = new SubpartitionIndexRange(0, 1);
final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().build();
ResourceID localLocation = ResourceID.generate();
SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.BLOCKING, subpartitionIndexRange, netEnv, localLocation, new TestingConnectionManager(), new TestingResultPartitionManager(new NoOpResultSubpartitionView()));
for (InputChannel channel : gate.getInputChannels().values()) {
if (channel instanceof ChannelStateHolder) {
((ChannelStateHolder) channel).setChannelStateWriter(ChannelStateWriter.NO_OP);
}
}
SubpartitionInfo info1 = createSubpartitionInfo(partitionIds[0], 0);
SubpartitionInfo info2 = createSubpartitionInfo(partitionIds[0], 1);
SubpartitionInfo info3 = createSubpartitionInfo(partitionIds[1], 0);
SubpartitionInfo info4 = createSubpartitionInfo(partitionIds[1], 1);
SubpartitionInfo info5 = createSubpartitionInfo(partitionIds[2], 0);
SubpartitionInfo info6 = createSubpartitionInfo(partitionIds[2], 1);
assertThat(gate.getInputChannels().size(), is(6));
assertThat(gate.getInputChannels().get(info1).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info2).getConsumedSubpartitionIndex(), is(1));
assertThat(gate.getInputChannels().get(info3).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info4).getConsumedSubpartitionIndex(), is(1));
assertThat(gate.getInputChannels().get(info5).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info6).getConsumedSubpartitionIndex(), is(1));
assertChannelsType(gate, LocalRecoveredInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteRecoveredInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
// test setup
gate.setup();
assertNotNull(gate.getBufferPool());
assertEquals(1, gate.getBufferPool().getNumberOfRequiredMemorySegments());
gate.finishReadRecoveredState();
while (!gate.getStateConsumedFuture().isDone()) {
gate.pollNext();
}
// test request partitions
gate.requestPartitions();
gate.pollNext();
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
for (InputChannel inputChannel : gate.getInputChannels().values()) {
if (inputChannel instanceof RemoteInputChannel) {
assertNotNull(((RemoteInputChannel) inputChannel).getPartitionRequestClient());
assertEquals(2, ((RemoteInputChannel) inputChannel).getInitialCredit());
} else if (inputChannel instanceof LocalInputChannel) {
assertNotNull(((LocalInputChannel) inputChannel).getSubpartitionView());
}
}
// test update channels
gate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(partitionIds[2], localLocation));
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info5, info6));
}
use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class SingleInputGateTest method testQueuedBuffers.
@Test
public void testQueuedBuffers() throws Exception {
final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
final BufferWritingResultPartition resultPartition = (BufferWritingResultPartition) new ResultPartitionBuilder().setResultPartitionManager(network.getResultPartitionManager()).setupBufferPoolFactoryFromNettyShuffleEnvironment(network).build();
final SingleInputGate inputGate = createInputGate(network, 2, ResultPartitionType.PIPELINED);
final ResultPartitionID localResultPartitionId = resultPartition.getPartitionId();
final InputChannel[] inputChannels = new InputChannel[2];
final RemoteInputChannel remoteInputChannel = InputChannelBuilder.newBuilder().setChannelIndex(1).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
inputChannels[0] = remoteInputChannel;
inputChannels[1] = InputChannelBuilder.newBuilder().setChannelIndex(0).setPartitionId(localResultPartitionId).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildLocalChannel(inputGate);
try (Closer closer = Closer.create()) {
closer.register(network::close);
closer.register(inputGate::close);
closer.register(resultPartition::release);
resultPartition.setup();
setupInputGate(inputGate, inputChannels);
remoteInputChannel.onBuffer(createBuffer(1), 0, 0);
assertEquals(1, inputGate.getNumberOfQueuedBuffers());
resultPartition.emitRecord(ByteBuffer.allocate(1), 0);
assertEquals(2, inputGate.getNumberOfQueuedBuffers());
}
}
use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class RemoteInputChannelTest method testPartitionNotFoundExceptionWhileRetriggeringRequest.
/**
* Tests that {@link RemoteInputChannel#retriggerSubpartitionRequest()} would throw the {@link
* PartitionNotFoundException} if backoff is 0.
*/
@Test
public void testPartitionNotFoundExceptionWhileRetriggeringRequest() throws Exception {
final RemoteInputChannel inputChannel = InputChannelTestUtils.createRemoteInputChannel(createSingleInputGate(1), 0, new TestingConnectionManager());
// Request partition to initialize client to avoid illegal state after retriggering
// partition
inputChannel.requestSubpartition();
// The default backoff is 0 then it would set PartitionNotFoundException on this channel
inputChannel.retriggerSubpartitionRequest();
try {
inputChannel.checkError();
fail("Should throw a PartitionNotFoundException.");
} catch (PartitionNotFoundException notFound) {
assertThat(inputChannel.getPartitionId(), is(notFound.getPartitionId()));
}
}
use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class UnalignedCheckpointsTest method createInputGate.
private CheckpointedInputGate createInputGate(int numberOfChannels, AbstractInvokable toNotify, boolean enableCheckpointsAfterTasksFinished) throws IOException {
final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setStateWriter(channelStateWriter).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
sequenceNumbers = new int[numberOfChannels];
gate.setup();
gate.requestPartitions();
return createCheckpointedInputGate(gate, toNotify, enableCheckpointsAfterTasksFinished);
}
use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.
private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
ConnectionManager connectionManager = new TestingConnectionManager();
SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
singleInputGate.setup();
MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
for (int i = 0; i < numberOfChannels; i++) {
((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
}
return checkpointedInputGate;
}
Aggregations