use of org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView in project flink by apache.
the class SingleInputGateTest method testBackwardsEventWithUninitializedChannel.
@Test
public void testBackwardsEventWithUninitializedChannel() throws Exception {
// Setup environment
TestingTaskEventPublisher taskEventPublisher = new TestingTaskEventPublisher();
TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(new NoOpResultSubpartitionView());
// Setup reader with one local and one unknown input channel
NettyShuffleEnvironment environment = createNettyShuffleEnvironment();
final SingleInputGate inputGate = createInputGate(environment, 2, ResultPartitionType.PIPELINED);
final InputChannel[] inputChannels = new InputChannel[2];
try (Closer closer = Closer.create()) {
closer.register(environment::close);
closer.register(inputGate::close);
// Local
ResultPartitionID localPartitionId = new ResultPartitionID();
inputChannels[0] = InputChannelBuilder.newBuilder().setPartitionId(localPartitionId).setPartitionManager(partitionManager).setTaskEventPublisher(taskEventPublisher).buildLocalChannel(inputGate);
// Unknown
ResultPartitionID unknownPartitionId = new ResultPartitionID();
inputChannels[1] = InputChannelBuilder.newBuilder().setChannelIndex(1).setPartitionId(unknownPartitionId).setPartitionManager(partitionManager).setTaskEventPublisher(taskEventPublisher).buildUnknownChannel(inputGate);
setupInputGate(inputGate, inputChannels);
// Only the local channel can request
assertEquals(1, partitionManager.counter);
// Send event backwards and initialize unknown channel afterwards
final TaskEvent event = new TestTaskEvent();
inputGate.sendTaskEvent(event);
// Only the local channel can send out the event
assertEquals(1, taskEventPublisher.counter);
// After the update, the pending event should be send to local channel
ResourceID location = ResourceID.generate();
inputGate.updateInputChannel(location, createRemoteWithIdAndLocation(unknownPartitionId.getPartitionId(), location));
assertEquals(2, partitionManager.counter);
assertEquals(2, taskEventPublisher.counter);
}
}
use of org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView in project flink by apache.
the class SingleInputGateTest method testUpdateChannelBeforeRequest.
/**
* Tests that an update channel does not trigger a partition request before the UDF has
* requested any partitions. Otherwise, this can lead to races when registering a listener at
* the gate (e.g. in UnionInputGate), which can result in missed buffer notifications at the
* listener.
*/
@Test
public void testUpdateChannelBeforeRequest() throws Exception {
SingleInputGate inputGate = createInputGate(1);
TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(new NoOpResultSubpartitionView());
InputChannel unknown = InputChannelBuilder.newBuilder().setPartitionManager(partitionManager).buildUnknownChannel(inputGate);
inputGate.setInputChannels(unknown);
// Update to a local channel and verify that no request is triggered
ResultPartitionID resultPartitionID = unknown.getPartitionId();
ResourceID location = ResourceID.generate();
inputGate.updateInputChannel(location, createRemoteWithIdAndLocation(resultPartitionID.getPartitionId(), location));
assertEquals(0, partitionManager.counter);
}
use of org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView in project flink by apache.
the class SingleInputGateTest method testSingleInputGateWithSubpartitionIndexRange.
@Test
public void testSingleInputGateWithSubpartitionIndexRange() throws IOException, InterruptedException {
IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
SubpartitionIndexRange subpartitionIndexRange = new SubpartitionIndexRange(0, 1);
final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().build();
ResourceID localLocation = ResourceID.generate();
SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.BLOCKING, subpartitionIndexRange, netEnv, localLocation, new TestingConnectionManager(), new TestingResultPartitionManager(new NoOpResultSubpartitionView()));
for (InputChannel channel : gate.getInputChannels().values()) {
if (channel instanceof ChannelStateHolder) {
((ChannelStateHolder) channel).setChannelStateWriter(ChannelStateWriter.NO_OP);
}
}
SubpartitionInfo info1 = createSubpartitionInfo(partitionIds[0], 0);
SubpartitionInfo info2 = createSubpartitionInfo(partitionIds[0], 1);
SubpartitionInfo info3 = createSubpartitionInfo(partitionIds[1], 0);
SubpartitionInfo info4 = createSubpartitionInfo(partitionIds[1], 1);
SubpartitionInfo info5 = createSubpartitionInfo(partitionIds[2], 0);
SubpartitionInfo info6 = createSubpartitionInfo(partitionIds[2], 1);
assertThat(gate.getInputChannels().size(), is(6));
assertThat(gate.getInputChannels().get(info1).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info2).getConsumedSubpartitionIndex(), is(1));
assertThat(gate.getInputChannels().get(info3).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info4).getConsumedSubpartitionIndex(), is(1));
assertThat(gate.getInputChannels().get(info5).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info6).getConsumedSubpartitionIndex(), is(1));
assertChannelsType(gate, LocalRecoveredInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteRecoveredInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
// test setup
gate.setup();
assertNotNull(gate.getBufferPool());
assertEquals(1, gate.getBufferPool().getNumberOfRequiredMemorySegments());
gate.finishReadRecoveredState();
while (!gate.getStateConsumedFuture().isDone()) {
gate.pollNext();
}
// test request partitions
gate.requestPartitions();
gate.pollNext();
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
for (InputChannel inputChannel : gate.getInputChannels().values()) {
if (inputChannel instanceof RemoteInputChannel) {
assertNotNull(((RemoteInputChannel) inputChannel).getPartitionRequestClient());
assertEquals(2, ((RemoteInputChannel) inputChannel).getInitialCredit());
} else if (inputChannel instanceof LocalInputChannel) {
assertNotNull(((LocalInputChannel) inputChannel).getSubpartitionView());
}
}
// test update channels
gate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(partitionIds[2], localLocation));
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info5, info6));
}
use of org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView in project flink by apache.
the class UnionInputGateTest method testUpdateInputChannel.
@Test
public void testUpdateInputChannel() throws Exception {
final SingleInputGate inputGate1 = createInputGate(1);
TestInputChannel inputChannel1 = new TestInputChannel(inputGate1, 0);
inputGate1.setInputChannels(inputChannel1);
final SingleInputGate inputGate2 = createInputGate(1);
TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(new NoOpResultSubpartitionView());
InputChannel unknownInputChannel2 = InputChannelBuilder.newBuilder().setPartitionManager(partitionManager).buildUnknownChannel(inputGate2);
inputGate2.setInputChannels(unknownInputChannel2);
UnionInputGate unionInputGate = new UnionInputGate(inputGate1, inputGate2);
ResultPartitionID resultPartitionID = unknownInputChannel2.getPartitionId();
ResourceID location = ResourceID.generate();
inputGate2.updateInputChannel(location, createRemoteWithIdAndLocation(resultPartitionID.getPartitionId(), location));
assertThat(unionInputGate.getChannel(0), Matchers.is(inputChannel1));
// Check that updated input channel is visible via UnionInputGate
assertThat(unionInputGate.getChannel(1), Matchers.is(inputGate2.getChannel(0)));
}
use of org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView in project flink by apache.
the class CreditBasedSequenceNumberingViewReaderTest method createNetworkSequenceViewReader.
private CreditBasedSequenceNumberingViewReader createNetworkSequenceViewReader(int initialCredit) throws Exception {
PartitionRequestQueue queue = new PartitionRequestQueue();
EmbeddedChannel channel = new EmbeddedChannel(queue);
channel.close();
CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(new InputChannelID(), initialCredit, queue);
reader.requestSubpartitionView((ignored1, ignored2, ignored3) -> new NoOpResultSubpartitionView(), new ResultPartitionID(), 0);
return reader;
}
Aggregations