use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class AlignedCheckpointsTest method createCheckpointedInputGate.
private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
gate.setup();
gate.requestPartitions();
return createCheckpointedInputGate(gate, toNotify);
}
use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class RemoteInputChannelTest method testOnFailedPartitionRequestDoesNotBlockNetworkThreads.
/**
* Test to guard against FLINK-13249.
*/
@Test
public void testOnFailedPartitionRequestDoesNotBlockNetworkThreads() throws Exception {
final long testBlockedWaitTimeoutMillis = 30_000L;
final PartitionProducerStateChecker partitionProducerStateChecker = (jobId, intermediateDataSetId, resultPartitionId) -> CompletableFuture.completedFuture(ExecutionState.RUNNING);
final NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
final Task task = new TestTaskBuilder(shuffleEnvironment).setPartitionProducerStateChecker(partitionProducerStateChecker).build();
final SingleInputGate inputGate = new SingleInputGateBuilder().setPartitionProducerStateProvider(task).build();
TestTaskBuilder.setTaskState(task, ExecutionState.RUNNING);
final OneShotLatch ready = new OneShotLatch();
final OneShotLatch blocker = new OneShotLatch();
final AtomicBoolean timedOutOrInterrupted = new AtomicBoolean(false);
final ConnectionManager blockingConnectionManager = new TestingConnectionManager() {
@Override
public PartitionRequestClient createPartitionRequestClient(ConnectionID connectionId) {
ready.trigger();
try {
// We block here, in a section that holds the
// SingleInputGate#requestLock
blocker.await(testBlockedWaitTimeoutMillis, TimeUnit.MILLISECONDS);
} catch (InterruptedException | TimeoutException e) {
timedOutOrInterrupted.set(true);
}
return new TestingPartitionRequestClient();
}
};
final RemoteInputChannel remoteInputChannel = InputChannelBuilder.newBuilder().setConnectionManager(blockingConnectionManager).buildRemoteChannel(inputGate);
inputGate.setInputChannels(remoteInputChannel);
final Thread simulatedNetworkThread = new Thread(() -> {
try {
ready.await();
// We want to make sure that our simulated network thread does not
// block on
// SingleInputGate#requestLock as well through this call.
remoteInputChannel.onFailedPartitionRequest();
// Will only give free the blocker if we did not block ourselves.
blocker.trigger();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
simulatedNetworkThread.start();
// The entry point to that will lead us into
// blockingConnectionManager#createPartitionRequestClient(...).
inputGate.requestPartitions();
simulatedNetworkThread.join();
Assert.assertFalse("Test ended by timeout or interruption - this indicates that the network thread was blocked.", timedOutOrInterrupted.get());
}
use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class SingleInputGateTest method testRequestBuffersWithRemoteInputChannel.
/**
* Tests that input gate requests and assigns network buffers for remote input channel.
*/
@Test
public void testRequestBuffersWithRemoteInputChannel() throws Exception {
final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
final SingleInputGate inputGate = createInputGate(network, 1, ResultPartitionType.PIPELINED_BOUNDED);
int buffersPerChannel = 2;
int extraNetworkBuffersPerGate = 8;
try (Closer closer = Closer.create()) {
closer.register(network::close);
closer.register(inputGate::close);
RemoteInputChannel remote = InputChannelBuilder.newBuilder().setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
inputGate.setInputChannels(remote);
inputGate.setup();
NetworkBufferPool bufferPool = network.getNetworkBufferPool();
// only the exclusive buffers should be assigned/available now
assertEquals(buffersPerChannel, remote.getNumberOfAvailableBuffers());
assertEquals(bufferPool.getTotalNumberOfMemorySegments() - buffersPerChannel - 1, bufferPool.getNumberOfAvailableMemorySegments());
// note: exclusive buffers are not handed out into LocalBufferPool and are thus not
// counted
assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
}
}
use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.
the class CheckpointBarrierTrackerTest method createCheckpointedInputGate.
// ------------------------------------------------------------------------
// Utils
// ------------------------------------------------------------------------
private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
gate.setup();
gate.requestPartitions();
return createCheckpointedInputGate(gate, toNotify);
}
Aggregations