Search in sources :

Example 6 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class AlignedCheckpointsTest method createCheckpointedInputGate.

private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify);
}
Also used : SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Example 7 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testOnFailedPartitionRequestDoesNotBlockNetworkThreads.

/**
 * Test to guard against FLINK-13249.
 */
@Test
public void testOnFailedPartitionRequestDoesNotBlockNetworkThreads() throws Exception {
    final long testBlockedWaitTimeoutMillis = 30_000L;
    final PartitionProducerStateChecker partitionProducerStateChecker = (jobId, intermediateDataSetId, resultPartitionId) -> CompletableFuture.completedFuture(ExecutionState.RUNNING);
    final NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    final Task task = new TestTaskBuilder(shuffleEnvironment).setPartitionProducerStateChecker(partitionProducerStateChecker).build();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setPartitionProducerStateProvider(task).build();
    TestTaskBuilder.setTaskState(task, ExecutionState.RUNNING);
    final OneShotLatch ready = new OneShotLatch();
    final OneShotLatch blocker = new OneShotLatch();
    final AtomicBoolean timedOutOrInterrupted = new AtomicBoolean(false);
    final ConnectionManager blockingConnectionManager = new TestingConnectionManager() {

        @Override
        public PartitionRequestClient createPartitionRequestClient(ConnectionID connectionId) {
            ready.trigger();
            try {
                // We block here, in a section that holds the
                // SingleInputGate#requestLock
                blocker.await(testBlockedWaitTimeoutMillis, TimeUnit.MILLISECONDS);
            } catch (InterruptedException | TimeoutException e) {
                timedOutOrInterrupted.set(true);
            }
            return new TestingPartitionRequestClient();
        }
    };
    final RemoteInputChannel remoteInputChannel = InputChannelBuilder.newBuilder().setConnectionManager(blockingConnectionManager).buildRemoteChannel(inputGate);
    inputGate.setInputChannels(remoteInputChannel);
    final Thread simulatedNetworkThread = new Thread(() -> {
        try {
            ready.await();
            // We want to make sure that our simulated network thread does not
            // block on
            // SingleInputGate#requestLock as well through this call.
            remoteInputChannel.onFailedPartitionRequest();
            // Will only give free the blocker if we did not block ourselves.
            blocker.trigger();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    });
    simulatedNetworkThread.start();
    // The entry point to that will lead us into
    // blockingConnectionManager#createPartitionRequestClient(...).
    inputGate.requestPartitions();
    simulatedNetworkThread.join();
    Assert.assertFalse("Test ended by timeout or interruption - this indicates that the network thread was blocked.", timedOutOrInterrupted.get());
}
Also used : TestTaskBuilder(org.apache.flink.runtime.taskmanager.TestTaskBuilder) Arrays(java.util.Arrays) Matchers.isA(org.hamcrest.Matchers.isA) AvailabilityUtil.assertAvailability(org.apache.flink.runtime.io.network.partition.AvailabilityUtil.assertAvailability) ProducerFailedException(org.apache.flink.runtime.io.network.partition.ProducerFailedException) TimeoutException(java.util.concurrent.TimeoutException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) Random(java.util.Random) PartitionRequestClient(org.apache.flink.runtime.io.network.PartitionRequestClient) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) AvailabilityUtil.assertPriorityAvailability(org.apache.flink.runtime.io.network.partition.AvailabilityUtil.assertPriorityAvailability) Lists(org.apache.flink.shaded.guava30.com.google.common.collect.Lists) Future(java.util.concurrent.Future) CheckpointException(org.apache.flink.runtime.checkpoint.CheckpointException) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) CheckpointStorageLocationReference.getDefault(org.apache.flink.runtime.state.CheckpointStorageLocationReference.getDefault) Assert.fail(org.junit.Assert.fail) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) EventSerializer(org.apache.flink.runtime.io.network.api.serialization.EventSerializer) TestBufferFactory.createBuffer(org.apache.flink.runtime.io.network.util.TestBufferFactory.createBuffer) ExpectedTestException(org.apache.flink.runtime.operators.testutils.ExpectedTestException) InputChannelTestUtils(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils) DataType(org.apache.flink.runtime.io.network.buffer.Buffer.DataType) ConnectionID(org.apache.flink.runtime.io.network.ConnectionID) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) PartitionNotFoundException(org.apache.flink.runtime.io.network.partition.PartitionNotFoundException) Collectors(java.util.stream.Collectors) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) Executors(java.util.concurrent.Executors) Matchers.any(org.mockito.Matchers.any) CloseableIterator(org.apache.flink.util.CloseableIterator) List(java.util.List) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) Matchers.contains(org.hamcrest.Matchers.contains) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) TestingPartitionRequestClient(org.apache.flink.runtime.io.network.TestingPartitionRequestClient) Matchers.is(org.hamcrest.Matchers.is) Queue(java.util.Queue) Mockito.mock(org.mockito.Mockito.mock) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) TestBufferFactory(org.apache.flink.runtime.io.network.util.TestBufferFactory) CHECKPOINT(org.apache.flink.runtime.checkpoint.CheckpointType.CHECKPOINT) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) InputChannelTestUtils.createSingleInputGate(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createSingleInputGate) Callable(java.util.concurrent.Callable) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ArrayList(java.util.ArrayList) Matchers.hasProperty(org.hamcrest.Matchers.hasProperty) BufferBuilder(org.apache.flink.runtime.io.network.buffer.BufferBuilder) EventSerializer.toBuffer(org.apache.flink.runtime.io.network.api.serialization.EventSerializer.toBuffer) ChannelStateWriter(org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) CheckpointOptions.alignedWithTimeout(org.apache.flink.runtime.checkpoint.CheckpointOptions.alignedWithTimeout) InputChannelInfo(org.apache.flink.runtime.checkpoint.channel.InputChannelInfo) Nullable(javax.annotation.Nullable) ExecutorService(java.util.concurrent.ExecutorService) MemorySegment(org.apache.flink.core.memory.MemorySegment) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) Assert.assertNotNull(org.junit.Assert.assertNotNull) BufferPool(org.apache.flink.runtime.io.network.buffer.BufferPool) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) PartitionProducerStateChecker(org.apache.flink.runtime.taskexecutor.PartitionProducerStateChecker) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) BufferBuilderTestUtils.buildSingleBuffer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSingleBuffer) BufferAndAvailability(org.apache.flink.runtime.io.network.partition.consumer.InputChannel.BufferAndAvailability) Task(org.apache.flink.runtime.taskmanager.Task) Assert.assertNull(org.junit.Assert.assertNull) NoOpBufferPool(org.apache.flink.runtime.io.network.buffer.NoOpBufferPool) Assert(org.junit.Assert) ArrayDeque(java.util.ArrayDeque) PartitionProducerStateProvider(org.apache.flink.runtime.io.network.partition.PartitionProducerStateProvider) Assert.assertEquals(org.junit.Assert.assertEquals) Task(org.apache.flink.runtime.taskmanager.Task) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) InputChannelTestUtils.createSingleInputGate(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createSingleInputGate) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ConnectionID(org.apache.flink.runtime.io.network.ConnectionID) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) TestTaskBuilder(org.apache.flink.runtime.taskmanager.TestTaskBuilder) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) PartitionProducerStateChecker(org.apache.flink.runtime.taskexecutor.PartitionProducerStateChecker) TestingPartitionRequestClient(org.apache.flink.runtime.io.network.TestingPartitionRequestClient) TimeoutException(java.util.concurrent.TimeoutException) Test(org.junit.Test)

Example 8 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class SingleInputGateTest method testRequestBuffersWithRemoteInputChannel.

/**
 * Tests that input gate requests and assigns network buffers for remote input channel.
 */
@Test
public void testRequestBuffersWithRemoteInputChannel() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final SingleInputGate inputGate = createInputGate(network, 1, ResultPartitionType.PIPELINED_BOUNDED);
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        RemoteInputChannel remote = InputChannelBuilder.newBuilder().setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
        inputGate.setInputChannels(remote);
        inputGate.setup();
        NetworkBufferPool bufferPool = network.getNetworkBufferPool();
        // only the exclusive buffers should be assigned/available now
        assertEquals(buffersPerChannel, remote.getNumberOfAvailableBuffers());
        assertEquals(bufferPool.getTotalNumberOfMemorySegments() - buffersPerChannel - 1, bufferPool.getNumberOfAvailableMemorySegments());
        // note: exclusive buffers are not handed out into LocalBufferPool and are thus not
        // counted
        assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
    }
}
Also used : Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) Test(org.junit.Test)

Example 9 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class CheckpointBarrierTrackerTest method createCheckpointedInputGate.

// ------------------------------------------------------------------------
// Utils
// ------------------------------------------------------------------------
private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify);
}
Also used : SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Aggregations

TestingConnectionManager (org.apache.flink.runtime.io.network.TestingConnectionManager)9 NettyShuffleEnvironment (org.apache.flink.runtime.io.network.NettyShuffleEnvironment)7 Test (org.junit.Test)6 NettyShuffleEnvironmentBuilder (org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder)5 NetworkBufferPool (org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)3 RemoteInputChannel (org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)3 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)3 SingleInputGateBuilder (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder)3 IOException (java.io.IOException)2 Optional (java.util.Optional)2 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)2 CheckpointType (org.apache.flink.runtime.checkpoint.CheckpointType)2 InputChannelInfo (org.apache.flink.runtime.checkpoint.channel.InputChannelInfo)2 CancelTaskException (org.apache.flink.runtime.execution.CancelTaskException)2 ConnectionID (org.apache.flink.runtime.io.network.ConnectionID)2 ConnectionManager (org.apache.flink.runtime.io.network.ConnectionManager)2 PartitionRequestClient (org.apache.flink.runtime.io.network.PartitionRequestClient)2 TestingPartitionRequestClient (org.apache.flink.runtime.io.network.TestingPartitionRequestClient)2 InputChannelTestUtils.createLocalInputChannel (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel)2 InputChannelTestUtils.createRemoteInputChannel (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel)2