Examples with ConnectionManager org.apache.flink.runtime.io.network.ConnectionManager used on opensource projects

Example 11 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithMixedChannels.

@Test
public void testConsumptionWithMixedChannels() throws Exception {
    final int numChannels = 61;
    final int numLocalChannels = 20;
    final int buffersPerChannel = 1000;
    // fill the local/remote decision
    List<Boolean> localOrRemote = new ArrayList<>(numChannels);
    for (int i = 0; i < numChannels; i++) {
        localOrRemote.add(i < numLocalChannels);
    }
    Collections.shuffle(localOrRemote);
    final ConnectionManager connManager = createDummyConnectionManager();
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final PipelinedSubpartition[] localPartitions = new PipelinedSubpartition[numLocalChannels];
    final ResultPartitionManager resultPartitionManager = createResultPartitionManager(localPartitions);
    final Source[] sources = new Source[numChannels];
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0, local = 0; i < numChannels; i++) {
        if (localOrRemote.get(i)) {
            // local channel
            PipelinedSubpartition psp = new PipelinedSubpartition(0, resultPartition);
            localPartitions[local++] = psp;
            sources[i] = new PipelinedSubpartitionSource(psp);
            LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
            gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        } else {
            //remote channel
            RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
            gate.setInputChannel(new IntermediateResultPartitionID(), channel);
            sources[i] = new RemoteChannelSource(channel);
        }
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 12 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testPartitionRequestExponentialBackoff.

@Test
public void testPartitionRequestExponentialBackoff() throws Exception {
    // Start with initial backoff, then keep doubling, and cap at max.
    int[] expectedDelays = { 500, 1000, 2000, 3000 };
    // Setup
    SingleInputGate inputGate = createSingleInputGate(1);
    ResultPartitionID partitionId = new ResultPartitionID();
    TestVerifyPartitionRequestClient client = new TestVerifyPartitionRequestClient();
    ConnectionManager connectionManager = new TestVerifyConnectionManager(client);
    RemoteInputChannel ch = createRemoteInputChannel(inputGate, connectionManager, partitionId, 500, 3000);
    // Initial request
    ch.requestSubpartition();
    client.verifyResult(partitionId, 0, 0);
    // Request subpartition and verify that the actual requests are delayed.
    for (int expected : expectedDelays) {
        ch.retriggerSubpartitionRequest();
        client.verifyResult(partitionId, 0, expected);
    }
    // Exception after backoff is greater than the maximum backoff.
    try {
        ch.retriggerSubpartitionRequest();
        ch.getNextBuffer();
        fail("Did not throw expected exception.");
    } catch (Exception expected) {
    }
}

Example 13 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testOnFailedPartitionRequestDoesNotBlockNetworkThreads.

/**
 * Test to guard against FLINK-13249.
 */
@Test
public void testOnFailedPartitionRequestDoesNotBlockNetworkThreads() throws Exception {
    final long testBlockedWaitTimeoutMillis = 30_000L;
    final PartitionProducerStateChecker partitionProducerStateChecker = (jobId, intermediateDataSetId, resultPartitionId) -> CompletableFuture.completedFuture(ExecutionState.RUNNING);
    final NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    final Task task = new TestTaskBuilder(shuffleEnvironment).setPartitionProducerStateChecker(partitionProducerStateChecker).build();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setPartitionProducerStateProvider(task).build();
    TestTaskBuilder.setTaskState(task, ExecutionState.RUNNING);
    final OneShotLatch ready = new OneShotLatch();
    final OneShotLatch blocker = new OneShotLatch();
    final AtomicBoolean timedOutOrInterrupted = new AtomicBoolean(false);
    final ConnectionManager blockingConnectionManager = new TestingConnectionManager() {

        @Override
        public PartitionRequestClient createPartitionRequestClient(ConnectionID connectionId) {
            ready.trigger();
            try {
                // We block here, in a section that holds the
                // SingleInputGate#requestLock
                blocker.await(testBlockedWaitTimeoutMillis, TimeUnit.MILLISECONDS);
            } catch (InterruptedException | TimeoutException e) {
                timedOutOrInterrupted.set(true);
            }
            return new TestingPartitionRequestClient();
        }
    };
    final RemoteInputChannel remoteInputChannel = InputChannelBuilder.newBuilder().setConnectionManager(blockingConnectionManager).buildRemoteChannel(inputGate);
    inputGate.setInputChannels(remoteInputChannel);
    final Thread simulatedNetworkThread = new Thread(() -> {
        try {
            ready.await();
            // We want to make sure that our simulated network thread does not
            // block on
            // SingleInputGate#requestLock as well through this call.
            remoteInputChannel.onFailedPartitionRequest();
            // Will only give free the blocker if we did not block ourselves.
            blocker.trigger();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    });
    simulatedNetworkThread.start();
    // The entry point to that will lead us into
    // blockingConnectionManager#createPartitionRequestClient(...).
    inputGate.requestPartitions();
    simulatedNetworkThread.join();
    Assert.assertFalse("Test ended by timeout or interruption - this indicates that the network thread was blocked.", timedOutOrInterrupted.get());
}

Example 14 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testPartitionConnectionException.

@Test(expected = PartitionConnectionException.class)
public void testPartitionConnectionException() throws IOException {
    final ConnectionManager connManager = new TestingExceptionConnectionManager();
    final SingleInputGate gate = createSingleInputGate(1);
    final RemoteInputChannel ch = InputChannelTestUtils.createRemoteInputChannel(gate, 0, connManager);
    gate.setInputChannels(ch);
    gate.requestPartitions();
    ch.getNextBuffer();
}