Examples with ConnectionManager org.apache.flink.runtime.io.network.ConnectionManager used on opensource projects

Example 6 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannelsPreFilled.

@Test
public void testFairConsumptionRemoteChannelsPreFilled() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = TestBufferFactory.createBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numberOfChannels];
    for (int i = 0; i < numberOfChannels; i++) {
        RemoteInputChannel channel = createRemoteInputChannel(gate, i, connManager);
        channels[i] = channel;
        for (int p = 0; p < buffersPerChannel; p++) {
            channel.onBuffer(mockBuffer, p, -1);
        }
        channel.onBuffer(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), buffersPerChannel, -1);
    }
    gate.setInputChannels(channels);
    gate.setup();
    gate.requestPartitions();
    // read all the buffers and the EOF event
    for (int i = numberOfChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNext());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == (min + 1));
    }
    assertFalse(gate.getNext().isPresent());
}

Example 7 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testProducerFailedException.

@Test(expected = CancelTaskException.class)
public void testProducerFailedException() throws Exception {
    ConnectionManager connManager = mock(ConnectionManager.class);
    when(connManager.createPartitionRequestClient(any(ConnectionID.class))).thenReturn(mock(PartitionRequestClient.class));
    final SingleInputGate gate = createSingleInputGate(1);
    final RemoteInputChannel ch = InputChannelTestUtils.createRemoteInputChannel(gate, 0, connManager);
    ch.onError(new ProducerFailedException(new RuntimeException("Expected test exception.")));
    ch.requestSubpartition();
    // Should throw an instance of CancelTaskException.
    ch.getNextBuffer();
}

Example 8 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testPartitionRequestNoBackoff.

@Test
public void testPartitionRequestNoBackoff() throws Exception {
    // Setup
    SingleInputGate inputGate = createSingleInputGate(1);
    ResultPartitionID partitionId = new ResultPartitionID();
    TestVerifyPartitionRequestClient client = new TestVerifyPartitionRequestClient();
    ConnectionManager connectionManager = new TestVerifyConnectionManager(client);
    RemoteInputChannel ch = createRemoteInputChannel(inputGate, connectionManager, partitionId, 0, 0);
    // No delay for first request
    ch.requestSubpartition();
    client.verifyResult(partitionId, 0, 0);
    // Exception, because backoff is disabled.
    try {
        ch.retriggerSubpartitionRequest();
        ch.getNextBuffer();
        fail("Did not throw expected exception.");
    } catch (Exception expected) {
    }
}

Example 9 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.

private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
    ConnectionManager connectionManager = new TestingConnectionManager();
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}

Example 10 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testPartitionRequestSingleBackoff.

@Test
public void testPartitionRequestSingleBackoff() throws Exception {
    // Setup
    SingleInputGate inputGate = createSingleInputGate(1);
    ResultPartitionID partitionId = new ResultPartitionID();
    TestVerifyPartitionRequestClient client = new TestVerifyPartitionRequestClient();
    ConnectionManager connectionManager = new TestVerifyConnectionManager(client);
    RemoteInputChannel ch = createRemoteInputChannel(inputGate, connectionManager, partitionId, 500, 500);
    // No delay for first request
    ch.requestSubpartition();
    client.verifyResult(partitionId, 0, 0);
    // Initial delay for second request
    ch.retriggerSubpartitionRequest();
    client.verifyResult(partitionId, 0, 500);
    // Exception after backoff is greater than the maximum backoff.
    try {
        ch.retriggerSubpartitionRequest();
        ch.getNextBuffer();
        fail("Did not throw expected exception.");
    } catch (Exception expected) {
    }
}