Examples with ConnectionManager org.apache.flink.runtime.io.network.ConnectionManager used on opensource projects

Example 1 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class TaskManagerServices method createNetworkEnvironment.

/**
	 * Creates the {@link NetworkEnvironment} from the given {@link TaskManagerServicesConfiguration}.
	 *
	 * @param taskManagerServicesConfiguration to construct the network environment from
	 * @return Network environment
	 * @throws IOException
	 */
private static NetworkEnvironment createNetworkEnvironment(TaskManagerServicesConfiguration taskManagerServicesConfiguration) throws IOException {
    NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskManagerServicesConfiguration.getNetworkConfig();
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(networkEnvironmentConfiguration.numNetworkBuffers(), networkEnvironmentConfiguration.networkBufferSize(), networkEnvironmentConfiguration.memoryType());
    ConnectionManager connectionManager;
    if (networkEnvironmentConfiguration.nettyConfig() != null) {
        connectionManager = new NettyConnectionManager(networkEnvironmentConfiguration.nettyConfig());
    } else {
        connectionManager = new LocalConnectionManager();
    }
    ResultPartitionManager resultPartitionManager = new ResultPartitionManager();
    TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
    KvStateRegistry kvStateRegistry = new KvStateRegistry();
    KvStateServer kvStateServer;
    if (taskManagerServicesConfiguration.getQueryableStateConfig().enabled()) {
        QueryableStateConfiguration qsConfig = taskManagerServicesConfiguration.getQueryableStateConfig();
        int numNetworkThreads = qsConfig.numServerThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numServerThreads();
        int numQueryThreads = qsConfig.numQueryThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numQueryThreads();
        kvStateServer = new KvStateServer(taskManagerServicesConfiguration.getTaskManagerAddress(), qsConfig.port(), numNetworkThreads, numQueryThreads, kvStateRegistry, new DisabledKvStateRequestStats());
    } else {
        kvStateServer = null;
    }
    // we start the network first, to make sure it can allocate its buffers first
    return new NetworkEnvironment(networkBufferPool, connectionManager, resultPartitionManager, taskEventDispatcher, kvStateRegistry, kvStateServer, networkEnvironmentConfiguration.ioMode(), networkEnvironmentConfiguration.partitionRequestInitialBackoff(), networkEnvironmentConfiguration.partitionRequestMaxBackoff(), networkEnvironmentConfiguration.networkBuffersPerChannel(), networkEnvironmentConfiguration.extraNetworkBuffersPerGate());
}

Example 2 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class RemoteInputChannelTest method createRemoteInputChannel.

private RemoteInputChannel createRemoteInputChannel(SingleInputGate inputGate, PartitionRequestClient partitionRequestClient, Tuple2<Integer, Integer> initialAndMaxRequestBackoff) throws IOException, InterruptedException {
    final ConnectionManager connectionManager = mock(ConnectionManager.class);
    when(connectionManager.createPartitionRequestClient(any(ConnectionID.class))).thenReturn(partitionRequestClient);
    return new RemoteInputChannel(inputGate, 0, new ResultPartitionID(), mock(ConnectionID.class), connectionManager, initialAndMaxRequestBackoff._1(), initialAndMaxRequestBackoff._2(), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
}

Example 3 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithRemoteChannels.

@Test
public void testConsumptionWithRemoteChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ConnectionManager connManager = createDummyConnectionManager();
    final Source[] sources = new Source[numChannels];
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        sources[i] = new RemoteChannelSource(channel);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 4 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class InputChannelTestUtils method createDummyConnectionManager.

public static ConnectionManager createDummyConnectionManager() throws Exception {
    final PartitionRequestClient mockClient = mock(PartitionRequestClient.class);
    final ConnectionManager connManager = mock(ConnectionManager.class);
    when(connManager.createPartitionRequestClient(any(ConnectionID.class))).thenReturn(mockClient);
    return connManager;
}

Example 5 with ConnectionManager

use of org.apache.flink.runtime.io.network.ConnectionManager in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannels.

@Test
public void testFairConsumptionRemoteChannels() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = TestBufferFactory.createBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numberOfChannels];
    final int[] channelSequenceNums = new int[numberOfChannels];
    for (int i = 0; i < numberOfChannels; i++) {
        RemoteInputChannel channel = createRemoteInputChannel(gate, i, connManager);
        channels[i] = channel;
    }
    channels[11].onBuffer(mockBuffer, 0, -1);
    channelSequenceNums[11]++;
    setupInputGate(gate, channels);
    // read all the buffers and the EOF event
    for (int i = 0; i < numberOfChannels * buffersPerChannel; i++) {
        assertNotNull(gate.getNext());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == (min + 1));
        if (i % (2 * numberOfChannels) == 0) {
            // add three buffers to each channel, in random order
            fillRandom(channels, channelSequenceNums, 3, mockBuffer);
        }
    }
}