Examples with NettyConfig org.apache.flink.runtime.io.network.netty.NettyConfig used on opensource projects

Example 1 with NettyConfig

use of org.apache.flink.runtime.io.network.netty.NettyConfig in project flink by apache.

the class TaskManagerServicesConfiguration method parseNetworkEnvironmentConfiguration.

// --------------------------------------------------------------------------
//  Parsing and checking the TaskManager Configuration
// --------------------------------------------------------------------------
/**
	 * Creates the {@link NetworkEnvironmentConfiguration} from the given {@link Configuration}.
	 *
	 * @param configuration to create the network environment configuration from
	 * @param localTaskManagerCommunication true if task manager communication is local
	 * @param taskManagerAddress address of the task manager
	 * @param slots to start the task manager with
	 * @return Network environment configuration
	 */
private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfiguration(Configuration configuration, boolean localTaskManagerCommunication, InetAddress taskManagerAddress, int slots) throws Exception {
    // ----> hosts / ports for communication and data exchange
    int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
    checkConfigParameter(dataport >= 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, "Leave config parameter empty or use 0 to let the system choose a port automatically.");
    checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, "Number of task slots must be at least one.");
    final int numNetworkBuffers = configuration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
    checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers, ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, "");
    final int pageSize = configuration.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
    // check page size of for minimum size
    checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize, ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
    // check page size for power of two
    checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize, ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, "Memory segment size must be a power of 2.");
    // check whether we use heap or off-heap memory
    final MemoryType memType;
    if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
        memType = MemoryType.OFF_HEAP;
    } else {
        memType = MemoryType.HEAP;
    }
    // TODO - this should be in the TaskManager, not the configuration
    if (memType == MemoryType.HEAP) {
        if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
            throw new Exception("Memory type is set to heap memory, but memory segment " + "factory has been initialized for off-heap memory segments");
        }
    } else {
        if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
            throw new Exception("Memory type is set to off-heap memory, but memory segment " + "factory has been initialized for heap memory segments");
        }
    }
    final NettyConfig nettyConfig;
    if (!localTaskManagerCommunication) {
        final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
        nettyConfig = new NettyConfig(taskManagerInetSocketAddress.getAddress(), taskManagerInetSocketAddress.getPort(), pageSize, slots, configuration);
    } else {
        nettyConfig = null;
    }
    // Default spill I/O mode for intermediate results
    final String syncOrAsync = configuration.getString(ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE);
    final IOManager.IOMode ioMode;
    if (syncOrAsync.equals("async")) {
        ioMode = IOManager.IOMode.ASYNC;
    } else {
        ioMode = IOManager.IOMode.SYNC;
    }
    int initialRequestBackoff = configuration.getInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL);
    int maxRequestBackoff = configuration.getInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX);
    int buffersPerChannel = configuration.getInteger(TaskManagerOptions.NETWORK_BUFFERS_PER_CHANNEL);
    int extraBuffersPerGate = configuration.getInteger(TaskManagerOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);
    return new NetworkEnvironmentConfiguration(numNetworkBuffers, pageSize, memType, ioMode, initialRequestBackoff, maxRequestBackoff, buffersPerChannel, extraBuffersPerGate, nettyConfig);
}