Examples with NetworkEnvironmentConfiguration org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration used on opensource projects

Example 1 with NetworkEnvironmentConfiguration

use of org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration in project flink by apache.

the class TaskManagerServices method createNetworkEnvironment.

/**
	 * Creates the {@link NetworkEnvironment} from the given {@link TaskManagerServicesConfiguration}.
	 *
	 * @param taskManagerServicesConfiguration to construct the network environment from
	 * @return Network environment
	 * @throws IOException
	 */
private static NetworkEnvironment createNetworkEnvironment(TaskManagerServicesConfiguration taskManagerServicesConfiguration) throws IOException {
    NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskManagerServicesConfiguration.getNetworkConfig();
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(networkEnvironmentConfiguration.numNetworkBuffers(), networkEnvironmentConfiguration.networkBufferSize(), networkEnvironmentConfiguration.memoryType());
    ConnectionManager connectionManager;
    if (networkEnvironmentConfiguration.nettyConfig() != null) {
        connectionManager = new NettyConnectionManager(networkEnvironmentConfiguration.nettyConfig());
    } else {
        connectionManager = new LocalConnectionManager();
    }
    ResultPartitionManager resultPartitionManager = new ResultPartitionManager();
    TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
    KvStateRegistry kvStateRegistry = new KvStateRegistry();
    KvStateServer kvStateServer;
    if (taskManagerServicesConfiguration.getQueryableStateConfig().enabled()) {
        QueryableStateConfiguration qsConfig = taskManagerServicesConfiguration.getQueryableStateConfig();
        int numNetworkThreads = qsConfig.numServerThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numServerThreads();
        int numQueryThreads = qsConfig.numQueryThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numQueryThreads();
        kvStateServer = new KvStateServer(taskManagerServicesConfiguration.getTaskManagerAddress(), qsConfig.port(), numNetworkThreads, numQueryThreads, kvStateRegistry, new DisabledKvStateRequestStats());
    } else {
        kvStateServer = null;
    }
    // we start the network first, to make sure it can allocate its buffers first
    return new NetworkEnvironment(networkBufferPool, connectionManager, resultPartitionManager, taskEventDispatcher, kvStateRegistry, kvStateServer, networkEnvironmentConfiguration.ioMode(), networkEnvironmentConfiguration.partitionRequestInitialBackoff(), networkEnvironmentConfiguration.partitionRequestMaxBackoff(), networkEnvironmentConfiguration.networkBuffersPerChannel(), networkEnvironmentConfiguration.extraNetworkBuffersPerGate());
}

Example 2 with NetworkEnvironmentConfiguration

use of org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration in project flink by apache.

the class TaskManagerServicesConfiguration method fromConfiguration.

// --------------------------------------------------------------------------------------------
//  Parsing of Flink configuration
// --------------------------------------------------------------------------------------------
/**
	 * Utility method to extract TaskManager config parameters from the configuration and to
	 * sanity check them.
	 *
	 * @param configuration The configuration.
	 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
	 * @param localCommunication True, to skip initializing the network stack.
	 *                                      Use only in cases where only one task manager runs.
	 * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
	 */
public static TaskManagerServicesConfiguration fromConfiguration(Configuration configuration, InetAddress remoteAddress, boolean localCommunication) throws Exception {
    // we need this because many configs have been written with a "-1" entry
    int slots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
    if (slots == -1) {
        slots = 1;
    }
    final String[] tmpDirs = configuration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(",|" + File.pathSeparator);
    final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(configuration, localCommunication, remoteAddress, slots);
    final QueryableStateConfiguration queryableStateConfig = localCommunication ? QueryableStateConfiguration.disabled() : parseQueryableStateConfiguration(configuration);
    // extract memory settings
    long configuredMemory = configuration.getLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    checkConfigParameter(configuredMemory == -1 || configuredMemory > 0, configuredMemory, ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, "MemoryManager needs at least one MB of memory. " + "If you leave this config parameter empty, the system automatically " + "pick a fraction of the available memory.");
    boolean preAllocateMemory = configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_PRE_ALLOCATE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE);
    float memoryFraction = configuration.getFloat(ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY, ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
    checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction, ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY, "MemoryManager fraction of the free memory must be between 0.0 and 1.0");
    final MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(configuration);
    long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();
    return new TaskManagerServicesConfiguration(remoteAddress, tmpDirs, networkConfig, queryableStateConfig, slots, configuredMemory, preAllocateMemory, memoryFraction, metricRegistryConfiguration, timerServiceShutdownTimeout);
}

Example 3 with NetworkEnvironmentConfiguration

use of org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration in project flink by apache.

the class TaskManagerServicesConfiguration method parseNetworkEnvironmentConfiguration.

// --------------------------------------------------------------------------
//  Parsing and checking the TaskManager Configuration
// --------------------------------------------------------------------------
/**
	 * Creates the {@link NetworkEnvironmentConfiguration} from the given {@link Configuration}.
	 *
	 * @param configuration to create the network environment configuration from
	 * @param localTaskManagerCommunication true if task manager communication is local
	 * @param taskManagerAddress address of the task manager
	 * @param slots to start the task manager with
	 * @return Network environment configuration
	 */
private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfiguration(Configuration configuration, boolean localTaskManagerCommunication, InetAddress taskManagerAddress, int slots) throws Exception {
    // ----> hosts / ports for communication and data exchange
    int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
    checkConfigParameter(dataport >= 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, "Leave config parameter empty or use 0 to let the system choose a port automatically.");
    checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, "Number of task slots must be at least one.");
    final int numNetworkBuffers = configuration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
    checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers, ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, "");
    final int pageSize = configuration.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
    // check page size of for minimum size
    checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize, ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
    // check page size for power of two
    checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize, ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, "Memory segment size must be a power of 2.");
    // check whether we use heap or off-heap memory
    final MemoryType memType;
    if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
        memType = MemoryType.OFF_HEAP;
    } else {
        memType = MemoryType.HEAP;
    }
    // TODO - this should be in the TaskManager, not the configuration
    if (memType == MemoryType.HEAP) {
        if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
            throw new Exception("Memory type is set to heap memory, but memory segment " + "factory has been initialized for off-heap memory segments");
        }
    } else {
        if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
            throw new Exception("Memory type is set to off-heap memory, but memory segment " + "factory has been initialized for heap memory segments");
        }
    }
    final NettyConfig nettyConfig;
    if (!localTaskManagerCommunication) {
        final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
        nettyConfig = new NettyConfig(taskManagerInetSocketAddress.getAddress(), taskManagerInetSocketAddress.getPort(), pageSize, slots, configuration);
    } else {
        nettyConfig = null;
    }
    // Default spill I/O mode for intermediate results
    final String syncOrAsync = configuration.getString(ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE);
    final IOManager.IOMode ioMode;
    if (syncOrAsync.equals("async")) {
        ioMode = IOManager.IOMode.ASYNC;
    } else {
        ioMode = IOManager.IOMode.SYNC;
    }
    int initialRequestBackoff = configuration.getInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL);
    int maxRequestBackoff = configuration.getInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX);
    int buffersPerChannel = configuration.getInteger(TaskManagerOptions.NETWORK_BUFFERS_PER_CHANNEL);
    int extraBuffersPerGate = configuration.getInteger(TaskManagerOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);
    return new NetworkEnvironmentConfiguration(numNetworkBuffers, pageSize, memType, ioMode, initialRequestBackoff, maxRequestBackoff, buffersPerChannel, extraBuffersPerGate, nettyConfig);
}