Examples with MemoryType org.apache.flink.core.memory.MemoryType used on opensource projects

Example 1 with MemoryType

use of org.apache.flink.core.memory.MemoryType in project flink by apache.

the class TaskManagerServicesConfiguration method parseNetworkEnvironmentConfiguration.

// --------------------------------------------------------------------------
//  Parsing and checking the TaskManager Configuration
// --------------------------------------------------------------------------
/**
	 * Creates the {@link NetworkEnvironmentConfiguration} from the given {@link Configuration}.
	 *
	 * @param configuration to create the network environment configuration from
	 * @param localTaskManagerCommunication true if task manager communication is local
	 * @param taskManagerAddress address of the task manager
	 * @param slots to start the task manager with
	 * @return Network environment configuration
	 */
private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfiguration(Configuration configuration, boolean localTaskManagerCommunication, InetAddress taskManagerAddress, int slots) throws Exception {
    // ----> hosts / ports for communication and data exchange
    int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
    checkConfigParameter(dataport >= 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, "Leave config parameter empty or use 0 to let the system choose a port automatically.");
    checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, "Number of task slots must be at least one.");
    final int numNetworkBuffers = configuration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
    checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers, ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, "");
    final int pageSize = configuration.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
    // check page size of for minimum size
    checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize, ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
    // check page size for power of two
    checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize, ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY, "Memory segment size must be a power of 2.");
    // check whether we use heap or off-heap memory
    final MemoryType memType;
    if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
        memType = MemoryType.OFF_HEAP;
    } else {
        memType = MemoryType.HEAP;
    }
    // TODO - this should be in the TaskManager, not the configuration
    if (memType == MemoryType.HEAP) {
        if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
            throw new Exception("Memory type is set to heap memory, but memory segment " + "factory has been initialized for off-heap memory segments");
        }
    } else {
        if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
            throw new Exception("Memory type is set to off-heap memory, but memory segment " + "factory has been initialized for heap memory segments");
        }
    }
    final NettyConfig nettyConfig;
    if (!localTaskManagerCommunication) {
        final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
        nettyConfig = new NettyConfig(taskManagerInetSocketAddress.getAddress(), taskManagerInetSocketAddress.getPort(), pageSize, slots, configuration);
    } else {
        nettyConfig = null;
    }
    // Default spill I/O mode for intermediate results
    final String syncOrAsync = configuration.getString(ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE);
    final IOManager.IOMode ioMode;
    if (syncOrAsync.equals("async")) {
        ioMode = IOManager.IOMode.ASYNC;
    } else {
        ioMode = IOManager.IOMode.SYNC;
    }
    int initialRequestBackoff = configuration.getInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL);
    int maxRequestBackoff = configuration.getInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX);
    int buffersPerChannel = configuration.getInteger(TaskManagerOptions.NETWORK_BUFFERS_PER_CHANNEL);
    int extraBuffersPerGate = configuration.getInteger(TaskManagerOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);
    return new NetworkEnvironmentConfiguration(numNetworkBuffers, pageSize, memType, ioMode, initialRequestBackoff, maxRequestBackoff, buffersPerChannel, extraBuffersPerGate, nettyConfig);
}

Example 2 with MemoryType

use of org.apache.flink.core.memory.MemoryType in project flink by apache.

the class TaskManagerServices method createMemoryManager.

/**
	 * Creates a {@link MemoryManager} from the given {@link TaskManagerServicesConfiguration}.
	 *
	 * @param taskManagerServicesConfiguration to create the memory manager from
	 * @return Memory manager
	 * @throws Exception
	 */
private static MemoryManager createMemoryManager(TaskManagerServicesConfiguration taskManagerServicesConfiguration) throws Exception {
    // computing the amount of memory to use depends on how much memory is available
    // it strictly needs to happen AFTER the network stack has been initialized
    MemoryType memType = taskManagerServicesConfiguration.getNetworkConfig().memoryType();
    // check if a value has been configured
    long configuredMemory = taskManagerServicesConfiguration.getConfiguredMemory();
    final long memorySize;
    boolean preAllocateMemory = taskManagerServicesConfiguration.isPreAllocateMemory();
    if (configuredMemory > 0) {
        if (preAllocateMemory) {
            LOG.info("Using {} MB for managed memory.", configuredMemory);
        } else {
            LOG.info("Limiting managed memory to {} MB, memory will be allocated lazily.", configuredMemory);
        }
        // megabytes to bytes
        memorySize = configuredMemory << 20;
    } else {
        float memoryFraction = taskManagerServicesConfiguration.getMemoryFraction();
        if (memType == MemoryType.HEAP) {
            long relativeMemSize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * memoryFraction);
            if (preAllocateMemory) {
                LOG.info("Using {} of the currently free heap space for managed heap memory ({} MB).", memoryFraction, relativeMemSize >> 20);
            } else {
                LOG.info("Limiting managed memory to {} of the currently free heap space ({} MB), " + "memory will be allocated lazily.", memoryFraction, relativeMemSize >> 20);
            }
            memorySize = relativeMemSize;
        } else if (memType == MemoryType.OFF_HEAP) {
            // The maximum heap memory has been adjusted according to the fraction
            long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
            long directMemorySize = (long) (maxMemory / (1.0 - memoryFraction) * memoryFraction);
            if (preAllocateMemory) {
                LOG.info("Using {} of the maximum memory size for managed off-heap memory ({} MB).", memoryFraction, directMemorySize >> 20);
            } else {
                LOG.info("Limiting managed memory to {} of the maximum memory size ({} MB)," + " memory will be allocated lazily.", memoryFraction, directMemorySize >> 20);
            }
            memorySize = directMemorySize;
        } else {
            throw new RuntimeException("No supported memory type detected.");
        }
    }
    // now start the memory manager
    final MemoryManager memoryManager;
    try {
        memoryManager = new MemoryManager(memorySize, taskManagerServicesConfiguration.getNumberOfSlots(), taskManagerServicesConfiguration.getNetworkConfig().networkBufferSize(), memType, preAllocateMemory);
    } catch (OutOfMemoryError e) {
        if (memType == MemoryType.HEAP) {
            throw new Exception("OutOfMemory error (" + e.getMessage() + ") while allocating the TaskManager heap memory (" + memorySize + " bytes).", e);
        } else if (memType == MemoryType.OFF_HEAP) {
            throw new Exception("OutOfMemory error (" + e.getMessage() + ") while allocating the TaskManager off-heap memory (" + memorySize + " bytes).Try increasing the maximum direct memory (-XX:MaxDirectMemorySize)", e);
        } else {
            throw e;
        }
    }
    return memoryManager;
}