Examples with ResourceConfig org.apache.sysml.yarn.ropt.ResourceConfig used on opensource projects

Example 1 with ResourceConfig

use of org.apache.sysml.yarn.ropt.ResourceConfig in project incubator-systemml by apache.

the class DMLYarnClientProxy method launchDMLYarnAppmaster.

public static boolean launchDMLYarnAppmaster(String dmlScriptStr, DMLConfig conf, String[] allArgs, Program rtprog) throws IOException, DMLRuntimeException {
    boolean ret = false;
    try {
        //optimize resources (and update configuration)
        if (DMLAppMasterUtils.isResourceOptimizerEnabled()) {
            LOG.warn("Optimization level '" + OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY + "' " + "is still in experimental state and not intended for production use.");
            YarnClusterConfig cc = YarnClusterAnalyzer.getClusterConfig();
            DMLAppMasterUtils.setupRemoteParallelTasks(cc);
            ArrayList<ProgramBlock> pb = DMLAppMasterUtils.getRuntimeProgramBlocks(rtprog);
            ResourceConfig rc = ResourceOptimizer.optimizeResourceConfig(pb, cc, GridEnumType.HYBRID_MEM_EXP_GRID, GridEnumType.HYBRID_MEM_EXP_GRID);
            conf.updateYarnMemorySettings(String.valueOf(YarnOptimizerUtils.toMB(rc.getCPResource())), rc.serialize());
        //alternative: only use the max mr memory for all statement blocks
        //conf.updateYarnMemorySettings(String.valueOf(rc.getCPResource()), String.valueOf(rc.getMaxMRResource()));
        }
        //launch dml yarn app master
        DMLYarnClient yclient = new DMLYarnClient(dmlScriptStr, conf, allArgs);
        ret = yclient.launchDMLYarnAppmaster();
    } catch (NoClassDefFoundError ex) {
        LOG.warn("Failed to instantiate DML yarn client " + "(NoClassDefFoundError: " + ex.getMessage() + "). " + "Resume with default client processing.");
        ret = false;
    }
    return ret;
}

Example 2 with ResourceConfig

use of org.apache.sysml.yarn.ropt.ResourceConfig in project incubator-systemml by apache.

the class DMLAppMasterUtils method setupConfigRemoteMaxMemory.

public static void setupConfigRemoteMaxMemory(DMLConfig conf) throws DMLRuntimeException {
    //set remote max memory (if in yarn appmaster context)
    if (DMLScript.isActiveAM()) {
        //set optimization level (for awareness of resource optimization)
        CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(conf);
        ConfigurationManager.setGlobalConfig(cconf);
        if (isResourceOptimizerEnabled()) {
            //handle optimized memory (mr memory budget per program block)
            //ensure cluster has been analyzed
            InfrastructureAnalyzer.getRemoteMaxMemoryMap();
            String memStr = conf.getTextValue(DMLConfig.YARN_MAPREDUCEMEM);
            ResourceConfig rc = ResourceConfig.deserialize(memStr);
            //keep resource config for later program mapping
            _rc = rc;
        } else {
            //handle user configuration
            if (conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM) > 0) {
                //ensure cluster has been analyzed
                InfrastructureAnalyzer.getRemoteMaxMemoryMap();
                //set max map and reduce memory (to be used by the compiler)
                //see GMR and parfor EMR and DPEMR for runtime configuration
                long mem = ((long) conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM)) * 1024 * 1024;
                InfrastructureAnalyzer.setRemoteMaxMemoryMap(mem);
                InfrastructureAnalyzer.setRemoteMaxMemoryReduce(mem);
            }
        }
    }
}