Examples with YarnClusterConfig org.apache.sysml.yarn.ropt.YarnClusterConfig used on opensource projects

Example 1 with YarnClusterConfig

use of org.apache.sysml.yarn.ropt.YarnClusterConfig in project incubator-systemml by apache.

the class DMLYarnClientProxy method launchDMLYarnAppmaster.

public static boolean launchDMLYarnAppmaster(String dmlScriptStr, DMLConfig conf, String[] allArgs, Program rtprog) throws IOException, DMLRuntimeException {
    boolean ret = false;
    try {
        //optimize resources (and update configuration)
        if (DMLAppMasterUtils.isResourceOptimizerEnabled()) {
            LOG.warn("Optimization level '" + OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY + "' " + "is still in experimental state and not intended for production use.");
            YarnClusterConfig cc = YarnClusterAnalyzer.getClusterConfig();
            DMLAppMasterUtils.setupRemoteParallelTasks(cc);
            ArrayList<ProgramBlock> pb = DMLAppMasterUtils.getRuntimeProgramBlocks(rtprog);
            ResourceConfig rc = ResourceOptimizer.optimizeResourceConfig(pb, cc, GridEnumType.HYBRID_MEM_EXP_GRID, GridEnumType.HYBRID_MEM_EXP_GRID);
            conf.updateYarnMemorySettings(String.valueOf(YarnOptimizerUtils.toMB(rc.getCPResource())), rc.serialize());
        //alternative: only use the max mr memory for all statement blocks
        //conf.updateYarnMemorySettings(String.valueOf(rc.getCPResource()), String.valueOf(rc.getMaxMRResource()));
        }
        //launch dml yarn app master
        DMLYarnClient yclient = new DMLYarnClient(dmlScriptStr, conf, allArgs);
        ret = yclient.launchDMLYarnAppmaster();
    } catch (NoClassDefFoundError ex) {
        LOG.warn("Failed to instantiate DML yarn client " + "(NoClassDefFoundError: " + ex.getMessage() + "). " + "Resume with default client processing.");
        ret = false;
    }
    return ret;
}