Examples with ExplainCounts org.apache.sysml.utils.Explain.ExplainCounts used on opensource projects

Example 1 with ExplainCounts

use of org.apache.sysml.utils.Explain.ExplainCounts in project incubator-systemml by apache.

the class ScriptExecutor method countCompiledMRJobsAndSparkInstructions.

/**
	 * Count the number of compiled MR Jobs/Spark Instructions in the runtime
	 * program and set this value in the statistics.
	 */
protected void countCompiledMRJobsAndSparkInstructions() {
    ExplainCounts counts = Explain.countDistributedOperations(runtimeProgram);
    Statistics.resetNoOfCompiledJobs(counts.numJobs);
}

Example 2 with ExplainCounts

use of org.apache.sysml.utils.Explain.ExplainCounts in project incubator-systemml by apache.

the class DMLScript method execute.

///////////////////////////////
// private internal interface 
// (core compilation and execute)
////////
/**
	 * The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
	 * and customized parameters have been put into _argVals
	 * 
	 * @param dmlScriptStr DML script string
	 * @param fnameOptConfig configuration file
	 * @param argVals map of argument values
	 * @param allArgs arguments
	 * @param scriptType type of script (DML or PyDML)
	 * @throws ParseException if ParseException occurs
	 * @throws IOException if IOException occurs
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws LanguageException if LanguageException occurs
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String, String> argVals, String[] allArgs, ScriptType scriptType) throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException {
    SCRIPT_TYPE = scriptType;
    //print basic time and environment info
    printStartExecInfo(dmlScriptStr);
    //Step 1: parse configuration files
    DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
    ConfigurationManager.setGlobalConfig(dmlconf);
    CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
    ConfigurationManager.setGlobalConfig(cconf);
    LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());
    //Step 2: set local/remote memory if requested (for compile in AM context) 
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        DMLAppMasterUtils.setupConfigRemoteMaxMemory(dmlconf);
    }
    //Step 3: parse dml script
    Statistics.startCompileTimer();
    ParserWrapper parser = ParserFactory.createParser(scriptType);
    DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
    //Step 4: construct HOP DAGs (incl LVA, validate, and setup)
    DMLTranslator dmlt = new DMLTranslator(prog);
    dmlt.liveVariableAnalysis(prog);
    dmlt.validateParseTree(prog);
    dmlt.constructHops(prog);
    //init working directories (before usage by following compilation steps)
    initHadoopExecution(dmlconf);
    //Step 5: rewrite HOP DAGs (incl IPA and memory estimates)
    dmlt.rewriteHopsDAG(prog);
    //Step 5.1: Generate code for the rewritten Hop dags 
    if (dmlconf.getBooleanValue(DMLConfig.CODEGEN)) {
        SpoofCompiler.PLAN_CACHE_POLICY = PlanCachePolicy.get(dmlconf.getBooleanValue(DMLConfig.CODEGEN_PLANCACHE), dmlconf.getIntValue(DMLConfig.CODEGEN_LITERALS) == 2);
        SpoofCompiler.setExecTypeSpecificJavaCompiler();
        if (SpoofCompiler.INTEGRATION == IntegrationType.HOPS)
            dmlt.codgenHopsDAG(prog);
    }
    //Step 6: construct lops (incl exec type and op selection)
    dmlt.constructLops(prog);
    if (LOG.isDebugEnabled()) {
        LOG.debug("\n********************** LOPS DAG *******************");
        dmlt.printLops(prog);
        dmlt.resetLopsDAGVisitStatus(prog);
    }
    //Step 7: generate runtime program
    Program rtprog = prog.getRuntimeProgram(dmlconf);
    //Step 7.1: Generate code for the rewritten Hop dags w/o modify
    if (dmlconf.getBooleanValue(DMLConfig.CODEGEN) && SpoofCompiler.INTEGRATION == IntegrationType.RUNTIME) {
        dmlt.codgenHopsDAG(rtprog);
    }
    //Step 8: [optional global data flow optimization]
    if (OptimizerUtils.isOptLevel(OptimizationLevel.O4_GLOBAL_TIME_MEMORY)) {
        LOG.warn("Optimization level '" + OptimizationLevel.O4_GLOBAL_TIME_MEMORY + "' " + "is still in experimental state and not intended for production use.");
        rtprog = GlobalOptimizerWrapper.optimizeProgram(prog, rtprog);
    }
    //launch SystemML appmaster (if requested and not already in launched AM)
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        if (!isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, dmlconf, allArgs, rtprog))
            //if AM launch unsuccessful, fall back to normal execute
            return;
        if (//in AM context (not failed AM launch)
        isActiveAM())
            DMLAppMasterUtils.setupProgramMappingRemoteMaxMemory(rtprog);
    }
    //Step 9: prepare statistics [and optional explain output]
    //count number compiled MR jobs / SP instructions	
    ExplainCounts counts = Explain.countDistributedOperations(rtprog);
    Statistics.resetNoOfCompiledJobs(counts.numJobs);
    //explain plan of program (hops or runtime)
    if (EXPLAIN != ExplainType.NONE) {
        LOG.info("EXPLAIN (" + EXPLAIN.toString() + "):\n" + Explain.explainMemoryBudget(counts) + "\n" + Explain.explainDegreeOfParallelism(counts) + Explain.explain(prog, rtprog, EXPLAIN));
    }
    Statistics.stopCompileTimer();
    //double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
    //System.out.println("Estimated costs: "+costs);
    //Step 10: execute runtime program
    ExecutionContext ec = null;
    try {
        ec = ExecutionContextFactory.createContext(rtprog);
        ScriptExecutorUtils.executeRuntimeProgram(rtprog, ec, dmlconf, STATISTICS ? STATISTICS_COUNT : 0);
    } finally {
        if (ec != null && ec instanceof SparkExecutionContext)
            ((SparkExecutionContext) ec).close();
        LOG.info("END DML run " + getDateTime());
        //cleanup scratch_space and all working dirs
        cleanupHadoopExecution(dmlconf);
    }
}