Examples with Program org.apache.sysml.runtime.controlprogram.Program used on opensource projects

Example 11 with Program

use of org.apache.sysml.runtime.controlprogram.Program in project incubator-systemml by apache.

the class ProgramConverter method parseProgram.

public static Program parseProgram(String in, int id) throws DMLRuntimeException {
    String lin = in.substring(PARFOR_PROG_BEGIN.length(), in.length() - PARFOR_PROG_END.length()).trim();
    Program prog = new Program();
    HashMap<String, FunctionProgramBlock> fc = parseFunctionProgramBlocks(lin, prog, id);
    for (Entry<String, FunctionProgramBlock> e : fc.entrySet()) {
        String[] keypart = e.getKey().split(Program.KEY_DELIM);
        String namespace = keypart[0];
        String name = keypart[1];
        prog.addFunctionProgramBlock(namespace, name, e.getValue());
    }
    return prog;
}

Example 12 with Program

use of org.apache.sysml.runtime.controlprogram.Program in project incubator-systemml by apache.

the class OptimizerRuleBased method rFindAndUnfoldRecursiveFunction.

protected void rFindAndUnfoldRecursiveFunction(OptNode n, ParForProgramBlock parfor, HashSet<ParForProgramBlock> recPBs, LocalVariableMap vars) throws DMLRuntimeException, HopsException, LanguageException {
    //unfold if found
    if (n.getNodeType() == NodeType.FUNCCALL && n.isRecursive()) {
        boolean exists = rContainsNode(n, parfor);
        if (exists) {
            String fnameKey = n.getParam(ParamType.OPSTRING);
            String[] names = fnameKey.split(Program.KEY_DELIM);
            String fnamespace = names[0];
            String fname = names[1];
            String fnameNew = FUNCTION_UNFOLD_NAMEPREFIX + fname;
            //unfold function
            FunctionOp fop = (FunctionOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
            Program prog = parfor.getProgram();
            DMLProgram dmlprog = parfor.getStatementBlock().getDMLProg();
            FunctionProgramBlock fpb = prog.getFunctionProgramBlock(fnamespace, fname);
            FunctionProgramBlock copyfpb = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<String>(), new HashSet<String>());
            prog.addFunctionProgramBlock(fnamespace, fnameNew, copyfpb);
            dmlprog.addFunctionStatementBlock(fnamespace, fnameNew, (FunctionStatementBlock) copyfpb.getStatementBlock());
            //replace function names in old subtree (link to new function)
            rReplaceFunctionNames(n, fname, fnameNew);
            //recreate sub opttree
            String fnameNewKey = fnamespace + Program.KEY_DELIM + fnameNew;
            OptNode nNew = new OptNode(NodeType.FUNCCALL);
            OptTreeConverter.getAbstractPlanMapping().putHopMapping(fop, nNew);
            nNew.setExecType(ExecType.CP);
            nNew.addParam(ParamType.OPSTRING, fnameNewKey);
            long parentID = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
            OptTreeConverter.getAbstractPlanMapping().getOptNode(parentID).exchangeChild(n, nNew);
            HashSet<String> memo = new HashSet<String>();
            //required if functionop not shared (because not replaced yet)
            memo.add(fnameKey);
            //requied if functionop shared (indirectly replaced)
            memo.add(fnameNewKey);
            for (int i = 0; i < copyfpb.getChildBlocks().size(); /*&& i<len*/
            i++) {
                ProgramBlock lpb = copyfpb.getChildBlocks().get(i);
                StatementBlock lsb = lpb.getStatementBlock();
                nNew.addChild(OptTreeConverter.rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
            //compute delta for recPB set (use for removing parfor)
            recPBs.removeAll(rGetAllParForPBs(n, new HashSet<ParForProgramBlock>()));
            recPBs.addAll(rGetAllParForPBs(nNew, new HashSet<ParForProgramBlock>()));
            //replace function names in new subtree (recursive link to new function)
            rReplaceFunctionNames(nNew, fname, fnameNew);
        }
        return;
    }
    //recursive invocation (only for non-recursive functions)
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) rFindAndUnfoldRecursiveFunction(c, parfor, recPBs, vars);
}

Example 13 with Program

use of org.apache.sysml.runtime.controlprogram.Program in project incubator-systemml by apache.

the class OptTreePlanChecker method checkProgramCorrectness.

public static void checkProgramCorrectness(ProgramBlock pb, StatementBlock sb, Set<String> fnStack) throws HopsException, DMLRuntimeException {
    Program prog = pb.getProgram();
    DMLProgram dprog = sb.getDMLProg();
    if (pb instanceof FunctionProgramBlock && sb instanceof FunctionStatementBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        for (int i = 0; i < fpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = fpb.getChildBlocks().get(i);
            StatementBlock sbc = fstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
    //checkLinksProgramStatementBlock(fpb, fsb);
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, wsb.getPredicateHops(), wpb.getPredicate(), fnStack);
        for (int i = 0; i < wpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = wpb.getChildBlocks().get(i);
            StatementBlock sbc = wstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(wpb, wsb);
    } else if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, isb.getPredicateHops(), ipb.getPredicate(), fnStack);
        for (int i = 0; i < ipb.getChildBlocksIfBody().size(); i++) {
            ProgramBlock pbc = ipb.getChildBlocksIfBody().get(i);
            StatementBlock sbc = istmt.getIfBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        for (int i = 0; i < ipb.getChildBlocksElseBody().size(); i++) {
            ProgramBlock pbc = ipb.getChildBlocksElseBody().get(i);
            StatementBlock sbc = istmt.getElseBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(ipb, isb);
    } else if (//incl parfor
    pb instanceof ForProgramBlock && sb instanceof ForStatementBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fstmt = (ForStatement) sb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, fsb.getFromHops(), fpb.getFromInstructions(), fnStack);
        checkHopDagCorrectness(prog, dprog, fsb.getToHops(), fpb.getToInstructions(), fnStack);
        checkHopDagCorrectness(prog, dprog, fsb.getIncrementHops(), fpb.getIncrementInstructions(), fnStack);
        for (int i = 0; i < fpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = fpb.getChildBlocks().get(i);
            StatementBlock sbc = fstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(fpb, fsb);
    } else {
        checkHopDagCorrectness(prog, dprog, sb.get_hops(), pb.getInstructions(), fnStack);
    //checkLinksProgramStatementBlock(pb, sb);
    }
}

Example 14 with Program

use of org.apache.sysml.runtime.controlprogram.Program in project incubator-systemml by apache.

the class DMLScript method execute.

///////////////////////////////
// private internal interface 
// (core compilation and execute)
////////
/**
	 * The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
	 * and customized parameters have been put into _argVals
	 * 
	 * @param dmlScriptStr DML script string
	 * @param fnameOptConfig configuration file
	 * @param argVals map of argument values
	 * @param allArgs arguments
	 * @param scriptType type of script (DML or PyDML)
	 * @throws ParseException if ParseException occurs
	 * @throws IOException if IOException occurs
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws LanguageException if LanguageException occurs
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String, String> argVals, String[] allArgs, ScriptType scriptType) throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException {
    SCRIPT_TYPE = scriptType;
    //print basic time and environment info
    printStartExecInfo(dmlScriptStr);
    //Step 1: parse configuration files
    DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
    ConfigurationManager.setGlobalConfig(dmlconf);
    CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
    ConfigurationManager.setGlobalConfig(cconf);
    LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());
    //Step 2: set local/remote memory if requested (for compile in AM context) 
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        DMLAppMasterUtils.setupConfigRemoteMaxMemory(dmlconf);
    }
    //Step 3: parse dml script
    Statistics.startCompileTimer();
    ParserWrapper parser = ParserFactory.createParser(scriptType);
    DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
    //Step 4: construct HOP DAGs (incl LVA, validate, and setup)
    DMLTranslator dmlt = new DMLTranslator(prog);
    dmlt.liveVariableAnalysis(prog);
    dmlt.validateParseTree(prog);
    dmlt.constructHops(prog);
    //init working directories (before usage by following compilation steps)
    initHadoopExecution(dmlconf);
    //Step 5: rewrite HOP DAGs (incl IPA and memory estimates)
    dmlt.rewriteHopsDAG(prog);
    //Step 5.1: Generate code for the rewritten Hop dags 
    if (dmlconf.getBooleanValue(DMLConfig.CODEGEN)) {
        SpoofCompiler.PLAN_CACHE_POLICY = PlanCachePolicy.get(dmlconf.getBooleanValue(DMLConfig.CODEGEN_PLANCACHE), dmlconf.getIntValue(DMLConfig.CODEGEN_LITERALS) == 2);
        SpoofCompiler.setExecTypeSpecificJavaCompiler();
        if (SpoofCompiler.INTEGRATION == IntegrationType.HOPS)
            dmlt.codgenHopsDAG(prog);
    }
    //Step 6: construct lops (incl exec type and op selection)
    dmlt.constructLops(prog);
    if (LOG.isDebugEnabled()) {
        LOG.debug("\n********************** LOPS DAG *******************");
        dmlt.printLops(prog);
        dmlt.resetLopsDAGVisitStatus(prog);
    }
    //Step 7: generate runtime program
    Program rtprog = prog.getRuntimeProgram(dmlconf);
    //Step 7.1: Generate code for the rewritten Hop dags w/o modify
    if (dmlconf.getBooleanValue(DMLConfig.CODEGEN) && SpoofCompiler.INTEGRATION == IntegrationType.RUNTIME) {
        dmlt.codgenHopsDAG(rtprog);
    }
    //Step 8: [optional global data flow optimization]
    if (OptimizerUtils.isOptLevel(OptimizationLevel.O4_GLOBAL_TIME_MEMORY)) {
        LOG.warn("Optimization level '" + OptimizationLevel.O4_GLOBAL_TIME_MEMORY + "' " + "is still in experimental state and not intended for production use.");
        rtprog = GlobalOptimizerWrapper.optimizeProgram(prog, rtprog);
    }
    //launch SystemML appmaster (if requested and not already in launched AM)
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        if (!isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, dmlconf, allArgs, rtprog))
            //if AM launch unsuccessful, fall back to normal execute
            return;
        if (//in AM context (not failed AM launch)
        isActiveAM())
            DMLAppMasterUtils.setupProgramMappingRemoteMaxMemory(rtprog);
    }
    //Step 9: prepare statistics [and optional explain output]
    //count number compiled MR jobs / SP instructions	
    ExplainCounts counts = Explain.countDistributedOperations(rtprog);
    Statistics.resetNoOfCompiledJobs(counts.numJobs);
    //explain plan of program (hops or runtime)
    if (EXPLAIN != ExplainType.NONE) {
        LOG.info("EXPLAIN (" + EXPLAIN.toString() + "):\n" + Explain.explainMemoryBudget(counts) + "\n" + Explain.explainDegreeOfParallelism(counts) + Explain.explain(prog, rtprog, EXPLAIN));
    }
    Statistics.stopCompileTimer();
    //double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
    //System.out.println("Estimated costs: "+costs);
    //Step 10: execute runtime program
    ExecutionContext ec = null;
    try {
        ec = ExecutionContextFactory.createContext(rtprog);
        ScriptExecutorUtils.executeRuntimeProgram(rtprog, ec, dmlconf, STATISTICS ? STATISTICS_COUNT : 0);
    } finally {
        if (ec != null && ec instanceof SparkExecutionContext)
            ((SparkExecutionContext) ec).close();
        LOG.info("END DML run " + getDateTime());
        //cleanup scratch_space and all working dirs
        cleanupHadoopExecution(dmlconf);
    }
}

Example 15 with Program

use of org.apache.sysml.runtime.controlprogram.Program in project incubator-systemml by apache.

the class GDFEnumOptimizer method costRuntimePlan.

private static double costRuntimePlan(Plan p) throws DMLRuntimeException {
    Program prog = p.getNode().getProgram();
    if (prog == null)
        throw new DMLRuntimeException("Program not available for runtime plan costing.");
    //put data flow configuration into program
    rSetRuntimePlanConfig(p, new HashMap<Long, Plan>());
    double costs = -1;
    if (COST_FULL_PROGRAMS || (p.getNode().getHop() == null || p.getNode().getProgramBlock() == null)) {
        //recompile entire runtime program
        Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
        _compiledPlans++;
        //cost entire runtime program
        ExecutionContext ec = ExecutionContextFactory.createContext(prog);
        costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
    } else {
        Hop currentHop = p.getNode().getHop();
        ProgramBlock pb = p.getNode().getProgramBlock();
        try {
            //keep the old dag roots
            ArrayList<Hop> oldRoots = pb.getStatementBlock().get_hops();
            Hop tmpHop = null;
            if (!(currentHop instanceof DataOp && ((DataOp) currentHop).isWrite())) {
                ArrayList<Hop> newRoots = new ArrayList<Hop>();
                tmpHop = new DataOp("_tmp", currentHop.getDataType(), currentHop.getValueType(), currentHop, DataOpTypes.TRANSIENTWRITE, "tmp");
                //ensure recursive visitstatus reset on recompile
                tmpHop.setVisited();
                newRoots.add(tmpHop);
                pb.getStatementBlock().set_hops(newRoots);
            }
            //recompile modified runtime program
            Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
            _compiledPlans++;
            //cost partial runtime program up to current hop
            ExecutionContext ec = ExecutionContextFactory.createContext(prog);
            costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
            //restore original hop dag
            if (tmpHop != null)
                HopRewriteUtils.removeChildReference(tmpHop, currentHop);
            pb.getStatementBlock().set_hops(oldRoots);
        } catch (HopsException ex) {
            throw new DMLRuntimeException(ex);
        }
    }
    //release forced data flow configuration from program
    rResetRuntimePlanConfig(p, new HashMap<Long, Plan>());
    _costedPlans++;
    return costs;
}