Examples with ForProgramBlock org.apache.sysml.runtime.controlprogram.ForProgramBlock used on opensource projects

Example 51 with ForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project incubator-systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNode.

public static OptNode rCreateAbstractOptNode(StatementBlock sb, ProgramBlock pb, LocalVariableMap vars, boolean topLevel, Set<String> memo) {
    OptNode node = null;
    if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement is = (IfStatement) isb.getStatement(0);
        node = new OptNode(NodeType.IF);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(isb.getBeginLine(), isb.getEndLine());
        // handle predicate
        isb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(isb.getPredicateHops(), vars, memo));
        // process if branch
        OptNode ifn = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, ifn);
        ifn.setExecType(ExecType.CP);
        node.addChild(ifn);
        int len = is.getIfBody().size();
        for (int i = 0; i < ipb.getChildBlocksIfBody().size() && i < len; i++) {
            ProgramBlock lpb = ipb.getChildBlocksIfBody().get(i);
            StatementBlock lsb = is.getIfBody().get(i);
            ifn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        // process else branch
        if (ipb.getChildBlocksElseBody() != null) {
            OptNode efn = new OptNode(NodeType.GENERIC);
            _hlMap.putProgMapping(sb, pb, efn);
            efn.setExecType(ExecType.CP);
            node.addChild(efn);
            int len2 = is.getElseBody().size();
            for (int i = 0; i < ipb.getChildBlocksElseBody().size() && i < len2; i++) {
                ProgramBlock lpb = ipb.getChildBlocksElseBody().get(i);
                StatementBlock lsb = is.getElseBody().get(i);
                efn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
        }
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement ws = (WhileStatement) wsb.getStatement(0);
        node = new OptNode(NodeType.WHILE);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(wsb.getBeginLine(), wsb.getEndLine());
        // handle predicate
        wsb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(wsb.getPredicateHops(), vars, memo));
        // process body
        int len = ws.getBody().size();
        for (int i = 0; i < wpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = wpb.getChildBlocks().get(i);
            StatementBlock lsb = ws.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ForProgramBlock && sb instanceof ForStatementBlock && !(pb instanceof ParForProgramBlock)) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.FOR);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        // determine number of iterations
        long N = OptimizerUtils.getNumIterations(fpb, vars, CostEstimator.FACTOR_NUM_ITERATIONS);
        node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(N));
        // handle predicate
        fsb.getFromHops().resetVisitStatus();
        fsb.getToHops().resetVisitStatus();
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
        node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
        if (fsb.getIncrementHops() != null)
            node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        // process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ParForProgramBlock && sb instanceof ParForStatementBlock) {
        ParForProgramBlock fpb = (ParForProgramBlock) pb;
        ParForStatementBlock fsb = (ParForStatementBlock) sb;
        ParForStatement fs = (ParForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.PARFOR);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        _hlMap.putProgMapping(sb, pb, node);
        node.setK(fpb.getDegreeOfParallelism());
        long N = fpb.getNumIterations();
        node.addParam(ParamType.NUM_ITERATIONS, (N != -1) ? String.valueOf(N) : String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
        switch(fpb.getExecMode()) {
            case LOCAL:
                node.setExecType(ExecType.CP);
                break;
            case REMOTE_MR:
            case REMOTE_MR_DP:
                node.setExecType(ExecType.MR);
                break;
            case REMOTE_SPARK:
            case REMOTE_SPARK_DP:
                node.setExecType(ExecType.SPARK);
                break;
            case UNSPECIFIED:
                node.setExecType(null);
        }
        if (!topLevel) {
            fsb.getFromHops().resetVisitStatus();
            fsb.getToHops().resetVisitStatus();
            if (fsb.getIncrementHops() != null)
                fsb.getIncrementHops().resetVisitStatus();
            node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
            node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
            if (fsb.getIncrementHops() != null)
                node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        }
        // process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        // parameters, add required parameters
        Map<String, String> lparams = fpb.getParForParams();
        node.addParam(ParamType.DATA_PARTITIONER, lparams.get(ParForStatementBlock.DATA_PARTITIONER));
        node.addParam(ParamType.TASK_PARTITIONER, lparams.get(ParForStatementBlock.TASK_PARTITIONER));
        node.addParam(ParamType.RESULT_MERGE, lparams.get(ParForStatementBlock.RESULT_MERGE));
    // TODO task size
    } else // last level program block
    {
        sb = pb.getStatementBlock();
        // process all hops
        node = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, node);
        node.addChilds(createAbstractOptNodes(sb.getHops(), vars, memo));
        node.setExecType(ExecType.CP);
        node.setLineNumbers(sb.getBeginLine(), sb.getEndLine());
        // TODO remove this workaround once this information can be obtained from hops/lops compiler
        if (node.isCPOnly()) {
            boolean isSparkExec = OptimizerUtils.isSparkExecutionMode();
            if (!isSparkExec && containsMRJobInstruction(pb, false, false))
                node.setExecType(ExecType.MR);
            else if (isSparkExec && containsMRJobInstruction(pb, false, true))
                node.setExecType(ExecType.SPARK);
        }
    }
    // final cleanup
    // NOTE: required because this function is also used to create subtrees
    node.checkAndCleanupLeafNodes();
    return node;
}

Example 52 with ForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project incubator-systemml by apache.

the class OptTreeConverter method rContainsMRJobInstruction.

public static boolean rContainsMRJobInstruction(ProgramBlock pb, boolean inclFunctions) {
    boolean ret = false;
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getPredicate(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getPredicate(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
        for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getFromInstructions(), true, true);
        ret |= containsMRJobInstruction(tmp.getToInstructions(), true, true);
        ret |= containsMRJobInstruction(tmp.getIncrementInstructions(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP)
    pb instanceof FunctionProgramBlock) {
    // do nothing
    } else {
        ret = containsMRJobInstruction(pb, true, true) || (inclFunctions && containsFunctionCallInstruction(pb));
    }
    return ret;
}

Example 53 with ForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project incubator-systemml by apache.

the class OptTreeConverter method replaceProgramBlock.

public static void replaceProgramBlock(OptNode parent, OptNode n, ProgramBlock pbOld, ProgramBlock pbNew, boolean rtMap) {
    ProgramBlock pbParent = null;
    if (rtMap)
        pbParent = (ProgramBlock) _rtMap.getMappedObject(parent.getID());
    else {
        if (parent.getNodeType() == NodeType.FUNCCALL) {
            FunctionOp fop = (FunctionOp) _hlMap.getMappedHop(parent.getID());
            pbParent = ((Program) _hlMap.getRootProgram()[1]).getFunctionProgramBlock(fop.getFunctionNamespace(), fop.getFunctionName());
        } else
            pbParent = (ProgramBlock) _hlMap.getMappedProg(parent.getID())[1];
    }
    if (pbParent instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pbParent;
        replaceProgramBlock(ipb.getChildBlocksIfBody(), pbOld, pbNew);
        replaceProgramBlock(ipb.getChildBlocksElseBody(), pbOld, pbNew);
    } else if (pbParent instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pbParent;
        replaceProgramBlock(wpb.getChildBlocks(), pbOld, pbNew);
    } else if (pbParent instanceof ForProgramBlock || pbParent instanceof ParForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pbParent;
        replaceProgramBlock(fpb.getChildBlocks(), pbOld, pbNew);
    } else if (pbParent instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pbParent;
        replaceProgramBlock(fpb.getChildBlocks(), pbOld, pbNew);
    } else
        throw new DMLRuntimeException("Optimizer doesn't support " + pbParent.getClass().getName());
    // update repository
    if (rtMap)
        _rtMap.replaceMapping(pbNew, n);
    else
        _hlMap.replaceMapping(pbNew, n);
}

Example 54 with ForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project incubator-systemml by apache.

the class OptTreePlanChecker method checkProgramCorrectness.

public static void checkProgramCorrectness(ProgramBlock pb, StatementBlock sb, Set<String> fnStack) {
    Program prog = pb.getProgram();
    DMLProgram dprog = sb.getDMLProg();
    if (pb instanceof FunctionProgramBlock && sb instanceof FunctionStatementBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        for (int i = 0; i < fpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = fpb.getChildBlocks().get(i);
            StatementBlock sbc = fstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
    // checkLinksProgramStatementBlock(fpb, fsb);
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, wsb.getPredicateHops(), wpb.getPredicate(), fnStack);
        for (int i = 0; i < wpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = wpb.getChildBlocks().get(i);
            StatementBlock sbc = wstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(wpb, wsb);
    } else if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, isb.getPredicateHops(), ipb.getPredicate(), fnStack);
        for (int i = 0; i < ipb.getChildBlocksIfBody().size(); i++) {
            ProgramBlock pbc = ipb.getChildBlocksIfBody().get(i);
            StatementBlock sbc = istmt.getIfBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        for (int i = 0; i < ipb.getChildBlocksElseBody().size(); i++) {
            ProgramBlock pbc = ipb.getChildBlocksElseBody().get(i);
            StatementBlock sbc = istmt.getElseBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(ipb, isb);
    } else if (// incl parfor
    pb instanceof ForProgramBlock && sb instanceof ForStatementBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fstmt = (ForStatement) sb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, fsb.getFromHops(), fpb.getFromInstructions(), fnStack);
        checkHopDagCorrectness(prog, dprog, fsb.getToHops(), fpb.getToInstructions(), fnStack);
        checkHopDagCorrectness(prog, dprog, fsb.getIncrementHops(), fpb.getIncrementInstructions(), fnStack);
        for (int i = 0; i < fpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = fpb.getChildBlocks().get(i);
            StatementBlock sbc = fstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(fpb, fsb);
    } else {
        checkHopDagCorrectness(prog, dprog, sb.getHops(), pb.getInstructions(), fnStack);
    // checkLinksProgramStatementBlock(pb, sb);
    }
}

Example 55 with ForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project incubator-systemml by apache.

the class Explain method countCompiledInstructions.

/**
 * Recursively counts the number of compiled MRJob instructions in the
 * given runtime program block.
 *
 * @param pb program block
 * @param counts explain countst
 * @param MR if true, count Hadoop instructions
 * @param CP if true, count CP instructions
 * @param SP if true, count Spark instructions
 */
private static void countCompiledInstructions(ProgramBlock pb, ExplainCounts counts, boolean MR, boolean CP, boolean SP) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        countCompiledInstructions(tmp.getPredicate(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        countCompiledInstructions(tmp.getPredicate(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) countCompiledInstructions(pb2, counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        countCompiledInstructions(tmp.getFromInstructions(), counts, MR, CP, SP);
        countCompiledInstructions(tmp.getToInstructions(), counts, MR, CP, SP);
        countCompiledInstructions(tmp.getIncrementInstructions(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    // additional parfor jobs counted during runtime
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        for (ProgramBlock pb2 : fpb.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else {
        countCompiledInstructions(pb.getInstructions(), counts, MR, CP, SP);
    }
}