Examples with StatementBlock org.apache.sysml.parser.StatementBlock used on opensource projects

Example 51 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNode.

public static OptNode rCreateAbstractOptNode(StatementBlock sb, ProgramBlock pb, LocalVariableMap vars, boolean topLevel, Set<String> memo) {
    OptNode node = null;
    if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement is = (IfStatement) isb.getStatement(0);
        node = new OptNode(NodeType.IF);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(isb.getBeginLine(), isb.getEndLine());
        // handle predicate
        isb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(isb.getPredicateHops(), vars, memo));
        // process if branch
        OptNode ifn = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, ifn);
        ifn.setExecType(ExecType.CP);
        node.addChild(ifn);
        int len = is.getIfBody().size();
        for (int i = 0; i < ipb.getChildBlocksIfBody().size() && i < len; i++) {
            ProgramBlock lpb = ipb.getChildBlocksIfBody().get(i);
            StatementBlock lsb = is.getIfBody().get(i);
            ifn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        // process else branch
        if (ipb.getChildBlocksElseBody() != null) {
            OptNode efn = new OptNode(NodeType.GENERIC);
            _hlMap.putProgMapping(sb, pb, efn);
            efn.setExecType(ExecType.CP);
            node.addChild(efn);
            int len2 = is.getElseBody().size();
            for (int i = 0; i < ipb.getChildBlocksElseBody().size() && i < len2; i++) {
                ProgramBlock lpb = ipb.getChildBlocksElseBody().get(i);
                StatementBlock lsb = is.getElseBody().get(i);
                efn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
        }
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement ws = (WhileStatement) wsb.getStatement(0);
        node = new OptNode(NodeType.WHILE);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(wsb.getBeginLine(), wsb.getEndLine());
        // handle predicate
        wsb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(wsb.getPredicateHops(), vars, memo));
        // process body
        int len = ws.getBody().size();
        for (int i = 0; i < wpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = wpb.getChildBlocks().get(i);
            StatementBlock lsb = ws.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ForProgramBlock && sb instanceof ForStatementBlock && !(pb instanceof ParForProgramBlock)) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.FOR);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        // determine number of iterations
        long N = OptimizerUtils.getNumIterations(fpb, vars, CostEstimator.FACTOR_NUM_ITERATIONS);
        node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(N));
        // handle predicate
        fsb.getFromHops().resetVisitStatus();
        fsb.getToHops().resetVisitStatus();
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
        node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
        if (fsb.getIncrementHops() != null)
            node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        // process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ParForProgramBlock && sb instanceof ParForStatementBlock) {
        ParForProgramBlock fpb = (ParForProgramBlock) pb;
        ParForStatementBlock fsb = (ParForStatementBlock) sb;
        ParForStatement fs = (ParForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.PARFOR);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        _hlMap.putProgMapping(sb, pb, node);
        node.setK(fpb.getDegreeOfParallelism());
        long N = fpb.getNumIterations();
        node.addParam(ParamType.NUM_ITERATIONS, (N != -1) ? String.valueOf(N) : String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
        switch(fpb.getExecMode()) {
            case LOCAL:
                node.setExecType(ExecType.CP);
                break;
            case REMOTE_MR:
            case REMOTE_MR_DP:
                node.setExecType(ExecType.MR);
                break;
            case REMOTE_SPARK:
            case REMOTE_SPARK_DP:
                node.setExecType(ExecType.SPARK);
                break;
            case UNSPECIFIED:
                node.setExecType(null);
        }
        if (!topLevel) {
            fsb.getFromHops().resetVisitStatus();
            fsb.getToHops().resetVisitStatus();
            if (fsb.getIncrementHops() != null)
                fsb.getIncrementHops().resetVisitStatus();
            node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
            node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
            if (fsb.getIncrementHops() != null)
                node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        }
        // process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        // parameters, add required parameters
        Map<String, String> lparams = fpb.getParForParams();
        node.addParam(ParamType.DATA_PARTITIONER, lparams.get(ParForStatementBlock.DATA_PARTITIONER));
        node.addParam(ParamType.TASK_PARTITIONER, lparams.get(ParForStatementBlock.TASK_PARTITIONER));
        node.addParam(ParamType.RESULT_MERGE, lparams.get(ParForStatementBlock.RESULT_MERGE));
    // TODO task size
    } else // last level program block
    {
        sb = pb.getStatementBlock();
        // process all hops
        node = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, node);
        node.addChilds(createAbstractOptNodes(sb.getHops(), vars, memo));
        node.setExecType(ExecType.CP);
        node.setLineNumbers(sb.getBeginLine(), sb.getEndLine());
        // TODO remove this workaround once this information can be obtained from hops/lops compiler
        if (node.isCPOnly()) {
            boolean isSparkExec = OptimizerUtils.isSparkExecutionMode();
            if (!isSparkExec && containsMRJobInstruction(pb, false, false))
                node.setExecType(ExecType.MR);
            else if (isSparkExec && containsMRJobInstruction(pb, false, true))
                node.setExecType(ExecType.SPARK);
        }
    }
    // final cleanup
    // NOTE: required because this function is also used to create subtrees
    node.checkAndCleanupLeafNodes();
    return node;
}

Example 52 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class OptTreePlanChecker method checkProgramCorrectness.

public static void checkProgramCorrectness(ProgramBlock pb, StatementBlock sb, Set<String> fnStack) {
    Program prog = pb.getProgram();
    DMLProgram dprog = sb.getDMLProg();
    if (pb instanceof FunctionProgramBlock && sb instanceof FunctionStatementBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        for (int i = 0; i < fpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = fpb.getChildBlocks().get(i);
            StatementBlock sbc = fstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
    // checkLinksProgramStatementBlock(fpb, fsb);
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, wsb.getPredicateHops(), wpb.getPredicate(), fnStack);
        for (int i = 0; i < wpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = wpb.getChildBlocks().get(i);
            StatementBlock sbc = wstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(wpb, wsb);
    } else if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, isb.getPredicateHops(), ipb.getPredicate(), fnStack);
        for (int i = 0; i < ipb.getChildBlocksIfBody().size(); i++) {
            ProgramBlock pbc = ipb.getChildBlocksIfBody().get(i);
            StatementBlock sbc = istmt.getIfBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        for (int i = 0; i < ipb.getChildBlocksElseBody().size(); i++) {
            ProgramBlock pbc = ipb.getChildBlocksElseBody().get(i);
            StatementBlock sbc = istmt.getElseBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(ipb, isb);
    } else if (// incl parfor
    pb instanceof ForProgramBlock && sb instanceof ForStatementBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fstmt = (ForStatement) sb.getStatement(0);
        checkHopDagCorrectness(prog, dprog, fsb.getFromHops(), fpb.getFromInstructions(), fnStack);
        checkHopDagCorrectness(prog, dprog, fsb.getToHops(), fpb.getToInstructions(), fnStack);
        checkHopDagCorrectness(prog, dprog, fsb.getIncrementHops(), fpb.getIncrementInstructions(), fnStack);
        for (int i = 0; i < fpb.getChildBlocks().size(); i++) {
            ProgramBlock pbc = fpb.getChildBlocks().get(i);
            StatementBlock sbc = fstmt.getBody().get(i);
            checkProgramCorrectness(pbc, sbc, fnStack);
        }
        checkLinksProgramStatementBlock(fpb, fsb);
    } else {
        checkHopDagCorrectness(prog, dprog, sb.getHops(), pb.getInstructions(), fnStack);
    // checkLinksProgramStatementBlock(pb, sb);
    }
}

Example 53 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method recompileLIX.

protected void recompileLIX(OptNode n, LocalVariableMap vars) {
    Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
    // set forced exec type
    h.setForcedExecType(LopProperties.ExecType.CP);
    n.setExecType(ExecType.CP);
    // recompile parent pb
    long pid = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
    OptNode nParent = OptTreeConverter.getAbstractPlanMapping().getOptNode(pid);
    Object[] o = OptTreeConverter.getAbstractPlanMapping().getMappedProg(pid);
    StatementBlock sb = (StatementBlock) o[0];
    ProgramBlock pb = (ProgramBlock) o[1];
    // keep modified estimated of partitioned rix (in same dag as lix)
    HashMap<Hop, Double> estRix = getPartitionedRIXEstimates(nParent);
    // construct new instructions
    ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(sb, sb.getHops(), vars, null, false, false, 0);
    pb.setInstructions(newInst);
    // reset all rix estimated (modified by recompile)
    resetPartitionRIXEstimates(estRix);
    // set new mem estimate (last, otherwise overwritten from recompile)
    h.setMemEstimate(_rm - 1);
}

Example 54 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rFindAndUnfoldRecursiveFunction.

protected void rFindAndUnfoldRecursiveFunction(OptNode n, ParForProgramBlock parfor, HashSet<ParForProgramBlock> recPBs, LocalVariableMap vars) {
    // unfold if found
    if (n.getNodeType() == NodeType.FUNCCALL && n.isRecursive()) {
        boolean exists = rContainsNode(n, parfor);
        if (exists) {
            String fnameKey = n.getParam(ParamType.OPSTRING);
            String[] names = fnameKey.split(Program.KEY_DELIM);
            String fnamespace = names[0];
            String fname = names[1];
            String fnameNew = FUNCTION_UNFOLD_NAMEPREFIX + fname;
            // unfold function
            FunctionOp fop = (FunctionOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
            Program prog = parfor.getProgram();
            DMLProgram dmlprog = parfor.getStatementBlock().getDMLProg();
            FunctionProgramBlock fpb = prog.getFunctionProgramBlock(fnamespace, fname);
            FunctionProgramBlock copyfpb = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<String>(), new HashSet<String>());
            prog.addFunctionProgramBlock(fnamespace, fnameNew, copyfpb);
            dmlprog.addFunctionStatementBlock(fnamespace, fnameNew, (FunctionStatementBlock) copyfpb.getStatementBlock());
            // replace function names in old subtree (link to new function)
            rReplaceFunctionNames(n, fname, fnameNew);
            // recreate sub opttree
            String fnameNewKey = fnamespace + Program.KEY_DELIM + fnameNew;
            OptNode nNew = new OptNode(NodeType.FUNCCALL);
            OptTreeConverter.getAbstractPlanMapping().putHopMapping(fop, nNew);
            nNew.setExecType(ExecType.CP);
            nNew.addParam(ParamType.OPSTRING, fnameNewKey);
            long parentID = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
            OptTreeConverter.getAbstractPlanMapping().getOptNode(parentID).exchangeChild(n, nNew);
            HashSet<String> memo = new HashSet<>();
            // required if functionop not shared (because not replaced yet)
            memo.add(fnameKey);
            // requied if functionop shared (indirectly replaced)
            memo.add(fnameNewKey);
            for (int i = 0; i < copyfpb.getChildBlocks().size(); /*&& i<len*/
            i++) {
                ProgramBlock lpb = copyfpb.getChildBlocks().get(i);
                StatementBlock lsb = lpb.getStatementBlock();
                nNew.addChild(OptTreeConverter.rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
            // compute delta for recPB set (use for removing parfor)
            recPBs.removeAll(rGetAllParForPBs(n, new HashSet<ParForProgramBlock>()));
            recPBs.addAll(rGetAllParForPBs(nNew, new HashSet<ParForProgramBlock>()));
            // replace function names in new subtree (recursive link to new function)
            rReplaceFunctionNames(nNew, fname, fnameNew);
        }
        return;
    }
    // recursive invocation (only for non-recursive functions)
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) rFindAndUnfoldRecursiveFunction(c, parfor, recPBs, vars);
}

Example 55 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class ProgramRecompiler method replaceConstantScalarVariables.

public static void replaceConstantScalarVariables(StatementBlock sb, LocalVariableMap vars) {
    if (sb instanceof IfStatementBlock) {
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement is = (IfStatement) sb.getStatement(0);
        replacePredicateLiterals(isb.getPredicateHops(), vars);
        for (StatementBlock lsb : is.getIfBody()) replaceConstantScalarVariables(lsb, vars);
        for (StatementBlock lsb : is.getElseBody()) replaceConstantScalarVariables(lsb, vars);
    } else if (sb instanceof WhileStatementBlock) {
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement ws = (WhileStatement) sb.getStatement(0);
        replacePredicateLiterals(wsb.getPredicateHops(), vars);
        for (StatementBlock lsb : ws.getBody()) replaceConstantScalarVariables(lsb, vars);
    } else if (// for or parfor
    sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        replacePredicateLiterals(fsb.getFromHops(), vars);
        replacePredicateLiterals(fsb.getToHops(), vars);
        replacePredicateLiterals(fsb.getIncrementHops(), vars);
        for (StatementBlock lsb : fs.getBody()) replaceConstantScalarVariables(lsb, vars);
    } else // last level block
    {
        ArrayList<Hop> hops = sb.getHops();
        if (hops != null) {
            // replace constant literals
            Hop.resetVisitStatus(hops);
            for (Hop hopRoot : hops) Recompiler.rReplaceLiterals(hopRoot, vars, true);
        }
    }
}