Examples with ParForProgramBlock org.apache.sysml.runtime.controlprogram.ParForProgramBlock used on opensource projects

Example 31 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class ProgramConverter method rcreateDeepCopyProgramBlocks.

/**
 * This recursively creates a deep copy of program blocks and transparently replaces filenames according to the
 * specified parallel worker in order to avoid conflicts between parworkers. This happens recursively in order
 * to support arbitrary control-flow constructs within a parfor.
 *
 * @param childBlocks child program blocks
 * @param pid ?
 * @param IDPrefix ?
 * @param fnStack ?
 * @param fnCreated ?
 * @param plain if true, full deep copy without id replacement
 * @param forceDeepCopy if true, force deep copy
 * @return list of program blocks
 */
public static ArrayList<ProgramBlock> rcreateDeepCopyProgramBlocks(ArrayList<ProgramBlock> childBlocks, long pid, int IDPrefix, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy) {
    ArrayList<ProgramBlock> tmp = new ArrayList<>();
    for (ProgramBlock pb : childBlocks) {
        Program prog = pb.getProgram();
        ProgramBlock tmpPB = null;
        if (pb instanceof WhileProgramBlock) {
            tmpPB = createDeepCopyWhileProgramBlock((WhileProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
        } else if (pb instanceof ForProgramBlock && !(pb instanceof ParForProgramBlock)) {
            tmpPB = createDeepCopyForProgramBlock((ForProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
        } else if (pb instanceof ParForProgramBlock) {
            ParForProgramBlock pfpb = (ParForProgramBlock) pb;
            if (ParForProgramBlock.ALLOW_NESTED_PARALLELISM)
                tmpPB = createDeepCopyParForProgramBlock(pfpb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
            else
                tmpPB = createDeepCopyForProgramBlock((ForProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
        } else if (pb instanceof IfProgramBlock) {
            tmpPB = createDeepCopyIfProgramBlock((IfProgramBlock) pb, pid, IDPrefix, prog, fnStack, fnCreated, plain, forceDeepCopy);
        } else // last-level program block
        {
            // general case use for most PBs
            tmpPB = new ProgramBlock(prog);
            // for recompile in the master node JVM
            tmpPB.setStatementBlock(createStatementBlockCopy(pb.getStatementBlock(), pid, plain, forceDeepCopy));
            // tmpPB.setStatementBlock(pb.getStatementBlock());
            tmpPB.setThreadID(pid);
        }
        // copy instructions
        tmpPB.setInstructions(createDeepCopyInstructionSet(pb.getInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true));
        // copy symbol table
        // tmpPB.setVariables( pb.getVariables() ); //implicit cloning
        tmp.add(tmpPB);
    }
    return tmp;
}

Example 32 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNode.

public static OptNode rCreateAbstractOptNode(StatementBlock sb, ProgramBlock pb, LocalVariableMap vars, boolean topLevel, Set<String> memo) {
    OptNode node = null;
    if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement is = (IfStatement) isb.getStatement(0);
        node = new OptNode(NodeType.IF);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(isb.getBeginLine(), isb.getEndLine());
        // handle predicate
        isb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(isb.getPredicateHops(), vars, memo));
        // process if branch
        OptNode ifn = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, ifn);
        ifn.setExecType(ExecType.CP);
        node.addChild(ifn);
        int len = is.getIfBody().size();
        for (int i = 0; i < ipb.getChildBlocksIfBody().size() && i < len; i++) {
            ProgramBlock lpb = ipb.getChildBlocksIfBody().get(i);
            StatementBlock lsb = is.getIfBody().get(i);
            ifn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        // process else branch
        if (ipb.getChildBlocksElseBody() != null) {
            OptNode efn = new OptNode(NodeType.GENERIC);
            _hlMap.putProgMapping(sb, pb, efn);
            efn.setExecType(ExecType.CP);
            node.addChild(efn);
            int len2 = is.getElseBody().size();
            for (int i = 0; i < ipb.getChildBlocksElseBody().size() && i < len2; i++) {
                ProgramBlock lpb = ipb.getChildBlocksElseBody().get(i);
                StatementBlock lsb = is.getElseBody().get(i);
                efn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
        }
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement ws = (WhileStatement) wsb.getStatement(0);
        node = new OptNode(NodeType.WHILE);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(wsb.getBeginLine(), wsb.getEndLine());
        // handle predicate
        wsb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(wsb.getPredicateHops(), vars, memo));
        // process body
        int len = ws.getBody().size();
        for (int i = 0; i < wpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = wpb.getChildBlocks().get(i);
            StatementBlock lsb = ws.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ForProgramBlock && sb instanceof ForStatementBlock && !(pb instanceof ParForProgramBlock)) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.FOR);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        // determine number of iterations
        long N = OptimizerUtils.getNumIterations(fpb, vars, CostEstimator.FACTOR_NUM_ITERATIONS);
        node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(N));
        // handle predicate
        fsb.getFromHops().resetVisitStatus();
        fsb.getToHops().resetVisitStatus();
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
        node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
        if (fsb.getIncrementHops() != null)
            node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        // process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ParForProgramBlock && sb instanceof ParForStatementBlock) {
        ParForProgramBlock fpb = (ParForProgramBlock) pb;
        ParForStatementBlock fsb = (ParForStatementBlock) sb;
        ParForStatement fs = (ParForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.PARFOR);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        _hlMap.putProgMapping(sb, pb, node);
        node.setK(fpb.getDegreeOfParallelism());
        long N = fpb.getNumIterations();
        node.addParam(ParamType.NUM_ITERATIONS, (N != -1) ? String.valueOf(N) : String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
        switch(fpb.getExecMode()) {
            case LOCAL:
                node.setExecType(ExecType.CP);
                break;
            case REMOTE_MR:
            case REMOTE_MR_DP:
                node.setExecType(ExecType.MR);
                break;
            case REMOTE_SPARK:
            case REMOTE_SPARK_DP:
                node.setExecType(ExecType.SPARK);
                break;
            case UNSPECIFIED:
                node.setExecType(null);
        }
        if (!topLevel) {
            fsb.getFromHops().resetVisitStatus();
            fsb.getToHops().resetVisitStatus();
            if (fsb.getIncrementHops() != null)
                fsb.getIncrementHops().resetVisitStatus();
            node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
            node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
            if (fsb.getIncrementHops() != null)
                node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        }
        // process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        // parameters, add required parameters
        Map<String, String> lparams = fpb.getParForParams();
        node.addParam(ParamType.DATA_PARTITIONER, lparams.get(ParForStatementBlock.DATA_PARTITIONER));
        node.addParam(ParamType.TASK_PARTITIONER, lparams.get(ParForStatementBlock.TASK_PARTITIONER));
        node.addParam(ParamType.RESULT_MERGE, lparams.get(ParForStatementBlock.RESULT_MERGE));
    // TODO task size
    } else // last level program block
    {
        sb = pb.getStatementBlock();
        // process all hops
        node = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, node);
        node.addChilds(createAbstractOptNodes(sb.getHops(), vars, memo));
        node.setExecType(ExecType.CP);
        node.setLineNumbers(sb.getBeginLine(), sb.getEndLine());
        // TODO remove this workaround once this information can be obtained from hops/lops compiler
        if (node.isCPOnly()) {
            boolean isSparkExec = OptimizerUtils.isSparkExecutionMode();
            if (!isSparkExec && containsMRJobInstruction(pb, false, false))
                node.setExecType(ExecType.MR);
            else if (isSparkExec && containsMRJobInstruction(pb, false, true))
                node.setExecType(ExecType.SPARK);
        }
    }
    // final cleanup
    // NOTE: required because this function is also used to create subtrees
    node.checkAndCleanupLeafNodes();
    return node;
}

Example 33 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptTreeConverter method rContainsMRJobInstruction.

public static boolean rContainsMRJobInstruction(ProgramBlock pb, boolean inclFunctions) {
    boolean ret = false;
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getPredicate(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getPredicate(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
        for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getFromInstructions(), true, true);
        ret |= containsMRJobInstruction(tmp.getToInstructions(), true, true);
        ret |= containsMRJobInstruction(tmp.getIncrementInstructions(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP)
    pb instanceof FunctionProgramBlock) {
    // do nothing
    } else {
        ret = containsMRJobInstruction(pb, true, true) || (inclFunctions && containsFunctionCallInstruction(pb));
    }
    return ret;
}

Example 34 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptTreeConverter method replaceProgramBlock.

public static void replaceProgramBlock(OptNode parent, OptNode n, ProgramBlock pbOld, ProgramBlock pbNew, boolean rtMap) {
    ProgramBlock pbParent = null;
    if (rtMap)
        pbParent = (ProgramBlock) _rtMap.getMappedObject(parent.getID());
    else {
        if (parent.getNodeType() == NodeType.FUNCCALL) {
            FunctionOp fop = (FunctionOp) _hlMap.getMappedHop(parent.getID());
            pbParent = ((Program) _hlMap.getRootProgram()[1]).getFunctionProgramBlock(fop.getFunctionNamespace(), fop.getFunctionName());
        } else
            pbParent = (ProgramBlock) _hlMap.getMappedProg(parent.getID())[1];
    }
    if (pbParent instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pbParent;
        replaceProgramBlock(ipb.getChildBlocksIfBody(), pbOld, pbNew);
        replaceProgramBlock(ipb.getChildBlocksElseBody(), pbOld, pbNew);
    } else if (pbParent instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pbParent;
        replaceProgramBlock(wpb.getChildBlocks(), pbOld, pbNew);
    } else if (pbParent instanceof ForProgramBlock || pbParent instanceof ParForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pbParent;
        replaceProgramBlock(fpb.getChildBlocks(), pbOld, pbNew);
    } else if (pbParent instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pbParent;
        replaceProgramBlock(fpb.getChildBlocks(), pbOld, pbNew);
    } else
        throw new DMLRuntimeException("Optimizer doesn't support " + pbParent.getClass().getName());
    // update repository
    if (rtMap)
        _rtMap.replaceMapping(pbNew, n);
    else
        _hlMap.replaceMapping(pbNew, n);
}

Example 35 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptimizerConstrained method rewriteSetDegreeOfParallelism.

// /////
// REWRITE set degree of parallelism
// /
@Override
protected void rewriteSetDegreeOfParallelism(OptNode n, double M, boolean flagNested) {
    // constraint awareness
    if (n.getK() > 0 && ConfigurationManager.isParallelParFor()) {
        // set parfor degree of parallelism
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
        pfpb.setDegreeOfParallelism(n.getK());
        // distribute remaining parallelism
        int remainParforK = getRemainingParallelismParFor(n.getK(), n.getK());
        int remainOpsK = getRemainingParallelismOps(_lkmaxCP, n.getK());
        rAssignRemainingParallelism(n, remainParforK, remainOpsK);
        LOG.debug(getOptMode() + " OPT: forced 'set degree of parallelism' - result=(see EXPLAIN)");
    } else
        super.rewriteSetDegreeOfParallelism(n, M, flagNested);
}