Examples with ProgramBlock org.apache.sysml.runtime.controlprogram.ProgramBlock used on opensource projects

Example 51 with ProgramBlock

use of org.apache.sysml.runtime.controlprogram.ProgramBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rFindAndUnfoldRecursiveFunction.

protected void rFindAndUnfoldRecursiveFunction(OptNode n, ParForProgramBlock parfor, HashSet<ParForProgramBlock> recPBs, LocalVariableMap vars) {
    // unfold if found
    if (n.getNodeType() == NodeType.FUNCCALL && n.isRecursive()) {
        boolean exists = rContainsNode(n, parfor);
        if (exists) {
            String fnameKey = n.getParam(ParamType.OPSTRING);
            String[] names = fnameKey.split(Program.KEY_DELIM);
            String fnamespace = names[0];
            String fname = names[1];
            String fnameNew = FUNCTION_UNFOLD_NAMEPREFIX + fname;
            // unfold function
            FunctionOp fop = (FunctionOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
            Program prog = parfor.getProgram();
            DMLProgram dmlprog = parfor.getStatementBlock().getDMLProg();
            FunctionProgramBlock fpb = prog.getFunctionProgramBlock(fnamespace, fname);
            FunctionProgramBlock copyfpb = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<String>(), new HashSet<String>());
            prog.addFunctionProgramBlock(fnamespace, fnameNew, copyfpb);
            dmlprog.addFunctionStatementBlock(fnamespace, fnameNew, (FunctionStatementBlock) copyfpb.getStatementBlock());
            // replace function names in old subtree (link to new function)
            rReplaceFunctionNames(n, fname, fnameNew);
            // recreate sub opttree
            String fnameNewKey = fnamespace + Program.KEY_DELIM + fnameNew;
            OptNode nNew = new OptNode(NodeType.FUNCCALL);
            OptTreeConverter.getAbstractPlanMapping().putHopMapping(fop, nNew);
            nNew.setExecType(ExecType.CP);
            nNew.addParam(ParamType.OPSTRING, fnameNewKey);
            long parentID = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
            OptTreeConverter.getAbstractPlanMapping().getOptNode(parentID).exchangeChild(n, nNew);
            HashSet<String> memo = new HashSet<>();
            // required if functionop not shared (because not replaced yet)
            memo.add(fnameKey);
            // requied if functionop shared (indirectly replaced)
            memo.add(fnameNewKey);
            for (int i = 0; i < copyfpb.getChildBlocks().size(); /*&& i<len*/
            i++) {
                ProgramBlock lpb = copyfpb.getChildBlocks().get(i);
                StatementBlock lsb = lpb.getStatementBlock();
                nNew.addChild(OptTreeConverter.rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
            // compute delta for recPB set (use for removing parfor)
            recPBs.removeAll(rGetAllParForPBs(n, new HashSet<ParForProgramBlock>()));
            recPBs.addAll(rGetAllParForPBs(nNew, new HashSet<ParForProgramBlock>()));
            // replace function names in new subtree (recursive link to new function)
            rReplaceFunctionNames(nNew, fname, fnameNew);
        }
        return;
    }
    // recursive invocation (only for non-recursive functions)
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) rFindAndUnfoldRecursiveFunction(c, parfor, recPBs, vars);
}

Example 52 with ProgramBlock

use of org.apache.sysml.runtime.controlprogram.ProgramBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rAssignRemainingParallelism.

protected void rAssignRemainingParallelism(OptNode n, int parforK, int opsK) {
    ArrayList<OptNode> childs = n.getChilds();
    if (childs != null) {
        boolean recompileSB = false;
        for (OptNode c : childs) {
            if (c.getNodeType() == NodeType.PARFOR) {
                // constrain max parfor parallelism by problem size
                int tmpN = Integer.parseInt(c.getParam(ParamType.NUM_ITERATIONS));
                int tmpK = (tmpN < parforK) ? tmpN : parforK;
                // set parfor degree of parallelism
                long id = c.getID();
                c.setK(tmpK);
                ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(id)[1];
                pfpb.setDegreeOfParallelism(tmpK);
                // distribute remaining parallelism
                int remainParforK = getRemainingParallelismParFor(parforK, tmpK);
                int remainOpsK = getRemainingParallelismOps(opsK, tmpK);
                rAssignRemainingParallelism(c, remainParforK, remainOpsK);
            } else if (c.getNodeType() == NodeType.HOP) {
                // set degree of parallelism for multi-threaded leaf nodes
                Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(c.getID());
                if (ConfigurationManager.isParallelMatrixOperations() && // abop, datagenop, qop, paramop
                h instanceof MultiThreadedHop && !(// only paramop-grpagg
                h instanceof ParameterizedBuiltinOp && !HopRewriteUtils.isValidOp(((ParameterizedBuiltinOp) h).getOp(), ParamBuiltinOp.GROUPEDAGG, ParamBuiltinOp.REXPAND)) && !(// only unaryop-cumulativeagg
                h instanceof UnaryOp && !((UnaryOp) h).isCumulativeUnaryOperation()) && !(// only reorgop-transpose
                h instanceof ReorgOp && ((ReorgOp) h).getOp() != ReOrgOp.TRANSPOSE)) {
                    MultiThreadedHop mhop = (MultiThreadedHop) h;
                    // set max constraint in hop
                    mhop.setMaxNumThreads(opsK);
                    // set optnode k (for explain)
                    c.setK(opsK);
                    // need to recompile SB, if changed constraint
                    recompileSB = true;
                } else // for all other multi-threaded hops set k=1 to simply debugging
                if (h instanceof MultiThreadedHop) {
                    MultiThreadedHop mhop = (MultiThreadedHop) h;
                    // set max constraint in hop
                    mhop.setMaxNumThreads(1);
                    // set optnode k (for explain)
                    c.setK(1);
                }
            } else
                rAssignRemainingParallelism(c, parforK, opsK);
        }
        // recompile statement block if required
        if (recompileSB) {
            try {
                // guaranteed to be a last-level block (see hop change)
                ProgramBlock pb = (ProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
                Recompiler.recompileProgramBlockInstructions(pb);
            } catch (Exception ex) {
                throw new DMLRuntimeException(ex);
            }
        }
    }
}

Example 53 with ProgramBlock

use of org.apache.sysml.runtime.controlprogram.ProgramBlock in project incubator-systemml by apache.

the class ProgramRecompiler method generatePartitialRuntimeProgram.

public static ArrayList<ProgramBlock> generatePartitialRuntimeProgram(Program rtprog, ArrayList<StatementBlock> sbs) {
    ArrayList<ProgramBlock> ret = new ArrayList<>();
    DMLConfig config = ConfigurationManager.getDMLConfig();
    // construct lops from hops if not existing
    DMLTranslator dmlt = new DMLTranslator(sbs.get(0).getDMLProg());
    for (StatementBlock sb : sbs) {
        dmlt.constructLops(sb);
    }
    // construct runtime program from lops
    for (StatementBlock sb : sbs) {
        ret.add(dmlt.createRuntimeProgramBlock(rtprog, sb, config));
    }
    // enhance runtime program by automatic operator fusion
    if (ConfigurationManager.isCodegenEnabled() && SpoofCompiler.INTEGRATION == IntegrationType.RUNTIME) {
        for (ProgramBlock pb : ret) dmlt.codgenHopsDAG(pb);
    }
    return ret;
}

Example 54 with ProgramBlock

use of org.apache.sysml.runtime.controlprogram.ProgramBlock in project incubator-systemml by apache.

the class Explain method countCompiledInstructions.

/**
 * Recursively counts the number of compiled MRJob instructions in the
 * given runtime program block.
 *
 * @param pb program block
 * @param counts explain countst
 * @param MR if true, count Hadoop instructions
 * @param CP if true, count CP instructions
 * @param SP if true, count Spark instructions
 */
private static void countCompiledInstructions(ProgramBlock pb, ExplainCounts counts, boolean MR, boolean CP, boolean SP) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        countCompiledInstructions(tmp.getPredicate(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        countCompiledInstructions(tmp.getPredicate(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) countCompiledInstructions(pb2, counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        countCompiledInstructions(tmp.getFromInstructions(), counts, MR, CP, SP);
        countCompiledInstructions(tmp.getToInstructions(), counts, MR, CP, SP);
        countCompiledInstructions(tmp.getIncrementInstructions(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    // additional parfor jobs counted during runtime
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        for (ProgramBlock pb2 : fpb.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else {
        countCompiledInstructions(pb.getInstructions(), counts, MR, CP, SP);
    }
}

Example 55 with ProgramBlock

use of org.apache.sysml.runtime.controlprogram.ProgramBlock in project incubator-systemml by apache.

the class ResourceOptimizer method pruneProgramBlocks.

private static ArrayList<ProgramBlock> pruneProgramBlocks(ArrayList<ProgramBlock> B) {
    // prune all program blocks w/o mr instructions (mr budget does not matter)
    if (PRUNING_SMALL) {
        ArrayList<ProgramBlock> Bp = new ArrayList<>();
        for (ProgramBlock pb : B) if (OptTreeConverter.containsMRJobInstruction(pb.getInstructions(), false, true))
            Bp.add(pb);
        B = Bp;
    }
    // prune all program blocks, where all mr hops are due to unknowns
    if (PRUNING_UNKNOWN) {
        ArrayList<ProgramBlock> Bp = new ArrayList<>();
        for (ProgramBlock pb : B) if (!pruneHasOnlyUnknownMR(pb))
            Bp.add(pb);
        B = Bp;
    }
    return B;
}