Examples with WhileProgramBlock org.apache.sysml.runtime.controlprogram.WhileProgramBlock used on opensource projects

Example 11 with WhileProgramBlock

use of org.apache.sysml.runtime.controlprogram.WhileProgramBlock in project incubator-systemml by apache.

the class ProgramRecompiler method rFindAndRecompileIndexingHOP.

/**
 * NOTE: if force is set, we set and recompile the respective indexing hops;
 * otherwise, we release the forced exec type and recompile again. Hence,
 * any changes can be exactly reverted with the same access behavior.
 *
 * @param sb statement block
 * @param pb program block
 * @param var variable
 * @param ec execution context
 * @param force if true, set and recompile the respective indexing hops
 */
public static void rFindAndRecompileIndexingHOP(StatementBlock sb, ProgramBlock pb, String var, ExecutionContext ec, boolean force) {
    if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement is = (IfStatement) sb.getStatement(0);
        // process if condition
        if (isb.getPredicateHops() != null)
            ipb.setPredicate(rFindAndRecompileIndexingHOP(isb.getPredicateHops(), ipb.getPredicate(), var, ec, force));
        // process if branch
        int len = is.getIfBody().size();
        for (int i = 0; i < ipb.getChildBlocksIfBody().size() && i < len; i++) {
            ProgramBlock lpb = ipb.getChildBlocksIfBody().get(i);
            StatementBlock lsb = is.getIfBody().get(i);
            rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
        }
        // process else branch
        if (ipb.getChildBlocksElseBody() != null) {
            int len2 = is.getElseBody().size();
            for (int i = 0; i < ipb.getChildBlocksElseBody().size() && i < len2; i++) {
                ProgramBlock lpb = ipb.getChildBlocksElseBody().get(i);
                StatementBlock lsb = is.getElseBody().get(i);
                rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
            }
        }
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement ws = (WhileStatement) sb.getStatement(0);
        // process while condition
        if (wsb.getPredicateHops() != null)
            wpb.setPredicate(rFindAndRecompileIndexingHOP(wsb.getPredicateHops(), wpb.getPredicate(), var, ec, force));
        // process body
        // robustness for potentially added problem blocks
        int len = ws.getBody().size();
        for (int i = 0; i < wpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = wpb.getChildBlocks().get(i);
            StatementBlock lsb = ws.getBody().get(i);
            rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
        }
    } else if (// for or parfor
    pb instanceof ForProgramBlock && sb instanceof ForStatementBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        if (fsb.getFromHops() != null)
            fpb.setFromInstructions(rFindAndRecompileIndexingHOP(fsb.getFromHops(), fpb.getFromInstructions(), var, ec, force));
        if (fsb.getToHops() != null)
            fpb.setToInstructions(rFindAndRecompileIndexingHOP(fsb.getToHops(), fpb.getToInstructions(), var, ec, force));
        if (fsb.getIncrementHops() != null)
            fpb.setIncrementInstructions(rFindAndRecompileIndexingHOP(fsb.getIncrementHops(), fpb.getIncrementInstructions(), var, ec, force));
        // process body
        // robustness for potentially added problem blocks
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
        }
    } else // last level program block
    {
        try {
            // process actual hops
            boolean ret = false;
            Hop.resetVisitStatus(sb.getHops());
            if (force) {
                // set forced execution type
                for (Hop h : sb.getHops()) ret |= rFindAndSetCPIndexingHOP(h, var);
            } else {
                // release forced execution type
                for (Hop h : sb.getHops()) ret |= rFindAndReleaseIndexingHOP(h, var);
            }
            // recompilation on-demand
            if (ret) {
                // construct new instructions
                ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(sb, sb.getHops(), ec.getVariables(), null, true, false, 0);
                pb.setInstructions(newInst);
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
    }
}

Example 12 with WhileProgramBlock

use of org.apache.sysml.runtime.controlprogram.WhileProgramBlock in project incubator-systemml by apache.

the class CostEstimator method rGetTimeEstimate.

private double rGetTimeEstimate(ProgramBlock pb, HashMap<String, VarStats> stats, HashSet<String> memoFunc, boolean recursive) {
    double ret = 0;
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
        ret *= DEFAULT_NUMITER;
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        if (recursive) {
            for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
            if (tmp.getChildBlocksElseBody() != null)
                for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) {
                    ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
                    // weighted sum
                    ret /= 2;
                }
        }
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
        ret *= getNumIterations(stats, tmp);
    } else if (pb instanceof FunctionProgramBlock && // see generic
    !(pb instanceof ExternalFunctionProgramBlock)) {
        FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
    } else {
        ArrayList<Instruction> tmp = pb.getInstructions();
        for (Instruction inst : tmp) {
            if (// CP
            inst instanceof CPInstruction) {
                // obtain stats from createvar, cpvar, rmvar, rand
                maintainCPInstVariableStatistics((CPInstruction) inst, stats);
                // extract statistics (instruction-specific)
                Object[] o = extractCPInstStatistics(inst, stats);
                VarStats[] vs = (VarStats[]) o[0];
                String[] attr = (String[]) o[1];
                // if(LOG.isDebugEnabled())
                // LOG.debug(inst);
                // call time estimation for inst
                ret += getCPInstTimeEstimate(inst, vs, attr);
                if (// functions
                inst instanceof FunctionCallCPInstruction) {
                    FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
                    String fkey = DMLProgram.constructFunctionKey(finst.getNamespace(), finst.getFunctionName());
                    // awareness of recursive functions, missing program
                    if (!memoFunc.contains(fkey) && pb.getProgram() != null) {
                        if (LOG.isDebugEnabled())
                            LOG.debug("Begin Function " + fkey);
                        memoFunc.add(fkey);
                        Program prog = pb.getProgram();
                        FunctionProgramBlock fpb = prog.getFunctionProgramBlock(finst.getNamespace(), finst.getFunctionName());
                        ret += rGetTimeEstimate(fpb, stats, memoFunc, recursive);
                        memoFunc.remove(fkey);
                        if (LOG.isDebugEnabled())
                            LOG.debug("End Function " + fkey);
                    }
                }
            } else if (// MR
            inst instanceof MRJobInstruction) {
                // obtain stats for job
                maintainMRJobInstVariableStatistics(inst, stats);
                // extract input statistics
                Object[] o = extractMRJobInstStatistics(inst, stats);
                VarStats[] vs = (VarStats[]) o[0];
                if (LOG.isDebugEnabled())
                    LOG.debug("Begin MRJob type=" + ((MRJobInstruction) inst).getJobType());
                // call time estimation for complex MR inst
                ret += getMRJobInstTimeEstimate(inst, vs, null);
                if (LOG.isDebugEnabled())
                    LOG.debug("End MRJob");
                // cleanup stats for job
                cleanupMRJobVariableStatistics(inst, stats);
            }
        }
    }
    return ret;
}

Example 13 with WhileProgramBlock

use of org.apache.sysml.runtime.controlprogram.WhileProgramBlock in project incubator-systemml by apache.

the class Recompiler method rRecompileProgramBlock2Forced.

private static void rRecompileProgramBlock2Forced(ProgramBlock pb, long tid, HashSet<String> fnStack, ExecType et) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock pbTmp = (WhileProgramBlock) pb;
        WhileStatementBlock sbTmp = (WhileStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
            pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock pbTmp = (IfProgramBlock) pb;
        IfStatementBlock sbTmp = (IfStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
            pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocksIfBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
        for (ProgramBlock pb2 : pbTmp.getChildBlocksElseBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock pbTmp = (ForProgramBlock) pb;
        ForStatementBlock sbTmp = (ForStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && sbTmp.getFromHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getFromInstructions(), true, true)))
            pbTmp.setFromInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getFromHops(), tid, et));
        if (sbTmp != null && sbTmp.getToHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getToInstructions(), true, true)))
            pbTmp.setToInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getToHops(), tid, et));
        if (sbTmp != null && sbTmp.getIncrementHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getIncrementInstructions(), true, true)))
            pbTmp.setIncrementInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getIncrementHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else {
        StatementBlock sb = pb.getStatementBlock();
        // would be invalid with permutation matrix mult across multiple dags)
        if (sb != null) {
            ArrayList<Instruction> tmp = pb.getInstructions();
            tmp = Recompiler.recompileHopsDag2Forced(sb, sb.getHops(), tid, et);
            pb.setInstructions(tmp);
        }
        // recompile functions
        if (OptTreeConverter.containsFunctionCallInstruction(pb)) {
            ArrayList<Instruction> tmp = pb.getInstructions();
            for (Instruction inst : tmp) if (inst instanceof FunctionCallCPInstruction) {
                FunctionCallCPInstruction func = (FunctionCallCPInstruction) inst;
                String fname = func.getFunctionName();
                String fnamespace = func.getNamespace();
                String fKey = DMLProgram.constructFunctionKey(fnamespace, fname);
                if (// memoization for multiple calls, recursion
                !fnStack.contains(fKey)) {
                    fnStack.add(fKey);
                    FunctionProgramBlock fpb = pb.getProgram().getFunctionProgramBlock(fnamespace, fname);
                    // recompile chains of functions
                    rRecompileProgramBlock2Forced(fpb, tid, fnStack, et);
                }
            }
        }
    }
}

Example 14 with WhileProgramBlock

use of org.apache.sysml.runtime.controlprogram.WhileProgramBlock in project incubator-systemml by apache.

the class Recompiler method rRecompileProgramBlock.

// ////////////////////////////
// private helper functions //
// ////////////////////////////
private static void rRecompileProgramBlock(ProgramBlock pb, LocalVariableMap vars, RecompileStatus status, long tid, ResetType resetRecompile) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) wpb.getStatementBlock();
        // recompile predicate
        recompileWhilePredicate(wpb, wsb, vars, status, tid, resetRecompile);
        // remove updated scalars because in loop
        removeUpdatedScalars(vars, wsb);
        // copy vars for later compare
        LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
        RecompileStatus oldStatus = (RecompileStatus) status.clone();
        for (ProgramBlock pb2 : wpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        if (reconcileUpdatedCallVarsLoops(oldVars, vars, wsb) | reconcileUpdatedCallVarsLoops(oldStatus, status, wsb)) {
            // second pass with unknowns if required
            recompileWhilePredicate(wpb, wsb, vars, status, tid, resetRecompile);
            for (ProgramBlock pb2 : wpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        }
        removeUpdatedScalars(vars, wsb);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) ipb.getStatementBlock();
        // recompile predicate
        recompileIfPredicate(ipb, isb, vars, status, tid, resetRecompile);
        // copy vars for later compare
        LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
        LocalVariableMap varsElse = (LocalVariableMap) vars.clone();
        RecompileStatus oldStatus = (RecompileStatus) status.clone();
        RecompileStatus statusElse = (RecompileStatus) status.clone();
        for (ProgramBlock pb2 : ipb.getChildBlocksIfBody()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        for (ProgramBlock pb2 : ipb.getChildBlocksElseBody()) rRecompileProgramBlock(pb2, varsElse, statusElse, tid, resetRecompile);
        reconcileUpdatedCallVarsIf(oldVars, vars, varsElse, isb);
        reconcileUpdatedCallVarsIf(oldStatus, status, statusElse, isb);
        removeUpdatedScalars(vars, ipb.getStatementBlock());
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) fpb.getStatementBlock();
        // recompile predicates
        recompileForPredicates(fpb, fsb, vars, status, tid, resetRecompile);
        // remove updated scalars because in loop
        removeUpdatedScalars(vars, fpb.getStatementBlock());
        // copy vars for later compare
        LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
        RecompileStatus oldStatus = (RecompileStatus) status.clone();
        for (ProgramBlock pb2 : fpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        if (reconcileUpdatedCallVarsLoops(oldVars, vars, fsb) | reconcileUpdatedCallVarsLoops(oldStatus, status, fsb)) {
            // second pass with unknowns if required
            recompileForPredicates(fpb, fsb, vars, status, tid, resetRecompile);
            for (ProgramBlock pb2 : fpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        }
        removeUpdatedScalars(vars, fpb.getStatementBlock());
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
    // do nothing
    } else {
        StatementBlock sb = pb.getStatementBlock();
        ArrayList<Instruction> tmp = pb.getInstructions();
        if (sb == null)
            return;
        // recompile all for stats propagation and recompile flags
        tmp = Recompiler.recompileHopsDag(sb, sb.getHops(), vars, status, true, false, tid);
        pb.setInstructions(tmp);
        // propagate stats across hops (should be executed on clone of vars)
        Recompiler.extractDAGOutputStatistics(sb.getHops(), vars);
        // reset recompilation flags (w/ special handling functions)
        if (ParForProgramBlock.RESET_RECOMPILATION_FLAGs && !containsRootFunctionOp(sb.getHops()) && resetRecompile.isReset()) {
            Hop.resetRecompilationFlag(sb.getHops(), ExecType.CP, resetRecompile);
            sb.updateRecompilationFlag();
        }
    }
}

Example 15 with WhileProgramBlock

use of org.apache.sysml.runtime.controlprogram.WhileProgramBlock in project incubator-systemml by apache.

the class Explain method explainProgramBlock.

// ////////////
// internal explain RUNTIME
private static String explainProgramBlock(ProgramBlock pb, int level) {
    StringBuilder sb = new StringBuilder();
    String offset = createOffset(level);
    if (pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        StatementBlock wsb = pb.getStatementBlock();
        sb.append(offset);
        if (wsb != null && !wsb.getUpdateInPlaceVars().isEmpty())
            sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ") [in-place=" + wsb.getUpdateInPlaceVars().toString() + "]\n");
        else
            sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ")\n");
        sb.append(explainInstructions(wpb.getPredicate(), level + 1));
        for (ProgramBlock pbc : wpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        sb.append(offset);
        sb.append("IF (lines " + ipb.getBeginLine() + "-" + ipb.getEndLine() + ")\n");
        sb.append(explainInstructions(ipb.getPredicate(), level + 1));
        for (ProgramBlock pbc : ipb.getChildBlocksIfBody()) sb.append(explainProgramBlock(pbc, level + 1));
        if (!ipb.getChildBlocksElseBody().isEmpty()) {
            sb.append(offset);
            sb.append("ELSE\n");
            for (ProgramBlock pbc : ipb.getChildBlocksElseBody()) sb.append(explainProgramBlock(pbc, level + 1));
        }
    } else if (// incl parfor
    pb instanceof ForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        StatementBlock fsb = pb.getStatementBlock();
        sb.append(offset);
        if (pb instanceof ParForProgramBlock)
            sb.append("PARFOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
        else {
            if (fsb != null && !fsb.getUpdateInPlaceVars().isEmpty())
                sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ") [in-place=" + fsb.getUpdateInPlaceVars().toString() + "]\n");
            else
                sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
        }
        sb.append(explainInstructions(fpb.getFromInstructions(), level + 1));
        sb.append(explainInstructions(fpb.getToInstructions(), level + 1));
        sb.append(explainInstructions(fpb.getIncrementInstructions(), level + 1));
        for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else {
        sb.append(offset);
        if (pb.getStatementBlock() != null)
            sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") [recompile=" + pb.getStatementBlock().requiresRecompilation() + "]\n");
        else
            sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") \n");
        sb.append(explainInstructions(pb.getInstructions(), level + 1));
    }
    return sb.toString();
}