Examples with ParForProgramBlock org.apache.sysml.runtime.controlprogram.ParForProgramBlock used on opensource projects

Example 26 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class ProgramConverter method rParseParForProgramBlock.

private static ParForProgramBlock rParseParForProgramBlock(String in, Program prog, int id) {
    String lin = in.substring(PARFOR_PB_PARFOR.length(), in.length() - PARFOR_PB_END.length());
    HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
    // inputs
    String iterVar = st.nextToken();
    ArrayList<ResultVar> resultVars = parseResultVariables(st.nextToken());
    HashMap<String, String> params = parseStringHashMap(st.nextToken());
    // instructions
    ArrayList<Instruction> from = parseInstructions(st.nextToken(), 0);
    ArrayList<Instruction> to = parseInstructions(st.nextToken(), 0);
    ArrayList<Instruction> incr = parseInstructions(st.nextToken(), 0);
    // exit instructions
    ArrayList<Instruction> exit = parseInstructions(st.nextToken(), 0);
    // program blocks //reset id to preinit state, replaced during exec
    ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, 0);
    ParForProgramBlock pfpb = new ParForProgramBlock(id, prog, iterVar, params, resultVars);
    // already done in top-level parfor
    pfpb.disableOptimization();
    pfpb.setFromInstructions(from);
    pfpb.setToInstructions(to);
    pfpb.setIncrementInstructions(incr);
    pfpb.setExitInstructions(exit);
    pfpb.setChildBlocks(pbs);
    return pfpb;
}

Example 27 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class Recompiler method rRecompileProgramBlock2Forced.

private static void rRecompileProgramBlock2Forced(ProgramBlock pb, long tid, HashSet<String> fnStack, ExecType et) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock pbTmp = (WhileProgramBlock) pb;
        WhileStatementBlock sbTmp = (WhileStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
            pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock pbTmp = (IfProgramBlock) pb;
        IfStatementBlock sbTmp = (IfStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
            pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocksIfBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
        for (ProgramBlock pb2 : pbTmp.getChildBlocksElseBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock pbTmp = (ForProgramBlock) pb;
        ForStatementBlock sbTmp = (ForStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && sbTmp.getFromHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getFromInstructions(), true, true)))
            pbTmp.setFromInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getFromHops(), tid, et));
        if (sbTmp != null && sbTmp.getToHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getToInstructions(), true, true)))
            pbTmp.setToInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getToHops(), tid, et));
        if (sbTmp != null && sbTmp.getIncrementHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getIncrementInstructions(), true, true)))
            pbTmp.setIncrementInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getIncrementHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else {
        StatementBlock sb = pb.getStatementBlock();
        // would be invalid with permutation matrix mult across multiple dags)
        if (sb != null) {
            ArrayList<Instruction> tmp = pb.getInstructions();
            tmp = Recompiler.recompileHopsDag2Forced(sb, sb.getHops(), tid, et);
            pb.setInstructions(tmp);
        }
        // recompile functions
        if (OptTreeConverter.containsFunctionCallInstruction(pb)) {
            ArrayList<Instruction> tmp = pb.getInstructions();
            for (Instruction inst : tmp) if (inst instanceof FunctionCallCPInstruction) {
                FunctionCallCPInstruction func = (FunctionCallCPInstruction) inst;
                String fname = func.getFunctionName();
                String fnamespace = func.getNamespace();
                String fKey = DMLProgram.constructFunctionKey(fnamespace, fname);
                if (// memoization for multiple calls, recursion
                !fnStack.contains(fKey)) {
                    fnStack.add(fKey);
                    FunctionProgramBlock fpb = pb.getProgram().getFunctionProgramBlock(fnamespace, fname);
                    // recompile chains of functions
                    rRecompileProgramBlock2Forced(fpb, tid, fnStack, et);
                }
            }
        }
    }
}

Example 28 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class Recompiler method rRecompileProgramBlock.

// ////////////////////////////
// private helper functions //
// ////////////////////////////
private static void rRecompileProgramBlock(ProgramBlock pb, LocalVariableMap vars, RecompileStatus status, long tid, ResetType resetRecompile) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) wpb.getStatementBlock();
        // recompile predicate
        recompileWhilePredicate(wpb, wsb, vars, status, tid, resetRecompile);
        // remove updated scalars because in loop
        removeUpdatedScalars(vars, wsb);
        // copy vars for later compare
        LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
        RecompileStatus oldStatus = (RecompileStatus) status.clone();
        for (ProgramBlock pb2 : wpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        if (reconcileUpdatedCallVarsLoops(oldVars, vars, wsb) | reconcileUpdatedCallVarsLoops(oldStatus, status, wsb)) {
            // second pass with unknowns if required
            recompileWhilePredicate(wpb, wsb, vars, status, tid, resetRecompile);
            for (ProgramBlock pb2 : wpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        }
        removeUpdatedScalars(vars, wsb);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) ipb.getStatementBlock();
        // recompile predicate
        recompileIfPredicate(ipb, isb, vars, status, tid, resetRecompile);
        // copy vars for later compare
        LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
        LocalVariableMap varsElse = (LocalVariableMap) vars.clone();
        RecompileStatus oldStatus = (RecompileStatus) status.clone();
        RecompileStatus statusElse = (RecompileStatus) status.clone();
        for (ProgramBlock pb2 : ipb.getChildBlocksIfBody()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        for (ProgramBlock pb2 : ipb.getChildBlocksElseBody()) rRecompileProgramBlock(pb2, varsElse, statusElse, tid, resetRecompile);
        reconcileUpdatedCallVarsIf(oldVars, vars, varsElse, isb);
        reconcileUpdatedCallVarsIf(oldStatus, status, statusElse, isb);
        removeUpdatedScalars(vars, ipb.getStatementBlock());
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) fpb.getStatementBlock();
        // recompile predicates
        recompileForPredicates(fpb, fsb, vars, status, tid, resetRecompile);
        // remove updated scalars because in loop
        removeUpdatedScalars(vars, fpb.getStatementBlock());
        // copy vars for later compare
        LocalVariableMap oldVars = (LocalVariableMap) vars.clone();
        RecompileStatus oldStatus = (RecompileStatus) status.clone();
        for (ProgramBlock pb2 : fpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        if (reconcileUpdatedCallVarsLoops(oldVars, vars, fsb) | reconcileUpdatedCallVarsLoops(oldStatus, status, fsb)) {
            // second pass with unknowns if required
            recompileForPredicates(fpb, fsb, vars, status, tid, resetRecompile);
            for (ProgramBlock pb2 : fpb.getChildBlocks()) rRecompileProgramBlock(pb2, vars, status, tid, resetRecompile);
        }
        removeUpdatedScalars(vars, fpb.getStatementBlock());
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
    // do nothing
    } else {
        StatementBlock sb = pb.getStatementBlock();
        ArrayList<Instruction> tmp = pb.getInstructions();
        if (sb == null)
            return;
        // recompile all for stats propagation and recompile flags
        tmp = Recompiler.recompileHopsDag(sb, sb.getHops(), vars, status, true, false, tid);
        pb.setInstructions(tmp);
        // propagate stats across hops (should be executed on clone of vars)
        Recompiler.extractDAGOutputStatistics(sb.getHops(), vars);
        // reset recompilation flags (w/ special handling functions)
        if (ParForProgramBlock.RESET_RECOMPILATION_FLAGs && !containsRootFunctionOp(sb.getHops()) && resetRecompile.isReset()) {
            Hop.resetRecompilationFlag(sb.getHops(), ExecType.CP, resetRecompile);
            sb.updateRecompilationFlag();
        }
    }
}

Example 29 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class Explain method explainProgramBlock.

// ////////////
// internal explain RUNTIME
private static String explainProgramBlock(ProgramBlock pb, int level) {
    StringBuilder sb = new StringBuilder();
    String offset = createOffset(level);
    if (pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        StatementBlock wsb = pb.getStatementBlock();
        sb.append(offset);
        if (wsb != null && !wsb.getUpdateInPlaceVars().isEmpty())
            sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ") [in-place=" + wsb.getUpdateInPlaceVars().toString() + "]\n");
        else
            sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ")\n");
        sb.append(explainInstructions(wpb.getPredicate(), level + 1));
        for (ProgramBlock pbc : wpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        sb.append(offset);
        sb.append("IF (lines " + ipb.getBeginLine() + "-" + ipb.getEndLine() + ")\n");
        sb.append(explainInstructions(ipb.getPredicate(), level + 1));
        for (ProgramBlock pbc : ipb.getChildBlocksIfBody()) sb.append(explainProgramBlock(pbc, level + 1));
        if (!ipb.getChildBlocksElseBody().isEmpty()) {
            sb.append(offset);
            sb.append("ELSE\n");
            for (ProgramBlock pbc : ipb.getChildBlocksElseBody()) sb.append(explainProgramBlock(pbc, level + 1));
        }
    } else if (// incl parfor
    pb instanceof ForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        StatementBlock fsb = pb.getStatementBlock();
        sb.append(offset);
        if (pb instanceof ParForProgramBlock)
            sb.append("PARFOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
        else {
            if (fsb != null && !fsb.getUpdateInPlaceVars().isEmpty())
                sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ") [in-place=" + fsb.getUpdateInPlaceVars().toString() + "]\n");
            else
                sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
        }
        sb.append(explainInstructions(fpb.getFromInstructions(), level + 1));
        sb.append(explainInstructions(fpb.getToInstructions(), level + 1));
        sb.append(explainInstructions(fpb.getIncrementInstructions(), level + 1));
        for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else {
        sb.append(offset);
        if (pb.getStatementBlock() != null)
            sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") [recompile=" + pb.getStatementBlock().requiresRecompilation() + "]\n");
        else
            sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") \n");
        sb.append(explainInstructions(pb.getInstructions(), level + 1));
    }
    return sb.toString();
}

Example 30 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class DMLTranslator method createRuntimeProgramBlock.

public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) {
    Dag<Lop> dag = null;
    Dag<Lop> pred_dag = null;
    ArrayList<Instruction> instruct;
    ArrayList<Instruction> pred_instruct = null;
    ProgramBlock retPB = null;
    // process While Statement - add runtime program blocks to program
    if (sb instanceof WhileStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<>();
        ((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create while program block
        WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
        // // process the body of the while statement block ////
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        for (StatementBlock sblock : wstmt.getBody()) {
            // process the body
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        retPB = rtpb;
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // process If Statement - add runtime program blocks to program
    if (sb instanceof IfStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<>();
        ((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create if program block
        IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
        // process the body of the if statement block
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        // process the if body
        for (StatementBlock sblock : istmt.getIfBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockIfBody(childBlock);
        }
        // process the else body
        for (StatementBlock sblock : istmt.getElseBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockElseBody(childBlock);
        }
        retPB = rtpb;
        // post processing for generating missing instructions
        // retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // NOTE: applies to ForStatementBlock and ParForStatementBlock
    if (sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        // create DAGs for loop predicates
        Dag<Lop> fromDag = new Dag<>();
        Dag<Lop> toDag = new Dag<>();
        Dag<Lop> incrementDag = new Dag<>();
        if (fsb.getFromHops() != null)
            fsb.getFromLops().addToDag(fromDag);
        if (fsb.getToHops() != null)
            fsb.getToLops().addToDag(toDag);
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementLops().addToDag(incrementDag);
        // create instructions for loop predicates
        ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
        ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
        ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
        // create for program block
        ForProgramBlock rtpb = null;
        IterablePredicate iterPred = fsb.getIterPredicate();
        if (sb instanceof ParForStatementBlock && ConfigurationManager.isParallelParFor()) {
            rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), iterPred.getParForParams(), ((ParForStatementBlock) sb).getResultVariables());
            ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
            // used for optimization and creating unscoped variables
            pfrtpb.setStatementBlock((ParForStatementBlock) sb);
        } else {
            // ForStatementBlock
            rtpb = new ForProgramBlock(prog, iterPred.getIterVar().getName());
        }
        rtpb.setFromInstructions(fromInstructions);
        rtpb.setToInstructions(toInstructions);
        rtpb.setIncrementInstructions(incrementInstructions);
        // process the body of the for statement block
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        for (StatementBlock sblock : fs.getBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        retPB = rtpb;
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // process function statement block - add runtime program blocks to program
    if (sb instanceof FunctionStatementBlock) {
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        FunctionProgramBlock rtpb = null;
        if (fstmt instanceof ExternalFunctionStatement) {
            // create external function program block
            String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
            boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
            StringBuilder buff = new StringBuilder();
            buff.append(config.getTextValue(DMLConfig.SCRATCH_SPACE));
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(Lop.PROCESS_PREFIX);
            buff.append(DMLScript.getUUID());
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
            buff.append(Lop.FILE_SEPARATOR);
            buff.append("PackageSupport");
            buff.append(Lop.FILE_SEPARATOR);
            String basedir = buff.toString();
            if (isCP) {
                rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            } else {
                rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            }
            if (!fstmt.getBody().isEmpty()) {
                LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
                throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
            }
        } else {
            // create function program block
            rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
            // process the function statement body
            for (StatementBlock sblock : fstmt.getBody()) {
                // process the body
                ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
                rtpb.addProgramBlock(childBlock);
            }
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
            LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
            throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
        }
        retPB = rtpb;
        // add location information
        retPB.setParseInfo(sb);
    } else {
        // handle general case
        ProgramBlock rtpb = new ProgramBlock(prog);
        // DAGs for Lops
        dag = new Dag<>();
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (sb.getLops() != null && !sb.getLops().isEmpty()) {
            for (Lop l : sb.getLops()) {
                l.addToDag(dag);
            }
            // Instructions for Lops DAGs
            instruct = dag.getJobs(sb, config);
            rtpb.addInstructions(instruct);
        }
        retPB = rtpb;
        // post processing for generating missing instructions
        // retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    }
    return retPB;
}