Examples with Dag org.apache.sysml.lops.compile.Dag used on opensource projects

Example 1 with Dag

use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.

the class Recompiler method recompileHopsDag.

/**
	 * A) Recompile basic program block hop DAG.
	 * 	
	 * We support to basic types inplace or via deep copy. Deep copy is the default and is required 
	 * in order to apply non-reversible rewrites. In-place is required in order to modify the existing
	 * hops (e.g., for parfor pre-recompilation).
	 * 
	 * @param sb statement block
	 * @param hops high-level operators
	 * @param vars local variable map
	 * @param status the recompile status
	 * @param inplace true if in place
	 * @param litreplace true if literal replacement
	 * @param tid thread id
	 * @return list of instructions
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 * @throws IOException if IOException occurs
	 */
public static ArrayList<Instruction> recompileHopsDag(StatementBlock sb, ArrayList<Hop> hops, LocalVariableMap vars, RecompileStatus status, boolean inplace, boolean litreplace, long tid) throws DMLRuntimeException, HopsException, LopsException, IOException {
    ArrayList<Instruction> newInst = null;
    //however, we create deep copies for most dags to allow for concurrent recompile
    synchronized (hops) {
        LOG.debug("\n**************** Optimizer (Recompile) *************\nMemory Budget = " + OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
        // prepare hops dag for recompile
        if (!inplace) {
            // deep copy hop dag (for non-reversable rewrites)
            hops = deepCopyHopsDag(hops);
        } else {
            // clear existing lops
            Hop.resetVisitStatus(hops);
            for (Hop hopRoot : hops) rClearLops(hopRoot);
        }
        // replace scalar reads with literals 
        if (!inplace && litreplace) {
            Hop.resetVisitStatus(hops);
            for (Hop hopRoot : hops) rReplaceLiterals(hopRoot, vars, false);
        }
        // refresh matrix characteristics (update stats)			
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) rUpdateStatistics(hopRoot, vars);
        // dynamic hop rewrites
        if (!inplace) {
            _rewriter.get().rewriteHopDAGs(hops, null);
            //update stats after rewrites
            Hop.resetVisitStatus(hops);
            for (Hop hopRoot : hops) rUpdateStatistics(hopRoot, vars);
        }
        // refresh memory estimates (based on updated stats,
        // before: init memo table with propagated worst-case estimates,
        // after: extract worst-case estimates from memo table 
        Hop.resetVisitStatus(hops);
        MemoTable memo = new MemoTable();
        memo.init(hops, status);
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) hopRoot.refreshMemEstimates(memo);
        memo.extract(hops, status);
        // codegen if enabled
        if (ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.CODEGEN) && SpoofCompiler.RECOMPILE_CODEGEN) {
            Hop.resetVisitStatus(hops);
            hops = SpoofCompiler.optimize(hops, true);
        }
        // construct lops			
        Dag<Lop> dag = new Dag<Lop>();
        for (Hop hopRoot : hops) {
            Lop lops = hopRoot.constructLops();
            lops.addToDag(dag);
        }
        // generate runtime instructions (incl piggybacking)
        newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
    }
    // replace thread ids in new instructions
    if (//only in parfor context
    tid != 0)
        newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
    // explain recompiled hops / instructions
    if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS) {
        LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines " + sb.getBeginLine() + "-" + sb.getEndLine() + "):\n" + Explain.explainHops(hops, 1));
    }
    if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME) {
        LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines " + sb.getBeginLine() + "-" + sb.getEndLine() + "):\n" + Explain.explain(newInst, 1));
    }
    return newInst;
}

Example 2 with Dag

use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.

the class Recompiler method recompileHopsDag.

/**
	 * B) Recompile predicate hop DAG (single root): 
	 * 
	 * Note: This overloaded method is required for predicate instructions because
	 * they have only a single hops DAG and we need to synchronize on the original 
	 * (shared) hops object. Hence, we cannot create any wrapper arraylist for each
	 * recompilation - this would result in race conditions for concurrent recompilation 
	 * in a parfor body. 	
	 * 
	 * Note: no statementblock passed because for predicate dags we dont have separate live variable analysis information.
	 * 
	 * @param hops high-level operator
	 * @param vars local variable map
	 * @param status recompile status
	 * @param inplace true if in place
	 * @param litreplace true if literal replacement
	 * @param tid thread id
	 * @return list of instructions
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 * @throws IOException if IOException occurs
	 */
public static ArrayList<Instruction> recompileHopsDag(Hop hops, LocalVariableMap vars, RecompileStatus status, boolean inplace, boolean litreplace, long tid) throws DMLRuntimeException, HopsException, LopsException, IOException {
    ArrayList<Instruction> newInst = null;
    //need for synchronization as we do temp changes in shared hops/lops
    synchronized (hops) {
        LOG.debug("\n**************** Optimizer (Recompile) *************\nMemory Budget = " + OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
        // prepare hops dag for recompile
        if (!inplace) {
            // deep copy hop dag (for non-reversable rewrites)
            //(this also clears existing lops in the created dag) 
            hops = deepCopyHopsDag(hops);
        } else {
            // clear existing lops
            hops.resetVisitStatus();
            rClearLops(hops);
        }
        // replace scalar reads with literals 
        if (!inplace && litreplace) {
            hops.resetVisitStatus();
            rReplaceLiterals(hops, vars, false);
        }
        // refresh matrix characteristics (update stats)			
        hops.resetVisitStatus();
        rUpdateStatistics(hops, vars);
        // dynamic hop rewrites
        if (!inplace) {
            _rewriter.get().rewriteHopDAG(hops, null);
            //update stats after rewrites
            hops.resetVisitStatus();
            rUpdateStatistics(hops, vars);
        }
        // refresh memory estimates (based on updated stats)
        MemoTable memo = new MemoTable();
        hops.resetVisitStatus();
        memo.init(hops, status);
        hops.resetVisitStatus();
        hops.refreshMemEstimates(memo);
        // codegen if enabled
        if (ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.CODEGEN) && SpoofCompiler.RECOMPILE_CODEGEN) {
            hops.resetVisitStatus();
            hops = SpoofCompiler.optimize(hops, false);
        }
        // construct lops			
        Dag<Lop> dag = new Dag<Lop>();
        Lop lops = hops.constructLops();
        lops.addToDag(dag);
        // generate runtime instructions (incl piggybacking)
        newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
    }
    // replace thread ids in new instructions
    if (//only in parfor context
    tid != 0)
        newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
    // explain recompiled instructions
    if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS)
        LOG.info("EXPLAIN RECOMPILE \nPRED (line " + hops.getBeginLine() + "):\n" + Explain.explain(hops, 1));
    if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME)
        LOG.info("EXPLAIN RECOMPILE \nPRED (line " + hops.getBeginLine() + "):\n" + Explain.explain(newInst, 1));
    return newInst;
}

Example 3 with Dag

use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.

the class DMLProgram method createRuntimeProgramBlock.

public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) throws IOException, LopsException, DMLRuntimeException {
    Dag<Lop> dag = null;
    Dag<Lop> pred_dag = null;
    ArrayList<Instruction> instruct;
    ArrayList<Instruction> pred_instruct = null;
    ProgramBlock retPB = null;
    // process While Statement - add runtime program blocks to program
    if (sb instanceof WhileStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<Lop>();
        ((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<Instruction>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create while program block
        WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
        if (rtpb.getPredicateResultVar() == null) {
            // e.g case : WHILE(continue)
            if (((WhileStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data) {
                String resultVar = ((WhileStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
                rtpb.setPredicateResultVar(resultVar);
            } else {
                LOG.error(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
                throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
            }
        }
        //// process the body of the while statement block ////
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        if (wsb.getNumStatements() > 1) {
            LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
            throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
        }
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        for (StatementBlock sblock : wstmt.getBody()) {
            // process the body
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (wsb.getLops() != null && !wsb.getLops().isEmpty()) {
            LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
            throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
        }
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else // process If Statement - add runtime program blocks to program
    if (sb instanceof IfStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<Lop>();
        ((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<Instruction>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create if program block
        IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
        if (rtpb.getPredicateResultVar() == null) {
            // e.g case : If(continue)
            if (((IfStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data) {
                String resultVar = ((IfStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
                rtpb.setPredicateResultVar(resultVar);
            } else {
                LOG.error(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
                throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
            }
        }
        // process the body of the if statement block
        IfStatementBlock isb = (IfStatementBlock) sb;
        if (isb.getNumStatements() > 1) {
            LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
            throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
        }
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        // process the if body
        for (StatementBlock sblock : istmt.getIfBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockIfBody(childBlock);
        }
        // process the else body
        for (StatementBlock sblock : istmt.getElseBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockElseBody(childBlock);
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (isb.getLops() != null && !isb.getLops().isEmpty()) {
            LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
            throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
        }
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else // NOTE: applies to ForStatementBlock and ParForStatementBlock
    if (sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        // create DAGs for loop predicates 
        Dag<Lop> fromDag = new Dag<Lop>();
        Dag<Lop> toDag = new Dag<Lop>();
        Dag<Lop> incrementDag = new Dag<Lop>();
        if (fsb.getFromHops() != null)
            fsb.getFromLops().addToDag(fromDag);
        if (fsb.getToHops() != null)
            fsb.getToLops().addToDag(toDag);
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementLops().addToDag(incrementDag);
        // create instructions for loop predicates			
        ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
        ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
        ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
        // create for program block
        String sbName = null;
        ForProgramBlock rtpb = null;
        IterablePredicate iterPred = fsb.getIterPredicate();
        String[] iterPredData = IterablePredicate.createIterablePredicateVariables(iterPred.getIterVar().getName(), fsb.getFromLops(), fsb.getToLops(), fsb.getIncrementLops());
        if (sb instanceof ParForStatementBlock) {
            sbName = "ParForStatementBlock";
            rtpb = new ParForProgramBlock(prog, iterPredData, iterPred.getParForParams());
            ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
            pfrtpb.setResultVariables(((ParForStatementBlock) sb).getResultVariables());
            //used for optimization and creating unscoped variables
            pfrtpb.setStatementBlock((ParForStatementBlock) sb);
        } else //ForStatementBlock
        {
            sbName = "ForStatementBlock";
            rtpb = new ForProgramBlock(prog, iterPredData);
        }
        rtpb.setFromInstructions(fromInstructions);
        rtpb.setToInstructions(toInstructions);
        rtpb.setIncrementInstructions(incrementInstructions);
        rtpb.setIterablePredicateVars(iterPredData);
        // process the body of the for statement block
        if (fsb.getNumStatements() > 1) {
            LOG.error(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement");
            throw new LopsException(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement");
        }
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        for (StatementBlock sblock : fs.getBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
            LOG.error(fsb.printBlockErrorLocation() + sbName + " should have no Lops");
            throw new LopsException(fsb.printBlockErrorLocation() + sbName + " should have no Lops");
        }
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else // process function statement block - add runtime program blocks to program
    if (sb instanceof FunctionStatementBlock) {
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        if (fsb.getNumStatements() > 1) {
            LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
            throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
        }
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        FunctionProgramBlock rtpb = null;
        if (fstmt instanceof ExternalFunctionStatement) {
            // create external function program block
            String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
            boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
            String scratchSpaceLoc = null;
            try {
                scratchSpaceLoc = config.getTextValue(DMLConfig.SCRATCH_SPACE);
            } catch (Exception e) {
                LOG.error(fsb.printBlockErrorLocation() + "could not retrieve parameter " + DMLConfig.SCRATCH_SPACE + " from DMLConfig");
            }
            StringBuilder buff = new StringBuilder();
            buff.append(scratchSpaceLoc);
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(Lop.PROCESS_PREFIX);
            buff.append(DMLScript.getUUID());
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
            buff.append(Lop.FILE_SEPARATOR);
            buff.append("PackageSupport");
            buff.append(Lop.FILE_SEPARATOR);
            String basedir = buff.toString();
            if (isCP) {
                rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            } else {
                rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            }
            if (!fstmt.getBody().isEmpty()) {
                LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
                throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
            }
        } else {
            // create function program block
            rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
            // process the function statement body
            for (StatementBlock sblock : fstmt.getBody()) {
                // process the body
                ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
                rtpb.addProgramBlock(childBlock);
            }
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
            LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
            throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
        }
        retPB = rtpb;
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else {
        // handle general case
        ProgramBlock rtpb = new ProgramBlock(prog);
        // DAGs for Lops
        dag = new Dag<Lop>();
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (sb.getLops() != null && !sb.getLops().isEmpty()) {
            for (Lop l : sb.getLops()) {
                l.addToDag(dag);
            }
            // Instructions for Lobs DAGs
            instruct = dag.getJobs(sb, config);
            rtpb.addInstructions(instruct);
        }
        /*// TODO: check with Doug
			// add instruction for a function call
			if (sb.getFunctionCallInst() != null){
				rtpb.addInstruction(sb.getFunctionCallInst());
			}*/
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    }
    return retPB;
}

Example 4 with Dag

use of org.apache.sysml.lops.compile.Dag in project systemml by apache.

the class Recompiler method recompile.

/**
 * Core internal primitive for the dynamic recompilation of any DAGs/predicate,
 * including all variants with slightly different configurations.
 *
 * @param sb statement block of DAG, null for predicates
 * @param hops list of DAG root nodes
 * @param vars symbol table
 * @param status recompilation status
 * @param inplace modify DAG in place, otherwise deep copy
 * @param replaceLit replace literals (only applicable on deep copy)
 * @param updateStats update statistics, rewrites, and memory estimates
 * @param forceEt force a given execution type, null for reset
 * @param pred recompile for predicate DAG
 * @param et given execution type
 * @param tid thread id, 0 for main or before worker creation
 * @return modified list of instructions
 */
private static ArrayList<Instruction> recompile(StatementBlock sb, ArrayList<Hop> hops, LocalVariableMap vars, RecompileStatus status, boolean inplace, boolean replaceLit, boolean updateStats, boolean forceEt, boolean pred, ExecType et, long tid) {
    // prepare hops dag for recompile
    if (!inplace) {
        // deep copy hop dag (for non-reversable rewrites)
        hops = deepCopyHopsDag(hops);
    } else {
        // clear existing lops
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) rClearLops(hopRoot);
    }
    // replace scalar reads with literals
    if (!inplace && replaceLit) {
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) rReplaceLiterals(hopRoot, vars, false);
    }
    // force exec type (et=null for reset)
    if (forceEt) {
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) rSetExecType(hopRoot, et);
        Hop.resetVisitStatus(hops);
    }
    // update statistics, rewrites, and mem estimates
    if (updateStats) {
        // refresh matrix characteristics (update stats)
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) rUpdateStatistics(hopRoot, vars);
        // dynamic hop rewrites
        if (!inplace) {
            _rewriter.get().rewriteHopDAG(hops, null);
            // update stats after rewrites
            Hop.resetVisitStatus(hops);
            for (Hop hopRoot : hops) rUpdateStatistics(hopRoot, vars);
        }
        // refresh memory estimates (based on updated stats,
        // before: init memo table with propagated worst-case estimates,
        // after: extract worst-case estimates from memo table
        Hop.resetVisitStatus(hops);
        MemoTable memo = new MemoTable();
        memo.init(hops, status);
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) hopRoot.refreshMemEstimates(memo);
        memo.extract(hops, status);
    }
    // codegen if enabled
    if (ConfigurationManager.isCodegenEnabled() && // not on reset
    !(forceEt && et == null) && SpoofCompiler.RECOMPILE_CODEGEN) {
        // create deep copy for in-place
        if (inplace)
            hops = deepCopyHopsDag(hops);
        Hop.resetVisitStatus(hops);
        hops = SpoofCompiler.optimize(hops, (status == null || !status.isInitialCodegen()));
    }
    // construct lops
    Dag<Lop> dag = new Dag<>();
    for (Hop hopRoot : hops) {
        Lop lops = hopRoot.constructLops();
        lops.addToDag(dag);
    }
    // generate runtime instructions (incl piggybacking)
    ArrayList<Instruction> newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
    // defer the explain of instructions after additional modifications
    if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS) {
        if (pred)
            logExplainPred(hops.get(0), newInst);
        else
            logExplainDAG(sb, hops, newInst);
    }
    return newInst;
}

Example 5 with Dag

use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.

the class Recompiler method recompileHopsDag2Forced.

/**
	 * C) Recompile basic program block hop DAG, but forced to CP.  
	 * 
	 * This happens always 'inplace', without statistics updates, and 
	 * without dynamic rewrites.
	 * 
	 * @param sb statement block
	 * @param hops list of high-level operators
	 * @param tid thread id
	 * @param et execution type
	 * @return list of instructions
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 * @throws IOException if IOException occurs
	 */
public static ArrayList<Instruction> recompileHopsDag2Forced(StatementBlock sb, ArrayList<Hop> hops, long tid, ExecType et) throws DMLRuntimeException, HopsException, LopsException, IOException {
    ArrayList<Instruction> newInst = null;
    //however, we create deep copies for most dags to allow for concurrent recompile
    synchronized (hops) {
        LOG.debug("\n**************** Optimizer (Recompile) *************\nMemory Budget = " + OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
        // clear existing lops
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) rClearLops(hopRoot);
        // update exec type
        Hop.resetVisitStatus(hops);
        for (Hop hopRoot : hops) rSetExecType(hopRoot, et);
        Hop.resetVisitStatus(hops);
        // construct lops			
        Dag<Lop> dag = new Dag<Lop>();
        for (Hop hopRoot : hops) {
            Lop lops = hopRoot.constructLops();
            lops.addToDag(dag);
        }
        // generate runtime instructions (incl piggybacking)
        newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
    }
    // replace thread ids in new instructions
    if (//only in parfor context
    tid != 0)
        newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
    return newInst;
}