Examples with Timing org.apache.sysml.runtime.controlprogram.parfor.stat.Timing used on opensource projects

Example 1 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class GDFEnumOptimizer method optimize.

@Override
public GDFGraph optimize(GDFGraph gdfgraph, Summary summary) throws DMLRuntimeException, HopsException, LopsException {
    Timing time = new Timing(true);
    Program prog = gdfgraph.getRuntimeProgram();
    ExecutionContext ec = ExecutionContextFactory.createContext(prog);
    ArrayList<GDFNode> roots = gdfgraph.getGraphRootNodes();
    //Step 1: baseline costing for branch and bound costs
    double initCosts = Double.MAX_VALUE;
    if (BRANCH_AND_BOUND_PRUNING) {
        initCosts = CostEstimationWrapper.getTimeEstimate(prog, ec);
        initCosts = initCosts * (1 + BRANCH_AND_BOUND_REL_THRES);
    }
    //Step 2: dynamic programming plan generation
    //(finally, pick optimal root plans over all interesting property sets)
    ArrayList<Plan> rootPlans = new ArrayList<Plan>();
    for (GDFNode node : roots) {
        PlanSet ps = enumOpt(node, _memo, initCosts);
        Plan optPlan = ps.getPlanWithMinCosts();
        rootPlans.add(optPlan);
    }
    long enumPlanMismatch = getPlanMismatches();
    //check for final containment of independent roots and pick optimal
    HashMap<Long, Plan> memo = new HashMap<Long, Plan>();
    resetPlanMismatches();
    for (Plan p : rootPlans) rSetRuntimePlanConfig(p, memo);
    long finalPlanMismatch = getPlanMismatches();
    //generate final runtime plan (w/ optimal config)
    Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
    ec = ExecutionContextFactory.createContext(prog);
    double optCosts = CostEstimationWrapper.getTimeEstimate(prog, ec);
    //maintain optimization summary statistics
    summary.setCostsInitial(initCosts);
    summary.setCostsOptimal(optCosts);
    summary.setNumEnumPlans(_enumeratedPlans);
    summary.setNumPrunedInvalidPlans(_prunedInvalidPlans);
    summary.setNumPrunedSuboptPlans(_prunedSuboptimalPlans);
    summary.setNumCompiledPlans(_compiledPlans);
    summary.setNumCostedPlans(_costedPlans);
    summary.setNumEnumPlanMismatch(enumPlanMismatch);
    summary.setNumFinalPlanMismatch(finalPlanMismatch);
    summary.setTimeOptim(time.stop());
    return gdfgraph;
}

Example 2 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class GlobalOptimizerWrapper method optimizeProgram.

public static Program optimizeProgram(DMLProgram prog, Program rtprog) throws DMLRuntimeException, HopsException, LopsException {
    LOG.debug("Starting global data flow optimization.");
    Timing time = new Timing(true);
    //create optimizer instance
    GlobalOptimizer optimizer = createGlobalOptimizer(OPTIM);
    //create global data flow graph
    Summary summary = new Summary();
    GDFGraph graph = GraphBuilder.constructGlobalDataFlowGraph(rtprog, summary);
    if (LOG.isDebugEnabled()) {
        LOG.debug("EXPLAIN GDFGraph:\n" + Explain.explainGDFNodes(graph.getGraphRootNodes(), 1));
    }
    //core global data flow optimization 
    graph = optimizer.optimize(graph, summary);
    //get the final runtime program
    rtprog = graph.getRuntimeProgram();
    //print global optimizer summary
    LOG.info(summary);
    LOG.debug("Finished global data flow optimization in " + time.stop() + " ms.");
    return rtprog;
}

Example 3 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class ResultMergeLocalAutomatic method executeSerialMerge.

@Override
public MatrixObject executeSerialMerge() {
    Timing time = new Timing(true);
    MatrixCharacteristics mc = _output.getMatrixCharacteristics();
    long rows = mc.getRows();
    long cols = mc.getCols();
    if (OptimizerRuleBased.isInMemoryResultMerge(rows, cols, OptimizerUtils.getLocalMemBudget()))
        _rm = new ResultMergeLocalMemory(_output, _inputs, _outputFName, _isAccum);
    else
        _rm = new ResultMergeLocalFile(_output, _inputs, _outputFName, _isAccum);
    MatrixObject ret = _rm.executeSerialMerge();
    LOG.trace("Automatic result merge (" + _rm.getClass().getName() + ") executed in " + time.stop() + "ms.");
    return ret;
}

Example 4 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class PiggybackingWorker method mergeMRJobInstructions.

protected LinkedList<MergedMRJobInstruction> mergeMRJobInstructions(LinkedList<Pair<Long, MRJobInstruction>> workingSet) throws IllegalAccessException {
    LinkedList<MergedMRJobInstruction> ret = new LinkedList<>();
    Timing time = new Timing(true);
    // NOTE currently all merged into one (might be invalid due to memory constraints)
    MergedMRJobInstruction minst = new MergedMRJobInstruction();
    for (Pair<Long, MRJobInstruction> inst : workingSet) {
        long instID = inst.getKey();
        MRJobInstruction instVal = inst.getValue();
        int numOutputs = instVal.getOutputs().length;
        // append to current merged instruction
        if (minst.inst == null) {
            // deep copy first instruction
            minst.inst = new MRJobInstruction(instVal);
            minst.addInstructionMetaData(instID, 0, numOutputs);
        } else {
            // merge other instructions
            if (minst.inst.isMergableMRJobInstruction(instVal)) {
                // add instruction to open merged instruction
                // before merge
                int offOutputs = minst.inst.getOutputs().length;
                minst.inst.mergeMRJobInstruction(instVal);
                minst.addInstructionMetaData(instID, offOutputs, numOutputs);
            } else {
                // close current merged instruction
                ret.add(minst);
                // open new merged instruction
                minst = new MergedMRJobInstruction();
                minst.inst = new MRJobInstruction(instVal);
                minst.addInstructionMetaData(instID, 0, numOutputs);
            }
        }
    }
    // close last open merged instruction
    ret.add(minst);
    // output log info for better understandability for users
    LOG.info("Merged MR-Job instructions: " + workingSet.size() + " --> " + ret.size() + " in " + time.stop() + "ms.");
    return ret;
}

Example 5 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class OptimizationWrapper method optimize.

/**
 * Called once per top-level parfor (during runtime, on parfor execute)
 * in order to optimize the specific parfor program block.
 *
 * NOTE: this is the default way to invoke parfor optimizers.
 *
 * @param type ?
 * @param sb parfor statement block
 * @param pb parfor program block
 * @param ec execution context
 * @param monitor ?
 */
public static void optimize(POptMode type, ParForStatementBlock sb, ParForProgramBlock pb, ExecutionContext ec, boolean monitor) {
    Timing time = new Timing(true);
    LOG.debug("ParFOR Opt: Running optimization for ParFOR(" + pb.getID() + ")");
    // set max contraints if not specified
    int ck = UtilFunctions.toInt(Math.max(InfrastructureAnalyzer.getCkMaxCP(), InfrastructureAnalyzer.getCkMaxMR()) * PAR_FACTOR_INFRASTRUCTURE);
    double cm = InfrastructureAnalyzer.getCmMax() * OptimizerUtils.MEM_UTIL_FACTOR;
    // execute optimizer
    optimize(type, ck, cm, sb, pb, ec, monitor);
    double timeVal = time.stop();
    LOG.debug("ParFOR Opt: Finished optimization for PARFOR(" + pb.getID() + ") in " + timeVal + "ms.");
    // System.out.println("ParFOR Opt: Finished optimization for PARFOR("+pb.getID()+") in "+timeVal+"ms.");
    if (monitor)
        StatisticMonitor.putPFStat(pb.getID(), Stat.OPT_T, timeVal);
}