Examples with Timing org.apache.sysml.runtime.controlprogram.parfor.stat.Timing used on opensource projects

Example 26 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project systemml by apache.

the class ParWorker method executeSetTask.

private void executeSetTask(Task task) {
    // monitoring start
    Timing time1 = null, time2 = null;
    if (_monitor) {
        time1 = new Timing(true);
        time2 = new Timing(true);
    }
    // core execution
    // foreach iteration in task, execute iteration body
    String lVarName = task.getVarName();
    for (IntObject indexVal : task.getIterations()) {
        // System.out.println(" EXECUTE ITERATION: "+indexVal.getName()+"="+indexVal.getIntValue());
        // set index values
        _ec.setVariable(lVarName, indexVal);
        // for each program block
        for (ProgramBlock pb : _childBlocks) pb.execute(_ec);
        _numIters++;
        if (_monitor)
            StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_ITER_T, time1.stop());
    }
    _numTasks++;
    // monitoring end
    if (_monitor) {
        StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_TASKSIZE, task.size());
        StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_TASK_T, time2.stop());
    }
}

Example 27 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project systemml by apache.

the class ResultMergeLocalAutomatic method executeSerialMerge.

@Override
public MatrixObject executeSerialMerge() {
    Timing time = new Timing(true);
    MatrixCharacteristics mc = _output.getMatrixCharacteristics();
    long rows = mc.getRows();
    long cols = mc.getCols();
    if (OptimizerRuleBased.isInMemoryResultMerge(rows, cols, OptimizerUtils.getLocalMemBudget()))
        _rm = new ResultMergeLocalMemory(_output, _inputs, _outputFName, _isAccum);
    else
        _rm = new ResultMergeLocalFile(_output, _inputs, _outputFName, _isAccum);
    MatrixObject ret = _rm.executeSerialMerge();
    LOG.trace("Automatic result merge (" + _rm.getClass().getName() + ") executed in " + time.stop() + "ms.");
    return ret;
}

Example 28 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project systemml by apache.

the class OptimizationWrapper method optimize.

@SuppressWarnings("unused")
private static void optimize(POptMode otype, int ck, double cm, ParForStatementBlock sb, ParForProgramBlock pb, ExecutionContext ec, boolean monitor) {
    Timing time = new Timing(true);
    // maintain statistics
    if (DMLScript.STATISTICS)
        Statistics.incrementParForOptimCount();
    // create specified optimizer
    Optimizer opt = createOptimizer(otype);
    CostModelType cmtype = opt.getCostModelType();
    LOG.trace("ParFOR Opt: Created optimizer (" + otype + "," + opt.getPlanInputType() + "," + opt.getCostModelType());
    OptTree tree = null;
    // recompile parfor body
    if (ConfigurationManager.isDynamicRecompilation()) {
        ForStatement fs = (ForStatement) sb.getStatement(0);
        // debug output before recompilation
        if (LOG.isDebugEnabled()) {
            try {
                tree = OptTreeConverter.createOptTree(ck, cm, opt.getPlanInputType(), sb, pb, ec);
                LOG.debug("ParFOR Opt: Input plan (before recompilation):\n" + tree.explain(false));
                OptTreeConverter.clear();
            } catch (Exception ex) {
                throw new DMLRuntimeException("Unable to create opt tree.", ex);
            }
        }
        // separate propagation required because recompile in-place without literal replacement)
        try {
            LocalVariableMap constVars = ProgramRecompiler.getReusableScalarVariables(sb.getDMLProg(), sb, ec.getVariables());
            ProgramRecompiler.replaceConstantScalarVariables(sb, constVars);
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
        // program rewrites (e.g., constant folding, branch removal) according to replaced literals
        try {
            ProgramRewriter rewriter = createProgramRewriterWithRuleSets();
            ProgramRewriteStatus state = new ProgramRewriteStatus();
            rewriter.rRewriteStatementBlockHopDAGs(sb, state);
            fs.setBody(rewriter.rRewriteStatementBlocks(fs.getBody(), state, true));
            if (state.getRemovedBranches()) {
                LOG.debug("ParFOR Opt: Removed branches during program rewrites, rebuilding runtime program");
                pb.setChildBlocks(ProgramRecompiler.generatePartitialRuntimeProgram(pb.getProgram(), fs.getBody()));
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
        // recompilation of parfor body and called functions (if safe)
        try {
            // core parfor body recompilation (based on symbol table entries)
            // * clone of variables in order to allow for statistics propagation across DAGs
            // (tid=0, because deep copies created after opt)
            LocalVariableMap tmp = (LocalVariableMap) ec.getVariables().clone();
            ResetType reset = ConfigurationManager.isCodegenEnabled() ? ResetType.RESET_KNOWN_DIMS : ResetType.RESET;
            Recompiler.recompileProgramBlockHierarchy(pb.getChildBlocks(), tmp, 0, reset);
            // inter-procedural optimization (based on previous recompilation)
            if (pb.hasFunctions()) {
                InterProceduralAnalysis ipa = new InterProceduralAnalysis(sb);
                Set<String> fcand = ipa.analyzeSubProgram();
                if (!fcand.isEmpty()) {
                    // regenerate runtime program of modified functions
                    for (String func : fcand) {
                        String[] funcparts = DMLProgram.splitFunctionKey(func);
                        FunctionProgramBlock fpb = pb.getProgram().getFunctionProgramBlock(funcparts[0], funcparts[1]);
                        // reset recompilation flags according to recompileOnce because it is only safe if function is recompileOnce
                        // because then recompiled for every execution (otherwise potential issues if func also called outside parfor)
                        ResetType reset2 = fpb.isRecompileOnce() ? reset : ResetType.NO_RESET;
                        Recompiler.recompileProgramBlockHierarchy(fpb.getChildBlocks(), new LocalVariableMap(), 0, reset2);
                    }
                }
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
    }
    // create opt tree (before optimization)
    try {
        tree = OptTreeConverter.createOptTree(ck, cm, opt.getPlanInputType(), sb, pb, ec);
        LOG.debug("ParFOR Opt: Input plan (before optimization):\n" + tree.explain(false));
    } catch (Exception ex) {
        throw new DMLRuntimeException("Unable to create opt tree.", ex);
    }
    // create cost estimator
    CostEstimator est = createCostEstimator(cmtype, ec.getVariables());
    LOG.trace("ParFOR Opt: Created cost estimator (" + cmtype + ")");
    // core optimize
    opt.optimize(sb, pb, tree, est, ec);
    LOG.debug("ParFOR Opt: Optimized plan (after optimization): \n" + tree.explain(false));
    // assert plan correctness
    if (CHECK_PLAN_CORRECTNESS && LOG.isDebugEnabled()) {
        try {
            OptTreePlanChecker.checkProgramCorrectness(pb, sb, new HashSet<String>());
            LOG.debug("ParFOR Opt: Checked plan and program correctness.");
        } catch (Exception ex) {
            throw new DMLRuntimeException("Failed to check program correctness.", ex);
        }
    }
    long ltime = (long) time.stop();
    LOG.trace("ParFOR Opt: Optimized plan in " + ltime + "ms.");
    if (DMLScript.STATISTICS)
        Statistics.incrementParForOptimTime(ltime);
    // cleanup phase
    OptTreeConverter.clear();
    // monitor stats
    if (monitor) {
        StatisticMonitor.putPFStat(pb.getID(), Stat.OPT_OPTIMIZER, otype.ordinal());
        StatisticMonitor.putPFStat(pb.getID(), Stat.OPT_NUMTPLANS, opt.getNumTotalPlans());
        StatisticMonitor.putPFStat(pb.getID(), Stat.OPT_NUMEPLANS, opt.getNumEvaluatedPlans());
    }
}

Example 29 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project systemml by apache.

the class OptimizationWrapper method optimize.

/**
 * Called once per top-level parfor (during runtime, on parfor execute)
 * in order to optimize the specific parfor program block.
 *
 * NOTE: this is the default way to invoke parfor optimizers.
 *
 * @param type ?
 * @param sb parfor statement block
 * @param pb parfor program block
 * @param ec execution context
 * @param monitor ?
 */
public static void optimize(POptMode type, ParForStatementBlock sb, ParForProgramBlock pb, ExecutionContext ec, boolean monitor) {
    Timing time = new Timing(true);
    LOG.debug("ParFOR Opt: Running optimization for ParFOR(" + pb.getID() + ")");
    // set max contraints if not specified
    int ck = UtilFunctions.toInt(Math.max(InfrastructureAnalyzer.getCkMaxCP(), InfrastructureAnalyzer.getCkMaxMR()) * PAR_FACTOR_INFRASTRUCTURE);
    double cm = InfrastructureAnalyzer.getCmMax() * OptimizerUtils.MEM_UTIL_FACTOR;
    // execute optimizer
    optimize(type, ck, cm, sb, pb, ec, monitor);
    double timeVal = time.stop();
    LOG.debug("ParFOR Opt: Finished optimization for PARFOR(" + pb.getID() + ") in " + timeVal + "ms.");
    // System.out.println("ParFOR Opt: Finished optimization for PARFOR("+pb.getID()+") in "+timeVal+"ms.");
    if (monitor)
        StatisticMonitor.putPFStat(pb.getID(), Stat.OPT_T, timeVal);
}

Example 30 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project systemml by apache.

the class FrameLeftIndexingTest method execDMLScriptviaJMLC.

private static ArrayList<String[][]> execDMLScriptviaJMLC(String testname, String[][] F1, String[][] M, boolean modelReuse) throws IOException {
    Timing time = new Timing(true);
    ArrayList<String[][]> ret = new ArrayList<String[][]>();
    // establish connection to SystemML
    Connection conn = new Connection();
    try {
        // prepare input arguments
        HashMap<String, String> args = new HashMap<String, String>();
        args.put("$TRANSFORM_SPEC1", "{ \"ids\": true ,\"recode\": [ 1, 2] }");
        args.put("$TRANSFORM_SPEC2", "{ \"ids\": true ,\"recode\": [ 1] }");
        // read and precompile script
        String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
        PreparedScript pstmt = conn.prepareScript(script, args, new String[] { "F1", "M" }, new String[] { "F2" }, false);
        if (modelReuse)
            pstmt.setFrame("M", M, true);
        // execute script multiple times
        for (int i = 0; i < nRuns; i++) {
            // bind input parameters
            if (!modelReuse)
                pstmt.setFrame("M", M);
            pstmt.setFrame("F1", F1);
            // execute script
            ResultVariables rs = pstmt.executeScript();
            // get output parameter
            String[][] Y = rs.getFrame("F2");
            // keep result for comparison
            ret.add(Y);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new IOException(ex);
    } finally {
        IOUtilFunctions.closeSilently(conn);
    }
    System.out.println("JMLC scoring w/ " + nRuns + " runs in " + time.stop() + "ms.");
    return ret;
}