Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 46 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class GDFEnumOptimizer method costRuntimePlan.

private static double costRuntimePlan(Plan p) throws DMLRuntimeException {
    Program prog = p.getNode().getProgram();
    if (prog == null)
        throw new DMLRuntimeException("Program not available for runtime plan costing.");
    //put data flow configuration into program
    rSetRuntimePlanConfig(p, new HashMap<Long, Plan>());
    double costs = -1;
    if (COST_FULL_PROGRAMS || (p.getNode().getHop() == null || p.getNode().getProgramBlock() == null)) {
        //recompile entire runtime program
        Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
        _compiledPlans++;
        //cost entire runtime program
        ExecutionContext ec = ExecutionContextFactory.createContext(prog);
        costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
    } else {
        Hop currentHop = p.getNode().getHop();
        ProgramBlock pb = p.getNode().getProgramBlock();
        try {
            //keep the old dag roots
            ArrayList<Hop> oldRoots = pb.getStatementBlock().get_hops();
            Hop tmpHop = null;
            if (!(currentHop instanceof DataOp && ((DataOp) currentHop).isWrite())) {
                ArrayList<Hop> newRoots = new ArrayList<Hop>();
                tmpHop = new DataOp("_tmp", currentHop.getDataType(), currentHop.getValueType(), currentHop, DataOpTypes.TRANSIENTWRITE, "tmp");
                //ensure recursive visitstatus reset on recompile
                tmpHop.setVisited();
                newRoots.add(tmpHop);
                pb.getStatementBlock().set_hops(newRoots);
            }
            //recompile modified runtime program
            Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
            _compiledPlans++;
            //cost partial runtime program up to current hop
            ExecutionContext ec = ExecutionContextFactory.createContext(prog);
            costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
            //restore original hop dag
            if (tmpHop != null)
                HopRewriteUtils.removeChildReference(tmpHop, currentHop);
            pb.getStatementBlock().set_hops(oldRoots);
        } catch (HopsException ex) {
            throw new DMLRuntimeException(ex);
        }
    }
    //release forced data flow configuration from program
    rResetRuntimePlanConfig(p, new HashMap<Long, Plan>());
    _costedPlans++;
    return costs;
}

Example 47 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class GDFEnumOptimizer method rSetRuntimePlanConfig.

private static void rSetRuntimePlanConfig(Plan p, HashMap<Long, Plan> memo) {
    ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    //basic memoization including containment check 
    if (memo.containsKey(p.getNode().getID())) {
        Plan pmemo = memo.get(p.getNode().getID());
        if (!p.getInterestingProperties().equals(pmemo.getInterestingProperties())) {
            //TODO this would require additional cleanup in special cases
            if (_resolve.resolveMismatch(pmemo.getRewriteConfig(), p.getRewriteConfig()))
                memo.put(p.getNode().getID(), p);
            //logging of encounter plan mismatch
            LOG.warn("Configuration mismatch on shared node (" + p.getNode().getHop().getHopID() + "). Falling back to heuristic '" + _resolve.getName() + "'.");
            LOG.warn(p.getInterestingProperties().toString());
            LOG.warn(memo.get(p.getNode().getID()).getInterestingProperties());
            _planMismatches++;
            return;
        }
    }
    //set plan configuration
    Hop hop = p.getNode().getHop();
    if (hop != null) {
        RewriteConfig rc = p.getRewriteConfig();
        //set exec type
        hop.setForcedExecType(rc.getExecType());
        //set blocksizes and reblock
        hop.setRowsInBlock(rc.getBlockSize());
        hop.setColsInBlock(rc.getBlockSize());
        if (//after blocksize update
        rc.getExecType() == CLUSTER) {
            //TODO double check dataop condition - side effect from plan validity
            boolean reblock = HopRewriteUtils.alwaysRequiresReblock(hop) || (hop.hasMatrixInputWithDifferentBlocksizes() && !(hop instanceof DataOp));
            hop.setRequiresReblock(reblock);
        } else
            hop.setRequiresReblock(false);
    }
    //process childs
    if (p.getChilds() != null)
        for (Plan c : p.getChilds()) rSetRuntimePlanConfig(c, memo);
    //memoization (mark as processed)
    memo.put(p.getNode().getID(), p);
}