Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 36 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNodes.

public static ArrayList<OptNode> rCreateAbstractOptNodes(Hop hop, LocalVariableMap vars, Set<String> memo) {
    ArrayList<OptNode> ret = new ArrayList<>();
    ArrayList<Hop> in = hop.getInput();
    if (hop.isVisited())
        return ret;
    // general case
    if (!(hop instanceof DataOp || hop instanceof LiteralOp || hop instanceof FunctionOp)) {
        OptNode node = new OptNode(NodeType.HOP);
        String opstr = hop.getOpString();
        node.addParam(ParamType.OPSTRING, opstr);
        // handle execution type
        LopProperties.ExecType et = (hop.getExecType() != null) ? hop.getExecType() : LopProperties.ExecType.CP;
        switch(et) {
            case CP:
            case GPU:
                node.setExecType(ExecType.CP);
                break;
            case SPARK:
                node.setExecType(ExecType.SPARK);
                break;
            case MR:
                node.setExecType(ExecType.MR);
                break;
            default:
                throw new DMLRuntimeException("Unsupported optnode exec type: " + et);
        }
        // handle degree of parallelism
        if (et == LopProperties.ExecType.CP && hop instanceof MultiThreadedHop) {
            MultiThreadedHop mtop = (MultiThreadedHop) hop;
            node.setK(OptimizerUtils.getConstrainedNumThreads(mtop.getMaxNumThreads()));
        }
        // assign node to return
        _hlMap.putHopMapping(hop, node);
        ret.add(node);
    } else // process function calls
    if (hop instanceof FunctionOp && INCLUDE_FUNCTIONS) {
        FunctionOp fhop = (FunctionOp) hop;
        String fname = fhop.getFunctionName();
        String fnspace = fhop.getFunctionNamespace();
        String fKey = fhop.getFunctionKey();
        Object[] prog = _hlMap.getRootProgram();
        OptNode node = new OptNode(NodeType.FUNCCALL);
        _hlMap.putHopMapping(fhop, node);
        node.setExecType(ExecType.CP);
        node.addParam(ParamType.OPSTRING, fKey);
        if (!fnspace.equals(DMLProgram.INTERNAL_NAMESPACE)) {
            FunctionProgramBlock fpb = ((Program) prog[1]).getFunctionProgramBlock(fnspace, fname);
            FunctionStatementBlock fsb = ((DMLProgram) prog[0]).getFunctionStatementBlock(fnspace, fname);
            FunctionStatement fs = (FunctionStatement) fsb.getStatement(0);
            // process body; NOTE: memo prevents inclusion of functions multiple times
            if (!memo.contains(fKey)) {
                memo.add(fKey);
                int len = fs.getBody().size();
                for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
                    ProgramBlock lpb = fpb.getChildBlocks().get(i);
                    StatementBlock lsb = fs.getBody().get(i);
                    node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
                }
                memo.remove(fKey);
            } else
                node.addParam(ParamType.RECURSIVE_CALL, "true");
        }
        ret.add(node);
    }
    if (in != null)
        for (Hop hin : in) if (// no need for opt nodes
        !(hin instanceof DataOp || hin instanceof LiteralOp))
            ret.addAll(rCreateAbstractOptNodes(hin, vars, memo));
    hop.setVisited();
    return ret;
}

Example 37 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class OptimizerRuleBased method rGetUIPConsumerList.

private void rGetUIPConsumerList(Hop hop, HashMap<String, ArrayList<UIPCandidateHop>> uipCandHopHM) throws DMLRuntimeException {
    if (hop.isVisited())
        return;
    if ((!(!hop.getParent().isEmpty() && hop.getParent().get(0) instanceof LeftIndexingOp)) && ((hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTREAD) || (hop instanceof ReorgOp && (((ReorgOp) hop).getOp() == ReOrgOp.RESHAPE || ((ReorgOp) hop).getOp() == ReOrgOp.TRANSPOSE)) || (hop instanceof FunctionOp))) {
        // If candidate's name is same as input hop.
        String uipCandiateID = hop.getName();
        ArrayList<UIPCandidateHop> uipCandHopList = uipCandHopHM.get(uipCandiateID);
        if (uipCandHopList != null) {
            for (UIPCandidateHop uipCandHop : uipCandHopList) {
                // Add consumers for candidate hop.
                ArrayList<Hop> consumerHops = uipCandHop.getConsumerHops();
                if (uipCandHop.getConsumerHops() == null)
                    consumerHops = new ArrayList<Hop>();
                consumerHops.add(getRootHop(hop));
                uipCandHop.setConsumerHops(consumerHops);
            }
        }
    }
    for (Hop hopIn : hop.getInput()) rGetUIPConsumerList(hopIn, uipCandHopHM);
    hop.setVisited();
}

Example 38 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class GraphBuilder method constructGDFGraph.

@SuppressWarnings("unchecked")
private static void constructGDFGraph(ProgramBlock pb, HashMap<String, GDFNode> roots) throws DMLRuntimeException, HopsException {
    if (pb instanceof FunctionProgramBlock) {
        throw new DMLRuntimeException("FunctionProgramBlocks not implemented yet.");
    } else if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) pb.getStatementBlock();
        //construct predicate node (conceptually sequence of from/to/incr)
        GDFNode pred = constructGDFGraph(wsb.getPredicateHops(), wpb, new HashMap<Long, GDFNode>(), roots);
        HashMap<String, GDFNode> inputs = constructLoopInputNodes(wpb, wsb, roots);
        HashMap<String, GDFNode> lroots = (HashMap<String, GDFNode>) inputs.clone();
        //process childs blocks
        for (ProgramBlock pbc : wpb.getChildBlocks()) constructGDFGraph(pbc, lroots);
        HashMap<String, GDFNode> outputs = constructLoopOutputNodes(wsb, lroots);
        GDFLoopNode lnode = new GDFLoopNode(wpb, pred, inputs, outputs);
        //construct crossblock nodes
        constructLoopOutputCrossBlockNodes(wsb, lnode, outputs, roots, wpb);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) pb.getStatementBlock();
        //construct predicate
        if (isb.getPredicateHops() != null) {
            Hop pred = isb.getPredicateHops();
            roots.put(pred.getName(), constructGDFGraph(pred, ipb, new HashMap<Long, GDFNode>(), roots));
        }
        //construct if and else branch separately
        HashMap<String, GDFNode> ifRoots = (HashMap<String, GDFNode>) roots.clone();
        HashMap<String, GDFNode> elseRoots = (HashMap<String, GDFNode>) roots.clone();
        for (ProgramBlock pbc : ipb.getChildBlocksIfBody()) constructGDFGraph(pbc, ifRoots);
        if (ipb.getChildBlocksElseBody() != null)
            for (ProgramBlock pbc : ipb.getChildBlocksElseBody()) constructGDFGraph(pbc, elseRoots);
        //merge data flow roots (if no else, elseRoots refer to original roots)
        reconcileMergeIfProgramBlockOutputs(ifRoots, elseRoots, roots, ipb);
    } else if (//incl parfor
    pb instanceof ForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) pb.getStatementBlock();
        //construct predicate node (conceptually sequence of from/to/incr)
        GDFNode pred = constructForPredicateNode(fpb, fsb, roots);
        HashMap<String, GDFNode> inputs = constructLoopInputNodes(fpb, fsb, roots);
        HashMap<String, GDFNode> lroots = (HashMap<String, GDFNode>) inputs.clone();
        //process childs blocks
        for (ProgramBlock pbc : fpb.getChildBlocks()) constructGDFGraph(pbc, lroots);
        HashMap<String, GDFNode> outputs = constructLoopOutputNodes(fsb, lroots);
        GDFLoopNode lnode = new GDFLoopNode(fpb, pred, inputs, outputs);
        //construct crossblock nodes
        constructLoopOutputCrossBlockNodes(fsb, lnode, outputs, roots, fpb);
    } else //last-level program block
    {
        StatementBlock sb = pb.getStatementBlock();
        ArrayList<Hop> hops = sb.get_hops();
        if (hops != null) {
            //create new local memo structure for local dag
            HashMap<Long, GDFNode> lmemo = new HashMap<Long, GDFNode>();
            for (Hop hop : hops) {
                //recursively construct GDF graph for hop dag root
                GDFNode root = constructGDFGraph(hop, pb, lmemo, roots);
                if (root == null)
                    throw new HopsException("GDFGraphBuilder: failed to constuct dag root for: " + Explain.explain(hop));
                //create cross block nodes for all transient writes
                if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTWRITE)
                    root = new GDFCrossBlockNode(hop, pb, root, hop.getName());
                //add GDF root node to global roots 
                roots.put(hop.getName(), root);
            }
        }
    }
}

Example 39 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class GraphBuilder method constructGDFGraph.

private static GDFNode constructGDFGraph(Hop hop, ProgramBlock pb, HashMap<Long, GDFNode> lmemo, HashMap<String, GDFNode> roots) {
    if (lmemo.containsKey(hop.getHopID()))
        return lmemo.get(hop.getHopID());
    //process childs recursively first
    ArrayList<GDFNode> inputs = new ArrayList<GDFNode>();
    for (Hop c : hop.getInput()) inputs.add(constructGDFGraph(c, pb, lmemo, roots));
    //connect transient reads to existing roots of data flow graph 
    if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTREAD) {
        inputs.add(roots.get(hop.getName()));
    }
    //add current hop
    GDFNode gnode = new GDFNode(hop, pb, inputs);
    //variables might never be bound to their logical variables names
    if (!IGNORE_UNBOUND_UPDATED_VARS) {
        //NOTE: currently disabled because unnecessary, if no transientwrite by definition included in other transientwrite
        if (pb.getStatementBlock() != null && pb.getStatementBlock().variablesUpdated().containsVariable(hop.getName())) {
            roots.put(hop.getName(), gnode);
        }
    }
    //memoize current node
    lmemo.put(hop.getHopID(), gnode);
    return gnode;
}

Example 40 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class InterProceduralAnalysis method rRemoveConstantBinaryOp.

private void rRemoveConstantBinaryOp(Hop hop, HashMap<String, Hop> mOnes) {
    if (hop.isVisited())
        return;
    if (hop instanceof BinaryOp && ((BinaryOp) hop).getOp() == OpOp2.MULT && !((BinaryOp) hop).isOuterVectorOperator() && hop.getInput().get(0).getDataType() == DataType.MATRIX && hop.getInput().get(1) instanceof DataOp && mOnes.containsKey(hop.getInput().get(1).getName())) {
        //replace matrix of ones with literal 1 (later on removed by
        //algebraic simplification rewrites; otherwise more complex
        //recursive processing of childs and rewiring required)
        HopRewriteUtils.removeChildReferenceByPos(hop, hop.getInput().get(1), 1);
        HopRewriteUtils.addChildReference(hop, new LiteralOp(1), 1);
    }
    //recursively process child nodes
    for (Hop c : hop.getInput()) rRemoveConstantBinaryOp(c, mOnes);
    hop.setVisited();
}