Examples with FunctionOp org.apache.sysml.hops.FunctionOp used on opensource projects

Example 6 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class InterProceduralAnalysis method analyzeSubProgram.

public Set<String> analyzeSubProgram(StatementBlock sb) throws HopsException, ParseException {
    DMLTranslator.resetHopsDAGVisitStatus(sb);
    //step 1: get candidates for statistics propagation into functions (if required)
    Map<String, Integer> fcandCounts = new HashMap<String, Integer>();
    Map<String, FunctionOp> fcandHops = new HashMap<String, FunctionOp>();
    Map<String, Set<Long>> fcandSafeNNZ = new HashMap<String, Set<Long>>();
    Set<String> allFCandKeys = new HashSet<String>();
    getFunctionCandidatesForStatisticPropagation(sb, fcandCounts, fcandHops);
    //cp before pruning
    allFCandKeys.addAll(fcandCounts.keySet());
    pruneFunctionCandidatesForStatisticPropagation(fcandCounts, fcandHops);
    determineFunctionCandidatesNNZPropagation(fcandHops, fcandSafeNNZ);
    DMLTranslator.resetHopsDAGVisitStatus(sb);
    if (!fcandCounts.isEmpty()) {
        //step 2: propagate statistics into functions and across DAGs
        //(callVars used to chain outputs/inputs of multiple functions calls) 
        LocalVariableMap callVars = new LocalVariableMap();
        propagateStatisticsAcrossBlock(sb, fcandCounts, callVars, fcandSafeNNZ, new HashSet<String>(), new HashSet<String>());
    }
    return fcandCounts.keySet();
}

Example 7 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class OptimizerRuleBased method rGetUIPConsumerList.

private void rGetUIPConsumerList(Hop hop, HashMap<String, ArrayList<UIPCandidateHop>> uipCandHopHM) throws DMLRuntimeException {
    if (hop.isVisited())
        return;
    if ((!(!hop.getParent().isEmpty() && hop.getParent().get(0) instanceof LeftIndexingOp)) && ((hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTREAD) || (hop instanceof ReorgOp && (((ReorgOp) hop).getOp() == ReOrgOp.RESHAPE || ((ReorgOp) hop).getOp() == ReOrgOp.TRANSPOSE)) || (hop instanceof FunctionOp))) {
        // If candidate's name is same as input hop.
        String uipCandiateID = hop.getName();
        ArrayList<UIPCandidateHop> uipCandHopList = uipCandHopHM.get(uipCandiateID);
        if (uipCandHopList != null) {
            for (UIPCandidateHop uipCandHop : uipCandHopList) {
                // Add consumers for candidate hop.
                ArrayList<Hop> consumerHops = uipCandHop.getConsumerHops();
                if (uipCandHop.getConsumerHops() == null)
                    consumerHops = new ArrayList<Hop>();
                consumerHops.add(getRootHop(hop));
                uipCandHop.setConsumerHops(consumerHops);
            }
        }
    }
    for (Hop hopIn : hop.getInput()) rGetUIPConsumerList(hopIn, uipCandHopHM);
    hop.setVisited();
}

Example 8 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class OptimizerRuleBased method rFindAndUnfoldRecursiveFunction.

protected void rFindAndUnfoldRecursiveFunction(OptNode n, ParForProgramBlock parfor, HashSet<ParForProgramBlock> recPBs, LocalVariableMap vars) throws DMLRuntimeException, HopsException, LanguageException {
    //unfold if found
    if (n.getNodeType() == NodeType.FUNCCALL && n.isRecursive()) {
        boolean exists = rContainsNode(n, parfor);
        if (exists) {
            String fnameKey = n.getParam(ParamType.OPSTRING);
            String[] names = fnameKey.split(Program.KEY_DELIM);
            String fnamespace = names[0];
            String fname = names[1];
            String fnameNew = FUNCTION_UNFOLD_NAMEPREFIX + fname;
            //unfold function
            FunctionOp fop = (FunctionOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
            Program prog = parfor.getProgram();
            DMLProgram dmlprog = parfor.getStatementBlock().getDMLProg();
            FunctionProgramBlock fpb = prog.getFunctionProgramBlock(fnamespace, fname);
            FunctionProgramBlock copyfpb = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<String>(), new HashSet<String>());
            prog.addFunctionProgramBlock(fnamespace, fnameNew, copyfpb);
            dmlprog.addFunctionStatementBlock(fnamespace, fnameNew, (FunctionStatementBlock) copyfpb.getStatementBlock());
            //replace function names in old subtree (link to new function)
            rReplaceFunctionNames(n, fname, fnameNew);
            //recreate sub opttree
            String fnameNewKey = fnamespace + Program.KEY_DELIM + fnameNew;
            OptNode nNew = new OptNode(NodeType.FUNCCALL);
            OptTreeConverter.getAbstractPlanMapping().putHopMapping(fop, nNew);
            nNew.setExecType(ExecType.CP);
            nNew.addParam(ParamType.OPSTRING, fnameNewKey);
            long parentID = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
            OptTreeConverter.getAbstractPlanMapping().getOptNode(parentID).exchangeChild(n, nNew);
            HashSet<String> memo = new HashSet<String>();
            //required if functionop not shared (because not replaced yet)
            memo.add(fnameKey);
            //requied if functionop shared (indirectly replaced)
            memo.add(fnameNewKey);
            for (int i = 0; i < copyfpb.getChildBlocks().size(); /*&& i<len*/
            i++) {
                ProgramBlock lpb = copyfpb.getChildBlocks().get(i);
                StatementBlock lsb = lpb.getStatementBlock();
                nNew.addChild(OptTreeConverter.rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
            //compute delta for recPB set (use for removing parfor)
            recPBs.removeAll(rGetAllParForPBs(n, new HashSet<ParForProgramBlock>()));
            recPBs.addAll(rGetAllParForPBs(nNew, new HashSet<ParForProgramBlock>()));
            //replace function names in new subtree (recursive link to new function)
            rReplaceFunctionNames(nNew, fname, fnameNew);
        }
        return;
    }
    //recursive invocation (only for non-recursive functions)
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) rFindAndUnfoldRecursiveFunction(c, parfor, recPBs, vars);
}

Example 9 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNodes.

public static ArrayList<OptNode> rCreateAbstractOptNodes(Hop hop, LocalVariableMap vars, Set<String> memo) throws DMLRuntimeException, HopsException {
    ArrayList<OptNode> ret = new ArrayList<OptNode>();
    ArrayList<Hop> in = hop.getInput();
    if (hop.isVisited())
        return ret;
    //general case
    if (!(hop instanceof DataOp || hop instanceof LiteralOp || hop instanceof FunctionOp)) {
        OptNode node = new OptNode(NodeType.HOP);
        String opstr = hop.getOpString();
        node.addParam(ParamType.OPSTRING, opstr);
        //handle execution type
        LopProperties.ExecType et = (hop.getExecType() != null) ? hop.getExecType() : LopProperties.ExecType.CP;
        switch(et) {
            case CP:
            case GPU:
                node.setExecType(ExecType.CP);
                break;
            case SPARK:
                node.setExecType(ExecType.SPARK);
                break;
            case MR:
                node.setExecType(ExecType.MR);
                break;
            default:
                throw new DMLRuntimeException("Unsupported optnode exec type: " + et);
        }
        //handle degree of parallelism
        if (et == LopProperties.ExecType.CP && hop instanceof MultiThreadedHop) {
            MultiThreadedHop mtop = (MultiThreadedHop) hop;
            node.setK(OptimizerUtils.getConstrainedNumThreads(mtop.getMaxNumThreads()));
        }
        //assign node to return
        _hlMap.putHopMapping(hop, node);
        ret.add(node);
    } else //process function calls
    if (hop instanceof FunctionOp && INCLUDE_FUNCTIONS) {
        FunctionOp fhop = (FunctionOp) hop;
        String fname = fhop.getFunctionName();
        String fnspace = fhop.getFunctionNamespace();
        String fKey = DMLProgram.constructFunctionKey(fnspace, fname);
        Object[] prog = _hlMap.getRootProgram();
        OptNode node = new OptNode(NodeType.FUNCCALL);
        _hlMap.putHopMapping(fhop, node);
        node.setExecType(ExecType.CP);
        node.addParam(ParamType.OPSTRING, fKey);
        if (!fnspace.equals(DMLProgram.INTERNAL_NAMESPACE)) {
            FunctionProgramBlock fpb = ((Program) prog[1]).getFunctionProgramBlock(fnspace, fname);
            FunctionStatementBlock fsb = ((DMLProgram) prog[0]).getFunctionStatementBlock(fnspace, fname);
            FunctionStatement fs = (FunctionStatement) fsb.getStatement(0);
            //process body; NOTE: memo prevents inclusion of functions multiple times
            if (!memo.contains(fKey)) {
                memo.add(fKey);
                int len = fs.getBody().size();
                for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
                    ProgramBlock lpb = fpb.getChildBlocks().get(i);
                    StatementBlock lsb = fs.getBody().get(i);
                    node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
                }
                memo.remove(fKey);
            } else
                node.addParam(ParamType.RECURSIVE_CALL, "true");
        }
        ret.add(node);
    }
    if (in != null)
        for (Hop hin : in) if (//no need for opt nodes
        !(hin instanceof DataOp || hin instanceof LiteralOp))
            ret.addAll(rCreateAbstractOptNodes(hin, vars, memo));
    hop.setVisited();
    return ret;
}

Example 10 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class OptTreeConverter method replaceProgramBlock.

public static void replaceProgramBlock(OptNode parent, OptNode n, ProgramBlock pbOld, ProgramBlock pbNew, boolean rtMap) throws DMLRuntimeException {
    ProgramBlock pbParent = null;
    if (rtMap)
        pbParent = (ProgramBlock) _rtMap.getMappedObject(parent.getID());
    else {
        if (parent.getNodeType() == NodeType.FUNCCALL) {
            FunctionOp fop = (FunctionOp) _hlMap.getMappedHop(parent.getID());
            pbParent = ((Program) _hlMap.getRootProgram()[1]).getFunctionProgramBlock(fop.getFunctionNamespace(), fop.getFunctionName());
        } else
            pbParent = (ProgramBlock) _hlMap.getMappedProg(parent.getID())[1];
    }
    if (pbParent instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pbParent;
        replaceProgramBlock(ipb.getChildBlocksIfBody(), pbOld, pbNew);
        replaceProgramBlock(ipb.getChildBlocksElseBody(), pbOld, pbNew);
    } else if (pbParent instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pbParent;
        replaceProgramBlock(wpb.getChildBlocks(), pbOld, pbNew);
    } else if (pbParent instanceof ForProgramBlock || pbParent instanceof ParForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pbParent;
        replaceProgramBlock(fpb.getChildBlocks(), pbOld, pbNew);
    } else if (pbParent instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pbParent;
        replaceProgramBlock(fpb.getChildBlocks(), pbOld, pbNew);
    } else
        throw new DMLRuntimeException("Optimizer doesn't support " + pbParent.getClass().getName());
    //update repository
    if (rtMap)
        _rtMap.replaceMapping(pbNew, n);
    else
        _hlMap.replaceMapping(pbNew, n);
}