Examples with FunctionStatementBlock org.apache.sysml.parser.FunctionStatementBlock used on opensource projects

Example 26 with FunctionStatementBlock

use of org.apache.sysml.parser.FunctionStatementBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rFindAndUnfoldRecursiveFunction.

protected void rFindAndUnfoldRecursiveFunction(OptNode n, ParForProgramBlock parfor, HashSet<ParForProgramBlock> recPBs, LocalVariableMap vars) {
    // unfold if found
    if (n.getNodeType() == NodeType.FUNCCALL && n.isRecursive()) {
        boolean exists = rContainsNode(n, parfor);
        if (exists) {
            String fnameKey = n.getParam(ParamType.OPSTRING);
            String[] names = fnameKey.split(Program.KEY_DELIM);
            String fnamespace = names[0];
            String fname = names[1];
            String fnameNew = FUNCTION_UNFOLD_NAMEPREFIX + fname;
            // unfold function
            FunctionOp fop = (FunctionOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
            Program prog = parfor.getProgram();
            DMLProgram dmlprog = parfor.getStatementBlock().getDMLProg();
            FunctionProgramBlock fpb = prog.getFunctionProgramBlock(fnamespace, fname);
            FunctionProgramBlock copyfpb = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<String>(), new HashSet<String>());
            prog.addFunctionProgramBlock(fnamespace, fnameNew, copyfpb);
            dmlprog.addFunctionStatementBlock(fnamespace, fnameNew, (FunctionStatementBlock) copyfpb.getStatementBlock());
            // replace function names in old subtree (link to new function)
            rReplaceFunctionNames(n, fname, fnameNew);
            // recreate sub opttree
            String fnameNewKey = fnamespace + Program.KEY_DELIM + fnameNew;
            OptNode nNew = new OptNode(NodeType.FUNCCALL);
            OptTreeConverter.getAbstractPlanMapping().putHopMapping(fop, nNew);
            nNew.setExecType(ExecType.CP);
            nNew.addParam(ParamType.OPSTRING, fnameNewKey);
            long parentID = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
            OptTreeConverter.getAbstractPlanMapping().getOptNode(parentID).exchangeChild(n, nNew);
            HashSet<String> memo = new HashSet<>();
            // required if functionop not shared (because not replaced yet)
            memo.add(fnameKey);
            // requied if functionop shared (indirectly replaced)
            memo.add(fnameNewKey);
            for (int i = 0; i < copyfpb.getChildBlocks().size(); /*&& i<len*/
            i++) {
                ProgramBlock lpb = copyfpb.getChildBlocks().get(i);
                StatementBlock lsb = lpb.getStatementBlock();
                nNew.addChild(OptTreeConverter.rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
            // compute delta for recPB set (use for removing parfor)
            recPBs.removeAll(rGetAllParForPBs(n, new HashSet<ParForProgramBlock>()));
            recPBs.addAll(rGetAllParForPBs(nNew, new HashSet<ParForProgramBlock>()));
            // replace function names in new subtree (recursive link to new function)
            rReplaceFunctionNames(nNew, fname, fnameNew);
        }
        return;
    }
    // recursive invocation (only for non-recursive functions)
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) rFindAndUnfoldRecursiveFunction(c, parfor, recPBs, vars);
}

Example 27 with FunctionStatementBlock

use of org.apache.sysml.parser.FunctionStatementBlock in project incubator-systemml by apache.

the class Explain method explain.

public static String explain(DMLProgram prog) {
    StringBuilder sb = new StringBuilder();
    // create header
    sb.append("\nPROGRAM\n");
    // Explain functions (if exists)
    if (prog.hasFunctionStatementBlocks()) {
        sb.append("--FUNCTIONS\n");
        // show function call graph
        sb.append("----FUNCTION CALL GRAPH\n");
        sb.append("------MAIN PROGRAM\n");
        FunctionCallGraph fgraph = new FunctionCallGraph(prog);
        sb.append(explainFunctionCallGraph(fgraph, new HashSet<String>(), null, 3));
        // show individual functions
        for (String namespace : prog.getNamespaces().keySet()) {
            for (String fname : prog.getFunctionStatementBlocks(namespace).keySet()) {
                FunctionStatementBlock fsb = prog.getFunctionStatementBlock(namespace, fname);
                FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
                String fkey = DMLProgram.constructFunctionKey(namespace, fname);
                if (fstmt instanceof ExternalFunctionStatement)
                    sb.append("----EXTERNAL FUNCTION " + fkey + "\n");
                else {
                    sb.append("----FUNCTION " + fkey + " [recompile=" + fsb.isRecompileOnce() + "]\n");
                    for (StatementBlock current : fstmt.getBody()) sb.append(explainStatementBlock(current, 3));
                }
            }
        }
    }
    // Explain main program
    sb.append("--MAIN PROGRAM\n");
    for (StatementBlock sblk : prog.getStatementBlocks()) sb.append(explainStatementBlock(sblk, 2));
    return sb.toString();
}

Example 28 with FunctionStatementBlock

use of org.apache.sysml.parser.FunctionStatementBlock in project incubator-systemml by apache.

the class Explain method getHopDAG.

private static StringBuilder getHopDAG(StatementBlock sb, StringBuilder nodes, ArrayList<Integer> lines, boolean withSubgraph) {
    StringBuilder builder = new StringBuilder();
    if (sb instanceof WhileStatementBlock) {
        addSubGraphHeader(builder, withSubgraph);
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        String label = null;
        if (!wsb.getUpdateInPlaceVars().isEmpty())
            label = "WHILE (lines " + wsb.getBeginLine() + "-" + wsb.getEndLine() + ") in-place=" + wsb.getUpdateInPlaceVars().toString() + "";
        else
            label = "WHILE (lines " + wsb.getBeginLine() + "-" + wsb.getEndLine() + ")";
        // TODO: Don't show predicate hops for now
        // builder.append(explainHop(wsb.getPredicateHops()));
        WhileStatement ws = (WhileStatement) sb.getStatement(0);
        for (StatementBlock current : ws.getBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
        addSubGraphFooter(builder, withSubgraph, label);
    } else if (sb instanceof IfStatementBlock) {
        addSubGraphHeader(builder, withSubgraph);
        IfStatementBlock ifsb = (IfStatementBlock) sb;
        String label = "IF (lines " + ifsb.getBeginLine() + "-" + ifsb.getEndLine() + ")";
        // TODO: Don't show predicate hops for now
        // builder.append(explainHop(ifsb.getPredicateHops(), level+1));
        IfStatement ifs = (IfStatement) sb.getStatement(0);
        for (StatementBlock current : ifs.getIfBody()) {
            builder.append(getHopDAG(current, nodes, lines, withSubgraph));
            addSubGraphFooter(builder, withSubgraph, label);
        }
        if (!ifs.getElseBody().isEmpty()) {
            addSubGraphHeader(builder, withSubgraph);
            label = "ELSE (lines " + ifsb.getBeginLine() + "-" + ifsb.getEndLine() + ")";
            for (StatementBlock current : ifs.getElseBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
            addSubGraphFooter(builder, withSubgraph, label);
        }
    } else if (sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        addSubGraphHeader(builder, withSubgraph);
        String label = "";
        if (sb instanceof ParForStatementBlock) {
            if (!fsb.getUpdateInPlaceVars().isEmpty())
                label = "PARFOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ") in-place=" + fsb.getUpdateInPlaceVars().toString() + "";
            else
                label = "PARFOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ")";
        } else {
            if (!fsb.getUpdateInPlaceVars().isEmpty())
                label = "FOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ") in-place=" + fsb.getUpdateInPlaceVars().toString() + "";
            else
                label = "FOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ")";
        }
        // TODO: Don't show predicate hops for now
        // if (fsb.getFromHops() != null)
        // builder.append(explainHop(fsb.getFromHops(), level+1));
        // if (fsb.getToHops() != null)
        // builder.append(explainHop(fsb.getToHops(), level+1));
        // if (fsb.getIncrementHops() != null)
        // builder.append(explainHop(fsb.getIncrementHops(), level+1));
        ForStatement fs = (ForStatement) sb.getStatement(0);
        for (StatementBlock current : fs.getBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
        addSubGraphFooter(builder, withSubgraph, label);
    } else if (sb instanceof FunctionStatementBlock) {
        FunctionStatement fsb = (FunctionStatement) sb.getStatement(0);
        addSubGraphHeader(builder, withSubgraph);
        String label = "Function (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ")";
        for (StatementBlock current : fsb.getBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
        addSubGraphFooter(builder, withSubgraph, label);
    } else {
        // For generic StatementBlock
        if (sb.requiresRecompilation()) {
            addSubGraphHeader(builder, withSubgraph);
        }
        ArrayList<Hop> hopsDAG = sb.getHops();
        if (hopsDAG != null && !hopsDAG.isEmpty()) {
            Hop.resetVisitStatus(hopsDAG);
            for (Hop hop : hopsDAG) builder.append(getHopDAG(hop, nodes, lines, withSubgraph));
            Hop.resetVisitStatus(hopsDAG);
        }
        if (sb.requiresRecompilation()) {
            builder.append("style=filled;\n");
            builder.append("color=lightgrey;\n");
            String label = "(lines " + sb.getBeginLine() + "-" + sb.getEndLine() + ") [recompile=" + sb.requiresRecompilation() + "]";
            addSubGraphFooter(builder, withSubgraph, label);
        }
    }
    return builder;
}

Example 29 with FunctionStatementBlock

use of org.apache.sysml.parser.FunctionStatementBlock in project incubator-systemml by apache.

the class GenerateClassesForMLContext method addFunctionMethods.

/**
 * Add methods to a derived script class to allow invocation of script
 * functions.
 *
 * @param scriptFilePath
 *            the path to a script file
 * @param ctNewScript
 *            the javassist compile-time class representation of a script
 */
public static void addFunctionMethods(String scriptFilePath, CtClass ctNewScript) {
    try {
        DMLProgram dmlProgram = dmlProgramFromScriptFilePath(scriptFilePath);
        if (dmlProgram == null) {
            System.out.println("Could not generate DML Program for: " + scriptFilePath);
            return;
        }
        Map<String, FunctionStatementBlock> defaultNsFsbsMap = dmlProgram.getFunctionStatementBlocks(DMLProgram.DEFAULT_NAMESPACE);
        List<FunctionStatementBlock> fsbs = new ArrayList<>();
        fsbs.addAll(defaultNsFsbsMap.values());
        for (FunctionStatementBlock fsb : fsbs) {
            ArrayList<Statement> sts = fsb.getStatements();
            for (Statement st : sts) {
                if (!(st instanceof FunctionStatement)) {
                    continue;
                }
                FunctionStatement fs = (FunctionStatement) st;
                String dmlFunctionCall = generateDmlFunctionCall(scriptFilePath, fs);
                String functionCallMethod = generateFunctionCallMethod(scriptFilePath, fs, dmlFunctionCall);
                CtMethod m = CtNewMethod.make(functionCallMethod, ctNewScript);
                ctNewScript.addMethod(m);
                addDescriptionFunctionCallMethod(fs, scriptFilePath, ctNewScript, false);
                addDescriptionFunctionCallMethod(fs, scriptFilePath, ctNewScript, true);
            }
        }
    } catch (LanguageException e) {
        System.out.println("Could not add function methods for " + ctNewScript.getName());
    } catch (CannotCompileException e) {
        System.out.println("Could not add function methods for " + ctNewScript.getName());
    } catch (RuntimeException e) {
        System.out.println("Could not add function methods for " + ctNewScript.getName());
    }
}

Example 30 with FunctionStatementBlock

use of org.apache.sysml.parser.FunctionStatementBlock in project incubator-systemml by apache.

the class InterProceduralAnalysis method propagateStatisticsIntoFunctions.

/**
	 * Propagate statistics from the calling program into a function
	 * block.
	 *
	 * @param prog  The DML program.
	 * @param hop HOP to propagate statistics into.
	 * @param fcand  Function candidates.
	 * @param callVars  Calling program's map of variables eligible for
	 *                     propagation.
	 * @param fcandSafeNNZ  Function candidate safe non-zeros.
	 * @param unaryFcands  Unary function candidates.
	 * @param fnStack  Function stack to determine current scope.
	 * @throws HopsException  If a HopsException occurs.
	 * @throws ParseException  If a ParseException occurs.
	 */
private void propagateStatisticsIntoFunctions(DMLProgram prog, Hop hop, Map<String, Integer> fcand, LocalVariableMap callVars, Map<String, Set<Long>> fcandSafeNNZ, Set<String> unaryFcands, Set<String> fnStack) throws HopsException, ParseException {
    if (hop.isVisited())
        return;
    for (Hop c : hop.getInput()) propagateStatisticsIntoFunctions(prog, c, fcand, callVars, fcandSafeNNZ, unaryFcands, fnStack);
    if (hop instanceof FunctionOp) {
        //maintain counters and investigate functions if not seen so far
        FunctionOp fop = (FunctionOp) hop;
        String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
        if (fop.getFunctionType() == FunctionType.DML) {
            FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
            FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
            if (fcand.containsKey(fkey) && //prevent recursion	
            !fnStack.contains(fkey)) {
                //maintain function call stack
                fnStack.add(fkey);
                //create mapping and populate symbol table for refresh
                LocalVariableMap tmpVars = new LocalVariableMap();
                populateLocalVariableMapForFunctionCall(fstmt, fop, callVars, tmpVars, fcandSafeNNZ.get(fkey), fcand.get(fkey));
                //recursively propagate statistics
                propagateStatisticsAcrossBlock(fsb, fcand, tmpVars, fcandSafeNNZ, unaryFcands, fnStack);
                //extract vars from symbol table, re-map and refresh main program
                extractFunctionCallReturnStatistics(fstmt, fop, tmpVars, callVars, true);
                //maintain function call stack
                fnStack.remove(fkey);
            } else if (unaryFcands.contains(fkey)) {
                extractFunctionCallEquivalentReturnStatistics(fstmt, fop, callVars);
            } else {
                extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
            }
        } else if (fop.getFunctionType() == FunctionType.EXTERNAL_FILE || fop.getFunctionType() == FunctionType.EXTERNAL_MEM) {
            //infer output size for known external functions
            FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
            ExternalFunctionStatement fstmt = (ExternalFunctionStatement) fsb.getStatement(0);
            if (PROPAGATE_KNOWN_UDF_STATISTICS)
                extractExternalFunctionCallReturnStatistics(fstmt, fop, callVars);
            else
                extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
        }
    }
    hop.setVisited();
}