Examples with FunctionOp org.apache.sysml.hops.FunctionOp used on opensource projects

Example 21 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class InterProceduralAnalysis method propagateStatisticsIntoFunctions.

/**
	 * Propagate statistics from the calling program into a function
	 * block.
	 *
	 * @param prog  The DML program.
	 * @param hop HOP to propagate statistics into.
	 * @param fcand  Function candidates.
	 * @param callVars  Calling program's map of variables eligible for
	 *                     propagation.
	 * @param fcandSafeNNZ  Function candidate safe non-zeros.
	 * @param unaryFcands  Unary function candidates.
	 * @param fnStack  Function stack to determine current scope.
	 * @throws HopsException  If a HopsException occurs.
	 * @throws ParseException  If a ParseException occurs.
	 */
private void propagateStatisticsIntoFunctions(DMLProgram prog, Hop hop, Map<String, Integer> fcand, LocalVariableMap callVars, Map<String, Set<Long>> fcandSafeNNZ, Set<String> unaryFcands, Set<String> fnStack) throws HopsException, ParseException {
    if (hop.isVisited())
        return;
    for (Hop c : hop.getInput()) propagateStatisticsIntoFunctions(prog, c, fcand, callVars, fcandSafeNNZ, unaryFcands, fnStack);
    if (hop instanceof FunctionOp) {
        //maintain counters and investigate functions if not seen so far
        FunctionOp fop = (FunctionOp) hop;
        String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
        if (fop.getFunctionType() == FunctionType.DML) {
            FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
            FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
            if (fcand.containsKey(fkey) && //prevent recursion	
            !fnStack.contains(fkey)) {
                //maintain function call stack
                fnStack.add(fkey);
                //create mapping and populate symbol table for refresh
                LocalVariableMap tmpVars = new LocalVariableMap();
                populateLocalVariableMapForFunctionCall(fstmt, fop, callVars, tmpVars, fcandSafeNNZ.get(fkey), fcand.get(fkey));
                //recursively propagate statistics
                propagateStatisticsAcrossBlock(fsb, fcand, tmpVars, fcandSafeNNZ, unaryFcands, fnStack);
                //extract vars from symbol table, re-map and refresh main program
                extractFunctionCallReturnStatistics(fstmt, fop, tmpVars, callVars, true);
                //maintain function call stack
                fnStack.remove(fkey);
            } else if (unaryFcands.contains(fkey)) {
                extractFunctionCallEquivalentReturnStatistics(fstmt, fop, callVars);
            } else {
                extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
            }
        } else if (fop.getFunctionType() == FunctionType.EXTERNAL_FILE || fop.getFunctionType() == FunctionType.EXTERNAL_MEM) {
            //infer output size for known external functions
            FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
            ExternalFunctionStatement fstmt = (ExternalFunctionStatement) fsb.getStatement(0);
            if (PROPAGATE_KNOWN_UDF_STATISTICS)
                extractExternalFunctionCallReturnStatistics(fstmt, fop, callVars);
            else
                extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
        }
    }
    hop.setVisited();
}

Example 22 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class InterProceduralAnalysis method analyzeProgram.

/**
	 * Public interface to perform IPA over a given DML program.
	 * 
	 * @param dmlp the dml program
	 * @throws HopsException if HopsException occurs
	 * @throws ParseException if ParseException occurs
	 * @throws LanguageException if LanguageException occurs
	 */
public void analyzeProgram(DMLProgram dmlp) throws HopsException, ParseException, LanguageException {
    FunctionCallGraph fgraph = new FunctionCallGraph(dmlp);
    //step 1: get candidates for statistics propagation into functions (if required)
    Map<String, Integer> fcandCounts = new HashMap<String, Integer>();
    Map<String, FunctionOp> fcandHops = new HashMap<String, FunctionOp>();
    Map<String, Set<Long>> fcandSafeNNZ = new HashMap<String, Set<Long>>();
    if (!dmlp.getFunctionStatementBlocks().isEmpty()) {
        for (//get candidates (over entire program)
        StatementBlock sb : //get candidates (over entire program)
        dmlp.getStatementBlocks()) getFunctionCandidatesForStatisticPropagation(sb, fcandCounts, fcandHops);
        pruneFunctionCandidatesForStatisticPropagation(fcandCounts, fcandHops);
        determineFunctionCandidatesNNZPropagation(fcandHops, fcandSafeNNZ);
        DMLTranslator.resetHopsDAGVisitStatus(dmlp);
    }
    //step 2: get unary dimension-preserving non-candidate functions
    Collection<String> unaryFcandTmp = fgraph.getReachableFunctions(fcandCounts.keySet());
    HashSet<String> unaryFcands = new HashSet<String>();
    if (!unaryFcandTmp.isEmpty() && UNARY_DIMS_PRESERVING_FUNS) {
        for (String tmp : unaryFcandTmp) if (isUnarySizePreservingFunction(dmlp.getFunctionStatementBlock(tmp)))
            unaryFcands.add(tmp);
    }
    //step 3: propagate statistics and scalars into functions and across DAGs
    if (!fcandCounts.isEmpty() || INTRA_PROCEDURAL_ANALYSIS) {
        //(callVars used to chain outputs/inputs of multiple functions calls) 
        LocalVariableMap callVars = new LocalVariableMap();
        for (//propagate stats into candidates
        StatementBlock sb : //propagate stats into candidates
        dmlp.getStatementBlocks()) propagateStatisticsAcrossBlock(sb, fcandCounts, callVars, fcandSafeNNZ, unaryFcands, new HashSet<String>());
    }
    //step 4: remove unused functions (e.g., inlined or never called)
    if (REMOVE_UNUSED_FUNCTIONS) {
        removeUnusedFunctions(dmlp, fgraph);
    }
    //step 5: flag functions with loops for 'recompile-on-entry'
    if (FLAG_FUNCTION_RECOMPILE_ONCE) {
        flagFunctionsForRecompileOnce(dmlp, fgraph);
    }
    //step 6: set global data flow properties
    if (REMOVE_UNNECESSARY_CHECKPOINTS && OptimizerUtils.isSparkExecutionMode()) {
        //remove unnecessary checkpoint before update 
        removeCheckpointBeforeUpdate(dmlp);
        //move necessary checkpoint after update
        moveCheckpointAfterUpdate(dmlp);
        //remove unnecessary checkpoint read-{write|uagg}
        removeCheckpointReadWrite(dmlp);
    }
    //step 7: remove constant binary ops
    if (REMOVE_CONSTANT_BINARY_OPS) {
        removeConstantBinaryOps(dmlp);
    }
}

Example 23 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class InterProceduralAnalysis method getFunctionCandidatesForStatisticPropagation.

private void getFunctionCandidatesForStatisticPropagation(DMLProgram prog, Hop hop, Map<String, Integer> fcandCounts, Map<String, FunctionOp> fcandHops) throws HopsException, ParseException {
    if (hop.isVisited())
        return;
    if (hop instanceof FunctionOp && !((FunctionOp) hop).getFunctionNamespace().equals(DMLProgram.INTERNAL_NAMESPACE)) {
        //maintain counters and investigate functions if not seen so far
        FunctionOp fop = (FunctionOp) hop;
        String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
        if (fcandCounts.containsKey(fkey)) {
            if (ALLOW_MULTIPLE_FUNCTION_CALLS) {
                //compare input matrix characteristics for both function calls
                //(if unknown or difference: maintain counter - this function is no candidate)
                boolean consistent = true;
                FunctionOp efop = fcandHops.get(fkey);
                int numInputs = efop.getInput().size();
                for (int i = 0; i < numInputs; i++) {
                    Hop h1 = efop.getInput().get(i);
                    Hop h2 = fop.getInput().get(i);
                    //check matrix and scalar sizes (if known dims, nnz known/unknown, 
                    // safeness of nnz propagation, determined later per input)
                    consistent &= (h1.dimsKnown() && h2.dimsKnown() && h1.getDim1() == h2.getDim1() && h1.getDim2() == h2.getDim2() && h1.getNnz() == h2.getNnz());
                    //check literal values (equi value)
                    if (h1 instanceof LiteralOp) {
                        consistent &= (h2 instanceof LiteralOp && HopRewriteUtils.isEqualValue((LiteralOp) h1, (LiteralOp) h2));
                    }
                }
                if (//if differences, do not propagate
                !consistent)
                    fcandCounts.put(fkey, fcandCounts.get(fkey) + 1);
            } else {
                //maintain counter (this function is no candidate)
                fcandCounts.put(fkey, fcandCounts.get(fkey) + 1);
            }
        } else {
            //first appearance
            //create a new count entry
            fcandCounts.put(fkey, 1);
            //keep the function call hop
            fcandHops.put(fkey, fop);
            FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
            getFunctionCandidatesForStatisticPropagation(fsb, fcandCounts, fcandHops);
        }
    }
    for (Hop c : hop.getInput()) getFunctionCandidatesForStatisticPropagation(prog, c, fcandCounts, fcandHops);
    hop.setVisited();
}