Examples with FunctionOp org.apache.sysml.hops.FunctionOp used on opensource projects

Example 11 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class DMLTranslator method processMultipleReturnBuiltinFunctionExpression.

/**
 * Construct HOps from parse tree: process BuiltinFunction Expressions in
 * MultiAssignment Statements. For all other builtin function expressions,
 * <code>processBuiltinFunctionExpression()</code> is used.
 *
 * @param source built-in function expression
 * @param targetList list of data identifiers
 * @param hops map of high-level operators
 * @return high-level operator
 */
private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpression source, ArrayList<DataIdentifier> targetList, HashMap<String, Hop> hops) {
    // Construct Hops for all inputs
    ArrayList<Hop> inputs = new ArrayList<>();
    inputs.add(processExpression(source.getFirstExpr(), null, hops));
    if (source.getSecondExpr() != null)
        inputs.add(processExpression(source.getSecondExpr(), null, hops));
    if (source.getThirdExpr() != null)
        inputs.add(processExpression(source.getThirdExpr(), null, hops));
    FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN;
    String nameSpace = DMLProgram.INTERNAL_NAMESPACE;
    // Create an array list to hold the outputs of this lop.
    // Exact list of outputs are added based on opcode.
    ArrayList<Hop> outputs = new ArrayList<>();
    // Construct Hop for current builtin function expression based on its type
    Hop currBuiltinOp = null;
    switch(source.getOpCode()) {
        case QR:
        case LU:
        case EIGEN:
        case SVD:
            // Number of outputs = size of targetList = #of identifiers in source.getOutputs
            String[] outputNames = new String[targetList.size()];
            for (int i = 0; i < targetList.size(); i++) {
                outputNames[i] = ((DataIdentifier) targetList.get(i)).getName();
                Hop output = new DataOp(outputNames[i], DataType.MATRIX, ValueType.DOUBLE, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[i]);
                outputs.add(output);
            }
            // Create the hop for current function call
            FunctionOp fcall = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), inputs, outputNames, outputs);
            currBuiltinOp = fcall;
            break;
        default:
            throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnBuiltinFunctionExpression(): " + source.getOpCode());
    }
    // set properties for created hops based on outputs of source expression
    for (int i = 0; i < source.getOutputs().length; i++) {
        setIdentifierParams(outputs.get(i), source.getOutputs()[i]);
        outputs.get(i).setParseInfo(source);
    }
    currBuiltinOp.setParseInfo(source);
    return currBuiltinOp;
}

Example 12 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class RewriteCompressedReblock method rAnalyzeHopDag.

private static void rAnalyzeHopDag(Hop current, ProbeStatus status) {
    if (current.isVisited())
        return;
    // process children recursively
    for (Hop input : current.getInput()) rAnalyzeHopDag(input, status);
    // handle source persistent read
    if (current.getHopID() == status.startHopID) {
        status.compMtx.add(getTmpName(current));
        status.foundStart = true;
    }
    // a) handle function calls
    if (current instanceof FunctionOp && hasCompressedInput(current, status)) {
        // TODO handle of functions in a more fine-grained manner
        // to cover special cases multiple calls where compressed
        // inputs might occur for different input parameters
        FunctionOp fop = (FunctionOp) current;
        String fkey = fop.getFunctionKey();
        if (!status.procFn.contains(fkey)) {
            // memoization to avoid redundant analysis and recursive calls
            status.procFn.add(fkey);
            // map inputs to function inputs
            FunctionStatementBlock fsb = status.prog.getFunctionStatementBlock(fkey);
            FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
            ProbeStatus status2 = new ProbeStatus(status);
            for (int i = 0; i < fop.getInput().size(); i++) if (status.compMtx.contains(getTmpName(fop.getInput().get(i))))
                status2.compMtx.add(fstmt.getInputParams().get(i).getName());
            // analyze function and merge meta info
            rAnalyzeProgram(fsb, status2);
            status.foundStart |= status2.foundStart;
            status.usedInLoop |= status2.usedInLoop;
            status.condUpdate |= status2.condUpdate;
            status.nonApplicable |= status2.nonApplicable;
            // map function outputs to outputs
            String[] outputs = fop.getOutputVariableNames();
            for (int i = 0; i < outputs.length; i++) if (status2.compMtx.contains(fstmt.getOutputParams().get(i).getName()))
                status.compMtx.add(outputs[i]);
        }
    } else // b) handle transient reads and writes (name mapping)
    if (HopRewriteUtils.isData(current, DataOpTypes.TRANSIENTWRITE) && status.compMtx.contains(getTmpName(current.getInput().get(0))))
        status.compMtx.add(current.getName());
    else if (HopRewriteUtils.isData(current, DataOpTypes.TRANSIENTREAD) && status.compMtx.contains(current.getName()))
        status.compMtx.add(getTmpName(current));
    else // c) handle applicable operations
    if (hasCompressedInput(current, status)) {
        // valid with uncompressed outputs
        boolean compUCOut = (// tsmm
        current instanceof AggBinaryOp && current.getDim2() <= current.getColsInBlock() && ((AggBinaryOp) current).checkTransposeSelf() == MMTSJType.LEFT) || // mvmm
        (current instanceof AggBinaryOp && (current.getDim1() == 1 || current.getDim2() == 1)) || (HopRewriteUtils.isTransposeOperation(current) && current.getParent().size() == 1 && current.getParent().get(0) instanceof AggBinaryOp && (current.getParent().get(0).getDim1() == 1 || current.getParent().get(0).getDim2() == 1)) || HopRewriteUtils.isAggUnaryOp(current, AggOp.SUM, AggOp.SUM_SQ, AggOp.MIN, AggOp.MAX);
        // valid with compressed outputs
        boolean compCOut = HopRewriteUtils.isBinaryMatrixScalarOperation(current) || HopRewriteUtils.isBinary(current, OpOp2.CBIND);
        boolean metaOp = HopRewriteUtils.isUnary(current, OpOp1.NROW, OpOp1.NCOL);
        status.nonApplicable |= !(compUCOut || compCOut || metaOp);
        if (compCOut)
            status.compMtx.add(getTmpName(current));
    }
    current.setVisited();
}

Example 13 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class Recompiler method rUpdateFunctionNames.

public static void rUpdateFunctionNames(Hop hop, long pid) {
    if (hop.isVisited())
        return;
    // update function names
    if (hop instanceof FunctionOp && ((FunctionOp) hop).getFunctionType() != FunctionType.MULTIRETURN_BUILTIN) {
        FunctionOp fop = (FunctionOp) hop;
        fop.setFunctionName(fop.getFunctionName() + ProgramConverter.CP_CHILD_THREAD + pid);
    }
    if (hop.getInput() != null)
        for (Hop c : hop.getInput()) rUpdateFunctionNames(c, pid);
    hop.setVisited();
}

Example 14 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class HopDagValidator method rValidateHop.

private static void rValidateHop(final Hop hop, final ValidatorState state) {
    final long id = hop.getHopID();
    // check visit status
    final boolean seen = !state.seen.add(id);
    if (seen != hop.isVisited()) {
        String parentIDs = hop.getParent().stream().map(h -> Long.toString(h.getHopID())).collect(Collectors.joining(", "));
        check(false, hop, parentIDs, seen);
    }
    // we saw the Hop previously, no need to re-validate
    if (seen)
        return;
    // check parent linking
    for (Hop parent : hop.getParent()) check(parent.getInput().contains(hop), hop, "not properly linked to its parent pid=%d %s", parent.getHopID(), parent.getClass().getName());
    final ArrayList<Hop> input = hop.getInput();
    final Expression.DataType dt = hop.getDataType();
    final Expression.ValueType vt = hop.getValueType();
    // check child linking
    for (Hop child : input) check(child.getParent().contains(hop), hop, "not properly linked to its child cid=%d %s", child.getHopID(), child.getClass().getName());
    // check empty children (other variable-length Hops must have at least one child)
    if (input.isEmpty())
        check(hop instanceof DataOp || hop instanceof FunctionOp || hop instanceof LiteralOp, hop, "is not a dataop/functionop/literal but has no children");
    // check Hop has a legal arity (number of children)
    hop.checkArity();
    // check Matrix data type Hops must have Double Value type
    if (dt == Expression.DataType.MATRIX)
        check(vt == Expression.ValueType.DOUBLE || vt == Expression.ValueType.INT, hop, "has Matrix type but Value Type %s is not DOUBLE", vt);
    // recursively process children
    for (Hop child : input) rValidateHop(child, state);
    hop.setVisited();
}

Example 15 with FunctionOp

use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNodes.

public static ArrayList<OptNode> rCreateAbstractOptNodes(Hop hop, LocalVariableMap vars, Set<String> memo) {
    ArrayList<OptNode> ret = new ArrayList<>();
    ArrayList<Hop> in = hop.getInput();
    if (hop.isVisited())
        return ret;
    // general case
    if (!(hop instanceof DataOp || hop instanceof LiteralOp || hop instanceof FunctionOp)) {
        OptNode node = new OptNode(NodeType.HOP);
        String opstr = hop.getOpString();
        node.addParam(ParamType.OPSTRING, opstr);
        // handle execution type
        LopProperties.ExecType et = (hop.getExecType() != null) ? hop.getExecType() : LopProperties.ExecType.CP;
        switch(et) {
            case CP:
            case GPU:
                node.setExecType(ExecType.CP);
                break;
            case SPARK:
                node.setExecType(ExecType.SPARK);
                break;
            case MR:
                node.setExecType(ExecType.MR);
                break;
            default:
                throw new DMLRuntimeException("Unsupported optnode exec type: " + et);
        }
        // handle degree of parallelism
        if (et == LopProperties.ExecType.CP && hop instanceof MultiThreadedHop) {
            MultiThreadedHop mtop = (MultiThreadedHop) hop;
            node.setK(OptimizerUtils.getConstrainedNumThreads(mtop.getMaxNumThreads()));
        }
        // assign node to return
        _hlMap.putHopMapping(hop, node);
        ret.add(node);
    } else // process function calls
    if (hop instanceof FunctionOp && INCLUDE_FUNCTIONS) {
        FunctionOp fhop = (FunctionOp) hop;
        String fname = fhop.getFunctionName();
        String fnspace = fhop.getFunctionNamespace();
        String fKey = fhop.getFunctionKey();
        Object[] prog = _hlMap.getRootProgram();
        OptNode node = new OptNode(NodeType.FUNCCALL);
        _hlMap.putHopMapping(fhop, node);
        node.setExecType(ExecType.CP);
        node.addParam(ParamType.OPSTRING, fKey);
        if (!fnspace.equals(DMLProgram.INTERNAL_NAMESPACE)) {
            FunctionProgramBlock fpb = ((Program) prog[1]).getFunctionProgramBlock(fnspace, fname);
            FunctionStatementBlock fsb = ((DMLProgram) prog[0]).getFunctionStatementBlock(fnspace, fname);
            FunctionStatement fs = (FunctionStatement) fsb.getStatement(0);
            // process body; NOTE: memo prevents inclusion of functions multiple times
            if (!memo.contains(fKey)) {
                memo.add(fKey);
                int len = fs.getBody().size();
                for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
                    ProgramBlock lpb = fpb.getChildBlocks().get(i);
                    StatementBlock lsb = fs.getBody().get(i);
                    node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
                }
                memo.remove(fKey);
            } else
                node.addParam(ParamType.RECURSIVE_CALL, "true");
        }
        ret.add(node);
    }
    if (in != null)
        for (Hop hin : in) if (// no need for opt nodes
        !(hin instanceof DataOp || hin instanceof LiteralOp))
            ret.addAll(rCreateAbstractOptNodes(hin, vars, memo));
    hop.setVisited();
    return ret;
}