Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 1 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class InterProceduralAnalysis method populateLocalVariableMapForFunctionCall.

private void populateLocalVariableMapForFunctionCall(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callvars, LocalVariableMap vars, Set<Long> inputSafeNNZ, Integer numCalls) throws HopsException {
    ArrayList<DataIdentifier> inputVars = fstmt.getInputParams();
    ArrayList<Hop> inputOps = fop.getInput();
    for (int i = 0; i < inputVars.size(); i++) {
        //create mapping between input hops and vars
        DataIdentifier dat = inputVars.get(i);
        Hop input = inputOps.get(i);
        if (input.getDataType() == DataType.MATRIX) {
            //propagate matrix characteristics
            MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
            MatrixCharacteristics mc = new MatrixCharacteristics(input.getDim1(), input.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), inputSafeNNZ.contains(input.getHopID()) ? input.getNnz() : -1);
            MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, null, null);
            mo.setMetaData(meta);
            vars.put(dat.getName(), mo);
        } else if (input.getDataType() == DataType.SCALAR) {
            //(for multiple calls, literal equivalence already checked)
            if (input instanceof LiteralOp) {
                vars.put(dat.getName(), ScalarObjectFactory.createScalarObject(input.getValueType(), (LiteralOp) input));
            } else //and input scalar is existing variable in symbol table
            if (PROPAGATE_SCALAR_VARS_INTO_FUN && numCalls != null && numCalls == 1 && input instanceof DataOp) {
                Data scalar = callvars.get(input.getName());
                if (scalar != null && scalar instanceof ScalarObject) {
                    vars.put(dat.getName(), scalar);
                }
            }
        }
    }
}

Example 2 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class GDFEnumOptimizer method enumNodePlans.

private static PlanSet enumNodePlans(GDFNode node, MemoStructure memo, double maxCosts) throws DMLRuntimeException {
    ArrayList<Plan> plans = new ArrayList<Plan>();
    ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    // CASE 1: core hop enumeration (other than persistent/transient read/write) 
    if (node.getNodeType() == NodeType.HOP_NODE && !(node.getHop() instanceof DataOp)) {
        //core rewrite enumeration for cp and mr
        enumHopNodePlans(node, plans);
    } else //CASE 2: dataop hop enumeration 
    if (node.getHop() instanceof DataOp) {
        DataOp dhop = (DataOp) node.getHop();
        if (dhop.getDataOpType() == DataOpTypes.PERSISTENTREAD) {
            //for persistent read the interesting properties are fixed by the input
            //but we can decide on output properties
            ExecType et = (dhop.getMemEstimate() > OptimizerUtils.getLocalMemBudget() || HopRewriteUtils.alwaysRequiresReblock(dhop)) ? CLUSTER : ExecType.CP;
            int[] blocksizes = (et == CLUSTER) ? BLOCK_SIZES : new int[] { BLOCK_SIZES[0] };
            for (Integer bs : blocksizes) {
                RewriteConfig rcmr = new RewriteConfig(et, bs, FileFormatTypes.BINARY);
                InterestingProperties ipsmr = rcmr.deriveInterestingProperties();
                Plan mrplan = new Plan(node, ipsmr, rcmr, null);
                plans.add(mrplan);
            }
        } else if (dhop.getDataOpType() == DataOpTypes.PERSISTENTWRITE) {
            //for persistent write the interesting properties are fixed by the given
            //write specification
            ExecType et = (dhop.getMemEstimate() > OptimizerUtils.getLocalMemBudget()) ? CLUSTER : ExecType.CP;
            RewriteConfig rcmr = new RewriteConfig(et, (int) dhop.getRowsInBlock(), dhop.getInputFormatType());
            InterestingProperties ipsmr = rcmr.deriveInterestingProperties();
            Plan mrplan = new Plan(node, ipsmr, rcmr, null);
            plans.add(mrplan);
        } else if (dhop.getDataOpType() == DataOpTypes.TRANSIENTREAD || dhop.getDataOpType() == DataOpTypes.TRANSIENTWRITE) {
            //note: full enumeration for transient read and write; otherwise the properties
            //of these hops are never set because pass-through plans refer to different hops
            enumHopNodePlans(node, plans);
        }
    } else //ENUMERATE LOOP PLANS
    if (node.getNodeType() == NodeType.LOOP_NODE) {
        //TODO consistency checks inputs and outputs (updated vars)
        GDFLoopNode lnode = (GDFLoopNode) node;
        //no additional pruning (validity, optimality) required
        for (GDFNode in : lnode.getLoopInputs().values()) enumOpt(in, memo, maxCosts);
        //step 1: enumerate loop plan, incl partitioning/checkpoints/reblock for inputs
        RewriteConfig rc = new RewriteConfig(ExecType.CP, -1, null);
        InterestingProperties ips = rc.deriveInterestingProperties();
        Plan lplan = new Plan(node, ips, rc, null);
        plans.add(lplan);
        //(predicate might be null if single variable)
        if (lnode.getLoopPredicate() != null)
            enumOpt(lnode.getLoopPredicate(), memo, maxCosts);
        //step 3: recursive call optimize on outputs
        //(return union of all output plans, later selected by output var)
        PlanSet Pout = new PlanSet();
        for (GDFNode out : lnode.getLoopOutputs().values()) Pout = Pout.union(enumOpt(out, memo, maxCosts));
        plans.addAll(Pout.getPlans());
    //note: global pruning later done when returning to enumOpt
    //for the entire loop node			
    } else //CREATE DUMMY CROSSBLOCK PLAN
    if (node.getNodeType() == NodeType.CROSS_BLOCK_NODE) {
    //do nothing (leads to pass-through on crossProductChild)
    }
    return new PlanSet(plans);
}

Example 3 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class TemplateCell method rConstructCplan.

protected void rConstructCplan(Hop hop, CPlanMemoTable memo, HashMap<Long, CNode> tmp, HashSet<Hop> inHops, boolean compileLiterals) {
    // memoization for common subexpression elimination and to avoid redundant work
    if (tmp.containsKey(hop.getHopID()))
        return;
    MemoTableEntry me = memo.getBest(hop.getHopID(), TemplateType.CELL);
    // recursively process required childs
    if (me != null && me.type.isIn(TemplateType.ROW, TemplateType.OUTER)) {
        CNodeData cdata = TemplateUtils.createCNodeData(hop, compileLiterals);
        tmp.put(hop.getHopID(), cdata);
        inHops.add(hop);
        return;
    }
    for (int i = 0; i < hop.getInput().size(); i++) {
        Hop c = hop.getInput().get(i);
        if (me != null && me.isPlanRef(i) && !(c instanceof DataOp) && (me.type != TemplateType.MAGG || memo.contains(c.getHopID(), TemplateType.CELL)))
            rConstructCplan(c, memo, tmp, inHops, compileLiterals);
        else if (me != null && (me.type == TemplateType.MAGG || me.type == TemplateType.CELL) && HopRewriteUtils.isMatrixMultiply(hop) && // skip transpose
        i == 0)
            rConstructCplan(c.getInput().get(0), memo, tmp, inHops, compileLiterals);
        else {
            CNodeData cdata = TemplateUtils.createCNodeData(c, compileLiterals);
            tmp.put(c.getHopID(), cdata);
            inHops.add(c);
        }
    }
    // construct cnode for current hop
    CNode out = null;
    if (hop instanceof UnaryOp) {
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        String primitiveOpName = ((UnaryOp) hop).getOp().name();
        out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName));
    } else if (hop instanceof BinaryOp) {
        BinaryOp bop = (BinaryOp) hop;
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
        String primitiveOpName = bop.getOp().name();
        // add lookups if required
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        cdata2 = TemplateUtils.wrapLookupIfNecessary(cdata2, hop.getInput().get(1));
        // construct binary cnode
        out = new CNodeBinary(cdata1, cdata2, BinType.valueOf(primitiveOpName));
    } else if (hop instanceof TernaryOp) {
        TernaryOp top = (TernaryOp) hop;
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
        CNode cdata3 = tmp.get(hop.getInput().get(2).getHopID());
        // add lookups if required
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        cdata2 = TemplateUtils.wrapLookupIfNecessary(cdata2, hop.getInput().get(1));
        cdata3 = TemplateUtils.wrapLookupIfNecessary(cdata3, hop.getInput().get(2));
        // construct ternary cnode, primitive operation derived from OpOp3
        out = new CNodeTernary(cdata1, cdata2, cdata3, TernaryType.valueOf(top.getOp().name()));
    } else if (hop instanceof ParameterizedBuiltinOp) {
        CNode cdata1 = tmp.get(((ParameterizedBuiltinOp) hop).getTargetHop().getHopID());
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        CNode cdata2 = tmp.get(((ParameterizedBuiltinOp) hop).getParameterHop("pattern").getHopID());
        CNode cdata3 = tmp.get(((ParameterizedBuiltinOp) hop).getParameterHop("replacement").getHopID());
        TernaryType ttype = (cdata2.isLiteral() && cdata2.getVarname().equals("Double.NaN")) ? TernaryType.REPLACE_NAN : TernaryType.REPLACE;
        out = new CNodeTernary(cdata1, cdata2, cdata3, ttype);
    } else if (hop instanceof IndexingOp) {
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        out = new CNodeTernary(cdata1, TemplateUtils.createCNodeData(new LiteralOp(hop.getInput().get(0).getDim2()), true), TemplateUtils.createCNodeData(hop.getInput().get(4), true), TernaryType.LOOKUP_RC1);
    } else if (HopRewriteUtils.isTransposeOperation(hop)) {
        out = TemplateUtils.skipTranspose(tmp.get(hop.getHopID()), hop, tmp, compileLiterals);
        // correct indexing types of existing lookups
        if (!HopRewriteUtils.containsOp(hop.getParent(), AggBinaryOp.class))
            TemplateUtils.rFlipVectorLookups(out);
        // maintain input hops
        if (out instanceof CNodeData && !inHops.contains(hop.getInput().get(0)))
            inHops.add(hop.getInput().get(0));
    } else if (hop instanceof AggUnaryOp) {
        // aggregation handled in template implementation (note: we do not compile
        // ^2 of SUM_SQ into the operator to simplify the detection of single operators)
        out = tmp.get(hop.getInput().get(0).getHopID());
    } else if (hop instanceof AggBinaryOp) {
        // (1) t(X)%*%X -> sum(X^2) and t(X) %*% Y -> sum(X*Y)
        if (HopRewriteUtils.isTransposeOfItself(hop.getInput().get(0), hop.getInput().get(1))) {
            CNode cdata1 = tmp.get(hop.getInput().get(1).getHopID());
            if (TemplateUtils.isColVector(cdata1))
                cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
            out = new CNodeUnary(cdata1, UnaryType.POW2);
        } else {
            CNode cdata1 = TemplateUtils.skipTranspose(tmp.get(hop.getInput().get(0).getHopID()), hop.getInput().get(0), tmp, compileLiterals);
            if (cdata1 instanceof CNodeData && !inHops.contains(hop.getInput().get(0).getInput().get(0)))
                inHops.add(hop.getInput().get(0).getInput().get(0));
            if (TemplateUtils.isColVector(cdata1))
                cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
            CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
            if (TemplateUtils.isColVector(cdata2))
                cdata2 = new CNodeUnary(cdata2, UnaryType.LOOKUP_R);
            out = new CNodeBinary(cdata1, cdata2, BinType.MULT);
        }
    }
    tmp.put(hop.getHopID(), out);
}

Example 4 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class IPAPassInlineFunctions method rCountOperators.

private static int rCountOperators(Hop current) {
    if (current.isVisited())
        return 0;
    int count = !(current instanceof DataOp || current instanceof LiteralOp) ? 1 : 0;
    for (Hop c : current.getInput()) count += rCountOperators(c);
    current.setVisited();
    return count;
}

Example 5 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class IPAPassRemoveConstantBinaryOps method rRemoveConstantBinaryOp.

private static void rRemoveConstantBinaryOp(Hop hop, HashMap<String, Hop> mOnes) {
    if (hop.isVisited())
        return;
    if (hop instanceof BinaryOp && ((BinaryOp) hop).getOp() == OpOp2.MULT && !((BinaryOp) hop).isOuterVectorOperator() && hop.getInput().get(0).getDataType() == DataType.MATRIX && hop.getInput().get(1) instanceof DataOp && mOnes.containsKey(hop.getInput().get(1).getName())) {
        // replace matrix of ones with literal 1 (later on removed by
        // algebraic simplification rewrites; otherwise more complex
        // recursive processing of childs and rewiring required)
        HopRewriteUtils.removeChildReferenceByPos(hop, hop.getInput().get(1), 1);
        HopRewriteUtils.addChildReference(hop, new LiteralOp(1), 1);
    }
    // recursively process child nodes
    for (Hop c : hop.getInput()) rRemoveConstantBinaryOp(c, mOnes);
    hop.setVisited();
}