Examples with Hop org.apache.sysml.hops.Hop used on opensource projects

Example 16 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class PlanSelectionFuseCostBased method rCheckMultiAggregate.

private static boolean rCheckMultiAggregate(Hop current, HashSet<Long> probe) {
    boolean ret = true;
    for (Hop c : current.getInput()) ret &= rCheckMultiAggregate(c, probe);
    ret &= !probe.contains(current.getHopID());
    return ret;
}

Example 17 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class CPlanMemoTable method pruneRedundant.

public void pruneRedundant(long hopID) {
    if (!contains(hopID))
        return;
    //prune redundant plans (i.e., equivalent) 
    setDistinct(hopID, _plans.get(hopID));
    //prune dominated plans (e.g., opened plan subsumed
    //by fused plan if single consumer of input)
    HashSet<MemoTableEntry> rmList = new HashSet<MemoTableEntry>();
    List<MemoTableEntry> list = _plans.get(hopID);
    Hop hop = _hopRefs.get(hopID);
    for (MemoTableEntry e1 : list) for (MemoTableEntry e2 : list) if (e1 != e2 && e1.subsumes(e2)) {
        //check that childs don't have multiple consumers
        boolean rmSafe = true;
        for (int i = 0; i <= 2; i++) rmSafe &= (e1.isPlanRef(i) && !e2.isPlanRef(i)) ? hop.getInput().get(i).getParent().size() == 1 : true;
        if (rmSafe)
            rmList.add(e2);
    }
    //update current entry list, by removing rmList
    remove(hop, rmList);
}

Example 18 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class TemplateUtils method rfindChildren.

private static void rfindChildren(Hop hop, HashSet<Hop> children) {
    if (hop instanceof UnaryOp || (hop instanceof BinaryOp && hop.getInput().get(0).getDataType() == DataType.MATRIX && TemplateUtils.isVectorOrScalar(hop.getInput().get(1))) || //unary operation or binary operaiton with one matrix and a scalar
    (hop instanceof BinaryOp && TemplateUtils.isVectorOrScalar(hop.getInput().get(0)) && hop.getInput().get(1).getDataType() == DataType.MATRIX) && hop.getDataType() == DataType.MATRIX) {
        if (!children.contains(hop))
            children.add(hop);
        Hop matrix = TemplateUtils.isMatrix(hop.getInput().get(0)) ? hop.getInput().get(0) : hop.getInput().get(1);
        rfindChildren(matrix, children);
    } else
        children.add(hop);
}

Example 19 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class TemplateUtils method findCommonChild.

private static Hop findCommonChild(Hop hop1, Hop hop2) {
    //this method assumes that each two nodes have at most one common child 
    LinkedHashSet<Hop> children1 = new LinkedHashSet<Hop>();
    LinkedHashSet<Hop> children2 = new LinkedHashSet<Hop>();
    rfindChildren(hop1, children1);
    rfindChildren(hop2, children2);
    //iterate on one set and find the first common child in the other set
    Iterator<Hop> iter = children1.iterator();
    while (iter.hasNext()) {
        Hop candidate = iter.next();
        if (children2.contains(candidate))
            return candidate;
    }
    return null;
}

Example 20 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class OptimizerRuleBased method isResultPartitionable.

protected boolean isResultPartitionable(OptNode n, ArrayList<String> resultVars, LocalVariableMap vars, String iterVarname) throws DMLRuntimeException {
    boolean ret = true;
    //check left indexing operator
    String opStr = n.getParam(ParamType.OPSTRING);
    if (opStr == null || !opStr.equals(LeftIndexingOp.OPSTRING))
        ret = false;
    Hop h = null;
    Hop base = null;
    if (ret) {
        h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
        base = h.getInput().get(0);
        //check result variable
        if (!resultVars.contains(base.getName()))
            ret = false;
    }
    //check access pattern, memory budget
    if (ret) {
        int dpf = 0;
        Hop inpRowL = h.getInput().get(2);
        Hop inpRowU = h.getInput().get(3);
        Hop inpColL = h.getInput().get(4);
        Hop inpColU = h.getInput().get(5);
        if ((inpRowL.getName().equals(iterVarname) && inpRowU.getName().equals(iterVarname)))
            //rowwise
            dpf = 1;
        if ((inpColL.getName().equals(iterVarname) && inpColU.getName().equals(iterVarname)))
            //colwise or cellwise
            dpf = (dpf == 0) ? 2 : 3;
        if (dpf == 0)
            ret = false;
        else {
            //check memory budget
            MatrixObject mo = (MatrixObject) vars.get(base.getName());
            if (//-1 valid because result var known during opt
            mo.getNnz() != 0)
                ret = false;
            //Note: for memory estimation the common case is sparse since remote_mr and individual tasks;
            //and in the dense case, we would not benefit from result partitioning
            boolean sparse = MatrixBlock.evalSparseFormatInMemory(base.getDim1(), base.getDim2(), base.getDim1());
            if (sparse) {
                //custom memory estimatation in order to account for structural properties
                //e.g., for rowwise we know that we only pay one sparserow overhead per task
                double memSparseBlock = estimateSizeSparseRowBlock(base.getDim1());
                double memSparseRow1 = estimateSizeSparseRow(base.getDim2(), base.getDim2());
                double memSparseRowMin = estimateSizeSparseRowMin(base.getDim2());
                double memTask1 = -1;
                int taskN = -1;
                switch(dpf) {
                    case //rowwise
                    1:
                        //sparse block and one sparse row per task
                        memTask1 = memSparseBlock + memSparseRow1;
                        taskN = (int) ((_rm - memSparseBlock) / memSparseRow1);
                        break;
                    case //colwise
                    2:
                        //sparse block, sparse row per row but shared over tasks
                        memTask1 = memSparseBlock + memSparseRowMin * base.getDim1();
                        taskN = estimateNumTasksSparseCol(_rm - memSparseBlock, base.getDim1());
                        break;
                    case //cellwise
                    3:
                        //sparse block and one minimal sparse row per task
                        memTask1 = memSparseBlock + memSparseRowMin;
                        taskN = (int) ((_rm - memSparseBlock) / memSparseRowMin);
                        break;
                }
                if (memTask1 > _rm || memTask1 < 0)
                    ret = false;
                else
                    n.addParam(ParamType.TASK_SIZE, String.valueOf(taskN));
            } else {
                //dense (no result partitioning possible)
                ret = false;
            }
        }
    }
    return ret;
}