Examples with Hop org.apache.sysml.hops.Hop used on opensource projects

Example 21 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class SpoofCompiler method cleanupCPlans.

/**
 * Cleanup generated cplans in order to remove unnecessary inputs created
 * during incremental construction. This is important as it avoids unnecessary
 * redundant computation.
 *
 * @param memo memoization table
 * @param cplans set of cplans
 */
private static HashMap<Long, Pair<Hop[], CNodeTpl>> cleanupCPlans(CPlanMemoTable memo, HashMap<Long, Pair<Hop[], CNodeTpl>> cplans) {
    HashMap<Long, Pair<Hop[], CNodeTpl>> cplans2 = new HashMap<>();
    CPlanOpRewriter rewriter = new CPlanOpRewriter();
    CPlanCSERewriter cse = new CPlanCSERewriter();
    for (Entry<Long, Pair<Hop[], CNodeTpl>> e : cplans.entrySet()) {
        CNodeTpl tpl = e.getValue().getValue();
        Hop[] inHops = e.getValue().getKey();
        // remove invalid plans with null inputs
        if (Arrays.stream(inHops).anyMatch(h -> (h == null)))
            continue;
        // perform simplifications and cse rewrites
        tpl = rewriter.simplifyCPlan(tpl);
        tpl = cse.eliminateCommonSubexpressions(tpl);
        // update input hops (order-preserving)
        HashSet<Long> inputHopIDs = tpl.getInputHopIDs(false);
        inHops = Arrays.stream(inHops).filter(p -> p != null && inputHopIDs.contains(p.getHopID())).toArray(Hop[]::new);
        cplans2.put(e.getKey(), new Pair<>(inHops, tpl));
        // remove invalid plans with column indexing on main input
        if (tpl instanceof CNodeCell || tpl instanceof CNodeRow) {
            CNodeData in1 = (CNodeData) tpl.getInput().get(0);
            boolean inclRC1 = !(tpl instanceof CNodeRow);
            if (rHasLookupRC1(tpl.getOutput(), in1, inclRC1) || isLookupRC1(tpl.getOutput(), in1, inclRC1)) {
                cplans2.remove(e.getKey());
                if (LOG.isTraceEnabled())
                    LOG.trace("Removed cplan due to invalid rc1 indexing on main input.");
            }
        } else if (tpl instanceof CNodeMultiAgg) {
            CNodeData in1 = (CNodeData) tpl.getInput().get(0);
            for (CNode output : ((CNodeMultiAgg) tpl).getOutputs()) if (rHasLookupRC1(output, in1, true) || isLookupRC1(output, in1, true)) {
                cplans2.remove(e.getKey());
                if (LOG.isTraceEnabled())
                    LOG.trace("Removed cplan due to invalid rc1 indexing on main input.");
            }
        }
        // remove invalid lookups on main input (all templates)
        CNodeData in1 = (CNodeData) tpl.getInput().get(0);
        if (tpl instanceof CNodeMultiAgg)
            rFindAndRemoveLookupMultiAgg((CNodeMultiAgg) tpl, in1);
        else
            rFindAndRemoveLookup(tpl.getOutput(), in1, !(tpl instanceof CNodeRow));
        // remove invalid row templates (e.g., unsatisfied blocksize constraint)
        if (tpl instanceof CNodeRow) {
            // check for invalid row cplan over column vector
            if (((CNodeRow) tpl).getRowType() == RowType.NO_AGG && tpl.getOutput().getDataType().isScalar()) {
                cplans2.remove(e.getKey());
                if (LOG.isTraceEnabled())
                    LOG.trace("Removed invalid row cplan w/o agg on column vector.");
            } else if (OptimizerUtils.isSparkExecutionMode()) {
                Hop hop = memo.getHopRefs().get(e.getKey());
                boolean isSpark = DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || OptimizerUtils.getTotalMemEstimate(inHops, hop, true) > OptimizerUtils.getLocalMemBudget();
                boolean invalidNcol = hop.getDataType().isMatrix() && (HopRewriteUtils.isTransposeOperation(hop) ? hop.getDim1() > hop.getRowsInBlock() : hop.getDim2() > hop.getColsInBlock());
                for (Hop in : inHops) invalidNcol |= (in.getDataType().isMatrix() && in.getDim2() > in.getColsInBlock());
                if (isSpark && invalidNcol) {
                    cplans2.remove(e.getKey());
                    if (LOG.isTraceEnabled())
                        LOG.trace("Removed invalid row cplan w/ ncol>ncolpb.");
                }
            }
        }
        // remove cplan w/ single op and w/o agg
        if ((tpl instanceof CNodeCell && ((CNodeCell) tpl).getCellType() == CellType.NO_AGG && TemplateUtils.hasSingleOperation(tpl)) || (tpl instanceof CNodeRow && (((CNodeRow) tpl).getRowType() == RowType.NO_AGG || ((CNodeRow) tpl).getRowType() == RowType.NO_AGG_B1 || ((CNodeRow) tpl).getRowType() == RowType.ROW_AGG) && TemplateUtils.hasSingleOperation(tpl)) || TemplateUtils.hasNoOperation(tpl)) {
            cplans2.remove(e.getKey());
            if (LOG.isTraceEnabled())
                LOG.trace("Removed cplan with single operation.");
        }
        // remove cplan if empty
        if (tpl.getOutput() instanceof CNodeData) {
            cplans2.remove(e.getKey());
            if (LOG.isTraceEnabled())
                LOG.trace("Removed empty cplan.");
        }
        // rename inputs (for codegen and plan caching)
        tpl.renameInputs();
    }
    return cplans2;
}

Example 22 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class ProgramConverter method createWhileStatementBlockCopy.

public static WhileStatementBlock createWhileStatementBlockCopy(WhileStatementBlock sb, long pid, boolean plain, boolean forceDeepCopy) {
    WhileStatementBlock ret = null;
    try {
        if (ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION) && // forced deep copy for function recompile
        sb != null && (Recompiler.requiresRecompilation(sb.getPredicateHops()) || forceDeepCopy)) {
            // create new statement (shallow copy livein/liveout for recompile, line numbers for explain)
            ret = new WhileStatementBlock();
            ret.setDMLProg(sb.getDMLProg());
            ret.setParseInfo(sb);
            ret.setLiveIn(sb.liveIn());
            ret.setLiveOut(sb.liveOut());
            ret.setUpdatedVariables(sb.variablesUpdated());
            ret.setReadVariables(sb.variablesRead());
            ret.setUpdateInPlaceVars(sb.getUpdateInPlaceVars());
            // shallow copy child statements
            ret.setStatements(sb.getStatements());
            // deep copy predicate hops dag for concurrent recompile
            Hop hops = Recompiler.deepCopyHopsDag(sb.getPredicateHops());
            ret.setPredicateHops(hops);
            ret.updatePredicateRecompilationFlag();
        } else {
            ret = sb;
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    return ret;
}

Example 23 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class CostEstimatorHops method getLeafNodeEstimate.

@Override
public double getLeafNodeEstimate(TestMeasure measure, OptNode node, ExecType et) {
    if (node.getNodeType() != NodeType.HOP)
        // generic optnode but no childs (e.g., PB for rmvar inst)
        return 0;
    if (measure != TestMeasure.MEMORY_USAGE)
        throw new DMLRuntimeException("Testmeasure " + measure + " not supported by cost model " + CostModelType.STATIC_MEM_METRIC + ".");
    // core mem estimation (use hops estimate)
    Hop h = _map.getMappedHop(node.getID());
    double value = h.getMemEstimate();
    if (// MR, null
    et != ExecType.CP)
        value = DEFAULT_MEM_MR;
    if (// no mem estimate
    value <= 0)
        value = CostEstimator.DEFAULT_MEM_ESTIMATE_CP;
    LOG.trace("Memory estimate (forced exec type) " + h.getName() + ", " + h.getOpString() + "(" + node.getExecType() + ")" + "=" + OptimizerRuleBased.toMB(value));
    return value;
}

Example 24 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class OptimizerRuleBased method isCPOnlyPossible.

protected boolean isCPOnlyPossible(OptNode n, double memBudget) {
    ExecType et = n.getExecType();
    boolean ret = (et == ExecType.CP);
    if (n.isLeaf() && et == getRemoteExecType()) {
        Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
        if (// e.g., -exec=hadoop
        h.getForcedExecType() != LopProperties.ExecType.MR && h.getForcedExecType() != LopProperties.ExecType.SPARK && // integer dims
        h.hasValidCPDimsAndSize()) {
            double mem = _cost.getLeafNodeEstimate(TestMeasure.MEMORY_USAGE, n, LopProperties.ExecType.CP);
            if (mem <= memBudget)
                ret = true;
        }
    }
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) {
            // early abort if already false
            if (!ret)
                break;
            ret &= isCPOnlyPossible(c, memBudget);
        }
    return ret;
}

Example 25 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class OptimizerRuleBased method resetPartitionRIXEstimates.

protected void resetPartitionRIXEstimates(HashMap<Hop, Double> estimates) {
    for (Entry<Hop, Double> e : estimates.entrySet()) {
        Hop h = e.getKey();
        double val = e.getValue();
        h.setMemEstimate(val);
    }
}