Examples with ParForStatementBlock org.apache.sysml.parser.ParForStatementBlock used on opensource projects

Example 36 with ParForStatementBlock

use of org.apache.sysml.parser.ParForStatementBlock in project systemml by apache.

the class OptimizerRuleBased method rewriteInjectSparkRepartition.

// /////
// REWRITE inject spark repartition for zipmm
// /
protected void rewriteInjectSparkRepartition(OptNode n, LocalVariableMap vars) {
    // get program blocks of root parfor
    Object[] progobj = OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID());
    ParForStatementBlock pfsb = (ParForStatementBlock) progobj[0];
    ParForProgramBlock pfpb = (ParForProgramBlock) progobj[1];
    ArrayList<String> ret = new ArrayList<>();
    if (// spark exec mode
    OptimizerUtils.isSparkExecutionMode() && // local parfor
    n.getExecType() == ExecType.CP && // at least 2 iterations
    _N > 1) {
        // collect candidates from zipmm spark instructions
        HashSet<String> cand = new HashSet<>();
        rCollectZipmmPartitioningCandidates(n, cand);
        // prune updated candidates
        HashSet<String> probe = new HashSet<>(pfsb.getReadOnlyParentVars());
        for (String var : cand) if (probe.contains(var))
            ret.add(var);
        // prune small candidates
        ArrayList<String> tmp = new ArrayList<>(ret);
        ret.clear();
        for (String var : tmp) if (vars.get(var) instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) vars.get(var);
            double sp = OptimizerUtils.getSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getNnz());
            double size = OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), sp);
            if (size > OptimizerUtils.getLocalMemBudget())
                ret.add(var);
        }
        // apply rewrite to parfor pb
        if (!ret.isEmpty()) {
            pfpb.setSparkRepartitionVariables(ret);
        }
    }
    _numEvaluatedPlans++;
    LOG.debug(getOptMode() + " OPT: rewrite 'inject spark input repartition' - result=" + ret.size() + " (" + ProgramConverter.serializeStringCollection(ret) + ")");
}

Example 37 with ParForStatementBlock

use of org.apache.sysml.parser.ParForStatementBlock in project systemml by apache.

the class Explain method getHopDAG.

private static StringBuilder getHopDAG(StatementBlock sb, StringBuilder nodes, ArrayList<Integer> lines, boolean withSubgraph) {
    StringBuilder builder = new StringBuilder();
    if (sb instanceof WhileStatementBlock) {
        addSubGraphHeader(builder, withSubgraph);
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        String label = null;
        if (!wsb.getUpdateInPlaceVars().isEmpty())
            label = "WHILE (lines " + wsb.getBeginLine() + "-" + wsb.getEndLine() + ") in-place=" + wsb.getUpdateInPlaceVars().toString() + "";
        else
            label = "WHILE (lines " + wsb.getBeginLine() + "-" + wsb.getEndLine() + ")";
        // TODO: Don't show predicate hops for now
        // builder.append(explainHop(wsb.getPredicateHops()));
        WhileStatement ws = (WhileStatement) sb.getStatement(0);
        for (StatementBlock current : ws.getBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
        addSubGraphFooter(builder, withSubgraph, label);
    } else if (sb instanceof IfStatementBlock) {
        addSubGraphHeader(builder, withSubgraph);
        IfStatementBlock ifsb = (IfStatementBlock) sb;
        String label = "IF (lines " + ifsb.getBeginLine() + "-" + ifsb.getEndLine() + ")";
        // TODO: Don't show predicate hops for now
        // builder.append(explainHop(ifsb.getPredicateHops(), level+1));
        IfStatement ifs = (IfStatement) sb.getStatement(0);
        for (StatementBlock current : ifs.getIfBody()) {
            builder.append(getHopDAG(current, nodes, lines, withSubgraph));
            addSubGraphFooter(builder, withSubgraph, label);
        }
        if (!ifs.getElseBody().isEmpty()) {
            addSubGraphHeader(builder, withSubgraph);
            label = "ELSE (lines " + ifsb.getBeginLine() + "-" + ifsb.getEndLine() + ")";
            for (StatementBlock current : ifs.getElseBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
            addSubGraphFooter(builder, withSubgraph, label);
        }
    } else if (sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        addSubGraphHeader(builder, withSubgraph);
        String label = "";
        if (sb instanceof ParForStatementBlock) {
            if (!fsb.getUpdateInPlaceVars().isEmpty())
                label = "PARFOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ") in-place=" + fsb.getUpdateInPlaceVars().toString() + "";
            else
                label = "PARFOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ")";
        } else {
            if (!fsb.getUpdateInPlaceVars().isEmpty())
                label = "FOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ") in-place=" + fsb.getUpdateInPlaceVars().toString() + "";
            else
                label = "FOR (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ")";
        }
        // TODO: Don't show predicate hops for now
        // if (fsb.getFromHops() != null)
        // builder.append(explainHop(fsb.getFromHops(), level+1));
        // if (fsb.getToHops() != null)
        // builder.append(explainHop(fsb.getToHops(), level+1));
        // if (fsb.getIncrementHops() != null)
        // builder.append(explainHop(fsb.getIncrementHops(), level+1));
        ForStatement fs = (ForStatement) sb.getStatement(0);
        for (StatementBlock current : fs.getBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
        addSubGraphFooter(builder, withSubgraph, label);
    } else if (sb instanceof FunctionStatementBlock) {
        FunctionStatement fsb = (FunctionStatement) sb.getStatement(0);
        addSubGraphHeader(builder, withSubgraph);
        String label = "Function (lines " + fsb.getBeginLine() + "-" + fsb.getEndLine() + ")";
        for (StatementBlock current : fsb.getBody()) builder.append(getHopDAG(current, nodes, lines, withSubgraph));
        addSubGraphFooter(builder, withSubgraph, label);
    } else {
        // For generic StatementBlock
        if (sb.requiresRecompilation()) {
            addSubGraphHeader(builder, withSubgraph);
        }
        ArrayList<Hop> hopsDAG = sb.getHops();
        if (hopsDAG != null && !hopsDAG.isEmpty()) {
            Hop.resetVisitStatus(hopsDAG);
            for (Hop hop : hopsDAG) builder.append(getHopDAG(hop, nodes, lines, withSubgraph));
            Hop.resetVisitStatus(hopsDAG);
        }
        if (sb.requiresRecompilation()) {
            builder.append("style=filled;\n");
            builder.append("color=lightgrey;\n");
            String label = "(lines " + sb.getBeginLine() + "-" + sb.getEndLine() + ") [recompile=" + sb.requiresRecompilation() + "]";
            addSubGraphFooter(builder, withSubgraph, label);
        }
    }
    return builder;
}

Example 38 with ParForStatementBlock

use of org.apache.sysml.parser.ParForStatementBlock in project systemml by apache.

the class ParForProgramBlock method getMinMemory.

private long getMinMemory(ExecutionContext ec) {
    long ret = -1;
    // if forced remote exec and single node
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE && _execMode == PExecMode.REMOTE_MR && _optMode == POptMode.NONE) {
        try {
            ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
            OptTree tree = OptTreeConverter.createAbstractOptTree(-1, -1, sb, this, new HashSet<String>(), ec);
            CostEstimator est = new CostEstimatorHops(OptTreeConverter.getAbstractPlanMapping());
            double mem = est.getEstimate(TestMeasure.MEMORY_USAGE, tree.getRoot());
            ret = (long) (mem * (1d / OptimizerUtils.MEM_UTIL_FACTOR));
        } catch (Exception e) {
            LOG.error("Failed to analyze minmum memory requirements.", e);
        }
    }
    return ret;
}

Example 39 with ParForStatementBlock

use of org.apache.sysml.parser.ParForStatementBlock in project systemml by apache.

the class ParForProgramBlock method exportMatricesToHDFS.

private void exportMatricesToHDFS(ExecutionContext ec, String... blacklistNames) {
    ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
    Set<String> blacklist = UtilFunctions.asSet(blacklistNames);
    if (LIVEVAR_AWARE_EXPORT && sb != null) {
        // optimization to prevent unnecessary export of matrices
        // export only variables that are read in the body
        VariableSet varsRead = sb.variablesRead();
        for (String key : ec.getVariables().keySet()) {
            if (varsRead.containsVariable(key) && !blacklist.contains(key)) {
                Data d = ec.getVariable(key);
                if (d.getDataType() == DataType.MATRIX)
                    ((MatrixObject) d).exportData(_replicationExport);
            }
        }
    } else {
        // export all matrices in symbol table
        for (String key : ec.getVariables().keySet()) {
            if (!blacklist.contains(key)) {
                Data d = ec.getVariable(key);
                if (d.getDataType() == DataType.MATRIX)
                    ((MatrixObject) d).exportData(_replicationExport);
            }
        }
    }
}