Examples with ParForProgramBlock org.apache.sysml.runtime.controlprogram.ParForProgramBlock used on opensource projects

Example 66 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptimizerConstrained method rewriteSetFusedDataPartitioningExecution.

// /////
// REWRITE set fused data partitioning / execution
// /
protected void rewriteSetFusedDataPartitioningExecution(OptNode pn, double M, boolean flagLIX, HashMap<String, PartitionFormat> partitionedMatrices, LocalVariableMap vars, PExecMode emode) {
    if (emode == PExecMode.REMOTE_MR_DP || emode == PExecMode.REMOTE_SPARK_DP) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
        // partitioned matrix
        if (partitionedMatrices.size() <= 0) {
            LOG.debug(getOptMode() + " OPT: unable to force 'set fused data partitioning and execution' - result=" + false);
            return;
        }
        String moVarname = partitionedMatrices.keySet().iterator().next();
        PartitionFormat moDpf = partitionedMatrices.get(moVarname);
        MatrixObject mo = (MatrixObject) vars.get(moVarname);
        if (rIsAccessByIterationVariable(pn, moVarname, pfpb.getIterVar()) && ((moDpf == PartitionFormat.ROW_WISE && mo.getNumRows() == _N) || (moDpf == PartitionFormat.COLUMN_WISE && mo.getNumColumns() == _N) || (moDpf._dpf == PDataPartitionFormat.ROW_BLOCK_WISE_N && mo.getNumRows() <= _N * moDpf._N) || (moDpf._dpf == PDataPartitionFormat.COLUMN_BLOCK_WISE_N && mo.getNumColumns() <= _N * moDpf._N))) {
            int k = (int) Math.min(_N, _rk2);
            if (emode == PExecMode.REMOTE_MR_DP) {
                pn.addParam(ParamType.DATA_PARTITIONER, "REMOTE_MR(fused)");
                // set fused exec type
                pfpb.setExecMode(PExecMode.REMOTE_MR_DP);
            } else {
                pn.addParam(ParamType.DATA_PARTITIONER, "REMOTE_SPARK(fused)");
                // set fused exec type
                pfpb.setExecMode(PExecMode.REMOTE_SPARK_DP);
            }
            pn.setK(k);
            pfpb.setDataPartitioner(PDataPartitioner.NONE);
            pfpb.enableColocatedPartitionedMatrix(moVarname);
            pfpb.setDegreeOfParallelism(k);
        }
        LOG.debug(getOptMode() + " OPT: force 'set fused data partitioning and execution' - result=" + true);
    } else
        super.rewriteSetFusedDataPartitioningExecution(pn, M, flagLIX, partitionedMatrices, vars);
}

Example 67 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptimizerConstrained method rewriteSetExecutionStategy.

// /////
// REWRITE set execution strategy
// /
@Override
protected boolean rewriteSetExecutionStategy(OptNode n, double M0, double M, double M2, double M3, boolean flagLIX) {
    boolean ret = false;
    // constraint awareness
    if (n.getExecType() != null && ConfigurationManager.isParallelParFor()) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
        PExecMode mode = PExecMode.LOCAL;
        if (n.getExecType() == ExecType.MR) {
            mode = PExecMode.REMOTE_MR;
        } else if (n.getExecType() == ExecType.SPARK) {
            mode = PExecMode.REMOTE_SPARK;
        }
        pfpb.setExecMode(mode);
        LOG.debug(getOptMode() + " OPT: forced 'set execution strategy' - result=" + mode);
    } else
        ret = super.rewriteSetExecutionStategy(n, M0, M, M2, M3, flagLIX);
    return ret;
}

Example 68 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteInjectSparkLoopCheckpointing.

// /////
// REWRITE inject spark loop checkpointing
// /
protected void rewriteInjectSparkLoopCheckpointing(OptNode n) {
    // get program blocks of root parfor
    Object[] progobj = OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID());
    ParForStatementBlock pfsb = (ParForStatementBlock) progobj[0];
    ParForStatement fs = (ParForStatement) pfsb.getStatement(0);
    ParForProgramBlock pfpb = (ParForProgramBlock) progobj[1];
    boolean applied = false;
    try {
        // apply hop rewrite inject spark checkpoints (but without context awareness)
        RewriteInjectSparkLoopCheckpointing rewrite = new RewriteInjectSparkLoopCheckpointing(false);
        ProgramRewriter rewriter = new ProgramRewriter(rewrite);
        ProgramRewriteStatus state = new ProgramRewriteStatus();
        rewriter.rRewriteStatementBlockHopDAGs(pfsb, state);
        fs.setBody(rewriter.rRewriteStatementBlocks(fs.getBody(), state, true));
        // recompile if additional checkpoints introduced
        if (state.getInjectedCheckpoints()) {
            pfpb.setChildBlocks(ProgramRecompiler.generatePartitialRuntimeProgram(pfpb.getProgram(), fs.getBody()));
            applied = true;
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'inject spark loop checkpointing' - result=" + applied);
}

Example 69 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteInjectSparkRepartition.

// /////
// REWRITE inject spark repartition for zipmm
// /
protected void rewriteInjectSparkRepartition(OptNode n, LocalVariableMap vars) {
    // get program blocks of root parfor
    Object[] progobj = OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID());
    ParForStatementBlock pfsb = (ParForStatementBlock) progobj[0];
    ParForProgramBlock pfpb = (ParForProgramBlock) progobj[1];
    ArrayList<String> ret = new ArrayList<>();
    if (// spark exec mode
    OptimizerUtils.isSparkExecutionMode() && // local parfor
    n.getExecType() == ExecType.CP && // at least 2 iterations
    _N > 1) {
        // collect candidates from zipmm spark instructions
        HashSet<String> cand = new HashSet<>();
        rCollectZipmmPartitioningCandidates(n, cand);
        // prune updated candidates
        HashSet<String> probe = new HashSet<>(pfsb.getReadOnlyParentVars());
        for (String var : cand) if (probe.contains(var))
            ret.add(var);
        // prune small candidates
        ArrayList<String> tmp = new ArrayList<>(ret);
        ret.clear();
        for (String var : tmp) if (vars.get(var) instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) vars.get(var);
            double sp = OptimizerUtils.getSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getNnz());
            double size = OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), sp);
            if (size > OptimizerUtils.getLocalMemBudget())
                ret.add(var);
        }
        // apply rewrite to parfor pb
        if (!ret.isEmpty()) {
            pfpb.setSparkRepartitionVariables(ret);
        }
    }
    _numEvaluatedPlans++;
    LOG.debug(getOptMode() + " OPT: rewrite 'inject spark input repartition' - result=" + ret.size() + " (" + ProgramConverter.serializeStringCollection(ret) + ")");
}

Example 70 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteDisableCPCaching.

// /////
// REWRITE disable CP caching
// /
protected void rewriteDisableCPCaching(OptNode pn, HashSet<ResultVar> inplaceResultVars, LocalVariableMap vars) {
    // assertions (warnings of corrupt optimizer decisions)
    if (pn.getNodeType() != NodeType.PARFOR)
        LOG.warn(getOptMode() + " OPT: Disable caching is only applicable for a ParFor node.");
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
    double M_sumInterm = rComputeSumMemoryIntermediates(pn, inplaceResultVars);
    boolean apply = false;
    if ((pfpb.getExecMode() == PExecMode.REMOTE_MR_DP || pfpb.getExecMode() == PExecMode.REMOTE_MR) && // all intermediates and operations fit into memory budget
    M_sumInterm < _rm) {
        // default is true
        pfpb.setCPCaching(false);
        apply = true;
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'disable CP caching' - result=" + apply + " (M=" + toMB(M_sumInterm) + ")");
}