Examples with ParForProgramBlock org.apache.sysml.runtime.controlprogram.ParForProgramBlock used on opensource projects

Example 81 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class OptimizerRuleBased method rGetAllParForPBs.

protected HashSet<ParForProgramBlock> rGetAllParForPBs(OptNode n, HashSet<ParForProgramBlock> pbs) {
    // collect parfor
    if (n.getNodeType() == NodeType.PARFOR) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
        pbs.add(pfpb);
    }
    // recursive invocation
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) rGetAllParForPBs(c, pbs);
    return pbs;
}

Example 82 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project incubator-systemml by apache.

the class Explain method countCompiledInstructions.

/**
 * Recursively counts the number of compiled MRJob instructions in the
 * given runtime program block.
 *
 * @param pb program block
 * @param counts explain countst
 * @param MR if true, count Hadoop instructions
 * @param CP if true, count CP instructions
 * @param SP if true, count Spark instructions
 */
private static void countCompiledInstructions(ProgramBlock pb, ExplainCounts counts, boolean MR, boolean CP, boolean SP) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        countCompiledInstructions(tmp.getPredicate(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        countCompiledInstructions(tmp.getPredicate(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) countCompiledInstructions(pb2, counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        countCompiledInstructions(tmp.getFromInstructions(), counts, MR, CP, SP);
        countCompiledInstructions(tmp.getToInstructions(), counts, MR, CP, SP);
        countCompiledInstructions(tmp.getIncrementInstructions(), counts, MR, CP, SP);
        for (ProgramBlock pb2 : tmp.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    // additional parfor jobs counted during runtime
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        for (ProgramBlock pb2 : fpb.getChildBlocks()) countCompiledInstructions(pb2, counts, MR, CP, SP);
    } else {
        countCompiledInstructions(pb.getInstructions(), counts, MR, CP, SP);
    }
}

Example 83 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptimizerRuleBased method rewriteSetFusedDataPartitioningExecution.

// /////
// REWRITE set fused data partitioning / execution
// /
/**
 * This dedicated execution mode can only be applied if all of the
 * following conditions are true:
 * - Only cp instructions in the parfor body
 * - Only one partitioned input
 * - number of iterations is equal to number of partitions (nrow/ncol)
 * - partitioned matrix access via plain iteration variables (no composed expressions)
 *   (this ensures that each partition is exactly read once)
 * - no left indexing (since by default static task partitioning)
 *
 * Furthermore, it should be only chosen if we already decided for remote partitioning
 * and otherwise would create a large number of partition files.
 *
 * NOTE: We already respect the reducer memory budget for plan correctness. However,
 * we miss optimization potential if the reducer budget is larger than the mapper budget
 * (if we were not able to select REMOTE_MR as execution strategy wrt mapper budget)
 * TODO modify 'set exec strategy' and related rewrites for conditional data partitioning.
 *
 * @param pn internal representation of a plan alternative for program blocks and instructions
 * @param M ?
 * @param flagLIX ?
 * @param partitionedMatrices map of data partition formats
 * @param vars local variable map
 */
protected void rewriteSetFusedDataPartitioningExecution(OptNode pn, double M, boolean flagLIX, HashMap<String, PartitionFormat> partitionedMatrices, LocalVariableMap vars) {
    // assertions (warnings of corrupt optimizer decisions)
    if (pn.getNodeType() != NodeType.PARFOR)
        LOG.warn(getOptMode() + " OPT: Fused data partitioning and execution is only applicable for a ParFor node.");
    boolean apply = false;
    String partitioner = pn.getParam(ParamType.DATA_PARTITIONER);
    PDataPartitioner REMOTE_DP = OptimizerUtils.isSparkExecutionMode() ? PDataPartitioner.REMOTE_SPARK : PDataPartitioner.REMOTE_MR;
    PExecMode REMOTE_DPE = OptimizerUtils.isSparkExecutionMode() ? PExecMode.REMOTE_SPARK_DP : PExecMode.REMOTE_MR_DP;
    // try to merge MR data partitioning and MR exec
    if ((// fits into remote memory of reducers
    pn.getExecType() == ExecType.MR && M < _rm2 || // MR/SP EXEC and CP body
    pn.getExecType() == ExecType.SPARK) && partitioner != null && // MR/SP partitioning
    partitioner.equals(REMOTE_DP.toString()) && // only one partitioned matrix
    partitionedMatrices.size() == 1) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
        // partitioned matrix
        String moVarname = partitionedMatrices.keySet().iterator().next();
        PartitionFormat moDpf = partitionedMatrices.get(moVarname);
        MatrixObject mo = (MatrixObject) vars.get(moVarname);
        if (rIsAccessByIterationVariable(pn, moVarname, pfpb.getIterVar()) && ((moDpf == PartitionFormat.ROW_WISE && mo.getNumRows() == _N) || (moDpf == PartitionFormat.COLUMN_WISE && mo.getNumColumns() == _N) || (moDpf._dpf == PDataPartitionFormat.ROW_BLOCK_WISE_N && mo.getNumRows() <= _N * moDpf._N) || (moDpf._dpf == PDataPartitionFormat.COLUMN_BLOCK_WISE_N && mo.getNumColumns() <= _N * moDpf._N))) {
            int k = (int) Math.min(_N, _rk2);
            pn.addParam(ParamType.DATA_PARTITIONER, REMOTE_DPE.toString() + "(fused)");
            pn.setK(k);
            // set fused exec type
            pfpb.setExecMode(REMOTE_DPE);
            pfpb.setDataPartitioner(PDataPartitioner.NONE);
            pfpb.enableColocatedPartitionedMatrix(moVarname);
            pfpb.setDegreeOfParallelism(k);
            apply = true;
        }
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'set fused data partitioning and execution' - result=" + apply);
}

Example 84 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptimizerRuleBased method rewriteSetOperationsExecType.

// /////
// REWRITE set operations exec type
// /
protected void rewriteSetOperationsExecType(OptNode pn, boolean recompile) {
    // set exec type in internal opt tree
    int count = setOperationExecType(pn, ExecType.CP);
    // recompile program (actual programblock modification)
    if (recompile && count <= 0)
        LOG.warn("OPT: Forced set operations exec type 'CP', but no operation requires recompile.");
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
    HashSet<String> fnStack = new HashSet<>();
    Recompiler.recompileProgramBlockHierarchy2Forced(pfpb.getChildBlocks(), 0, fnStack, LopProperties.ExecType.CP);
    // debug output
    LOG.debug(getOptMode() + " OPT: rewrite 'set operation exec type CP' - result=" + count);
}

Example 85 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptimizerRuleBased method rewriteInjectSparkLoopCheckpointing.

// /////
// REWRITE inject spark loop checkpointing
// /
protected void rewriteInjectSparkLoopCheckpointing(OptNode n) {
    // get program blocks of root parfor
    Object[] progobj = OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID());
    ParForStatementBlock pfsb = (ParForStatementBlock) progobj[0];
    ParForStatement fs = (ParForStatement) pfsb.getStatement(0);
    ParForProgramBlock pfpb = (ParForProgramBlock) progobj[1];
    boolean applied = false;
    try {
        // apply hop rewrite inject spark checkpoints (but without context awareness)
        RewriteInjectSparkLoopCheckpointing rewrite = new RewriteInjectSparkLoopCheckpointing(false);
        ProgramRewriter rewriter = new ProgramRewriter(rewrite);
        ProgramRewriteStatus state = new ProgramRewriteStatus();
        rewriter.rRewriteStatementBlockHopDAGs(pfsb, state);
        fs.setBody(rewriter.rRewriteStatementBlocks(fs.getBody(), state, true));
        // recompile if additional checkpoints introduced
        if (state.getInjectedCheckpoints()) {
            pfpb.setChildBlocks(ProgramRecompiler.generatePartitialRuntimeProgram(pfpb.getProgram(), fs.getBody()));
            applied = true;
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'inject spark loop checkpointing' - result=" + applied);
}