Examples with PResultMerge org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PResultMerge used on opensource projects

Example 1 with PResultMerge

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PResultMerge in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteSetResultMerge.

// /////
// REWRITE set result merge
// /
protected void rewriteSetResultMerge(OptNode n, LocalVariableMap vars, boolean inLocal) {
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
    PResultMerge REMOTE = OptimizerUtils.isSparkExecutionMode() ? PResultMerge.REMOTE_SPARK : PResultMerge.REMOTE_MR;
    PResultMerge ret = null;
    // investigate details of current parfor node
    boolean flagRemoteParFOR = (n.getExecType() == getRemoteExecType());
    boolean flagLargeResult = hasLargeTotalResults(n, pfpb.getResultVariables(), vars, true);
    boolean flagRemoteLeftIndexing = hasResultMRLeftIndexing(n, pfpb.getResultVariables(), vars, true);
    boolean flagCellFormatWoCompare = determineFlagCellFormatWoCompare(pfpb.getResultVariables(), vars);
    boolean flagOnlyInMemResults = hasOnlyInMemoryResults(n, pfpb.getResultVariables(), vars, true);
    // MR, if remote exec, and w/compare (prevent huge transfer/merge costs)
    if (flagRemoteParFOR && flagLargeResult) {
        ret = REMOTE;
    } else // CP, if all results in mem
    if (flagOnlyInMemResults) {
        ret = PResultMerge.LOCAL_MEM;
    } else // benefit for large matrices outweigths potentially unnecessary MR jobs for smaller matrices)
    if ((flagRemoteParFOR || flagRemoteLeftIndexing) && !(flagCellFormatWoCompare && ResultMergeLocalFile.ALLOW_COPY_CELLFILES)) {
        ret = REMOTE;
    } else // CP, otherwise (decide later if in mem or file-based)
    {
        ret = PResultMerge.LOCAL_AUTOMATIC;
    }
    // modify rtprog
    pfpb.setResultMerge(ret);
    // modify plan
    n.addParam(ParamType.RESULT_MERGE, ret.toString());
    // recursively apply rewrite for parfor nodes
    if (n.getChilds() != null)
        rInvokeSetResultMerge(n.getChilds(), vars, inLocal && !flagRemoteParFOR);
    _numEvaluatedPlans++;
    LOG.debug(getOptMode() + " OPT: rewrite 'set result merge' - result=" + ret);
}

Example 2 with PResultMerge

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PResultMerge in project systemml by apache.

the class OptimizerRuleBased method rewriteSetResultMerge.

// /////
// REWRITE set result merge
// /
protected void rewriteSetResultMerge(OptNode n, LocalVariableMap vars, boolean inLocal) {
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
    PResultMerge REMOTE = OptimizerUtils.isSparkExecutionMode() ? PResultMerge.REMOTE_SPARK : PResultMerge.REMOTE_MR;
    PResultMerge ret = null;
    // investigate details of current parfor node
    boolean flagRemoteParFOR = (n.getExecType() == getRemoteExecType());
    boolean flagLargeResult = hasLargeTotalResults(n, pfpb.getResultVariables(), vars, true);
    boolean flagRemoteLeftIndexing = hasResultMRLeftIndexing(n, pfpb.getResultVariables(), vars, true);
    boolean flagCellFormatWoCompare = determineFlagCellFormatWoCompare(pfpb.getResultVariables(), vars);
    boolean flagOnlyInMemResults = hasOnlyInMemoryResults(n, pfpb.getResultVariables(), vars, true);
    // MR, if remote exec, and w/compare (prevent huge transfer/merge costs)
    if (flagRemoteParFOR && flagLargeResult) {
        ret = REMOTE;
    } else // CP, if all results in mem
    if (flagOnlyInMemResults) {
        ret = PResultMerge.LOCAL_MEM;
    } else // benefit for large matrices outweigths potentially unnecessary MR jobs for smaller matrices)
    if ((flagRemoteParFOR || flagRemoteLeftIndexing) && !(flagCellFormatWoCompare && ResultMergeLocalFile.ALLOW_COPY_CELLFILES)) {
        ret = REMOTE;
    } else // CP, otherwise (decide later if in mem or file-based)
    {
        ret = PResultMerge.LOCAL_AUTOMATIC;
    }
    // modify rtprog
    pfpb.setResultMerge(ret);
    // modify plan
    n.addParam(ParamType.RESULT_MERGE, ret.toString());
    // recursively apply rewrite for parfor nodes
    if (n.getChilds() != null)
        rInvokeSetResultMerge(n.getChilds(), vars, inLocal && !flagRemoteParFOR);
    _numEvaluatedPlans++;
    LOG.debug(getOptMode() + " OPT: rewrite 'set result merge' - result=" + ret);
}