Examples with ParForProgramBlock org.apache.sysml.runtime.controlprogram.ParForProgramBlock used on opensource projects

Example 96 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptimizerRuleBased method rGetAllParForPBs.

protected HashSet<ParForProgramBlock> rGetAllParForPBs(OptNode n, HashSet<ParForProgramBlock> pbs) {
    // collect parfor
    if (n.getNodeType() == NodeType.PARFOR) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
        pbs.add(pfpb);
    }
    // recursive invocation
    if (!n.isLeaf())
        for (OptNode c : n.getChilds()) rGetAllParForPBs(c, pbs);
    return pbs;
}

Example 97 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptTreeConverter method rContainsMRJobInstruction.

public static boolean rContainsMRJobInstruction(ProgramBlock pb, boolean inclFunctions) {
    boolean ret = false;
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getPredicate(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getPredicate(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
        for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        ret = containsMRJobInstruction(tmp.getFromInstructions(), true, true);
        ret |= containsMRJobInstruction(tmp.getToInstructions(), true, true);
        ret |= containsMRJobInstruction(tmp.getIncrementInstructions(), true, true);
        if (ret)
            return ret;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) {
            ret = rContainsMRJobInstruction(pb2, inclFunctions);
            if (ret)
                return ret;
        }
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP)
    pb instanceof FunctionProgramBlock) {
    // do nothing
    } else {
        ret = containsMRJobInstruction(pb, true, true) || (inclFunctions && containsFunctionCallInstruction(pb));
    }
    return ret;
}

Example 98 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptimizerConstrained method rewriteSetDataPartitioner.

// /////
// REWRITE set data partitioner
// /
@Override
protected boolean rewriteSetDataPartitioner(OptNode n, LocalVariableMap vars, HashMap<String, PartitionFormat> partitionedMatrices, double thetaM) {
    // call rewrite first to obtain partitioning information
    String initPlan = n.getParam(ParamType.DATA_PARTITIONER);
    boolean blockwise = super.rewriteSetDataPartitioner(n, vars, partitionedMatrices, thetaM);
    // constraint awareness
    if (!initPlan.equals(PDataPartitioner.UNSPECIFIED.name())) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
        pfpb.setDataPartitioner(PDataPartitioner.valueOf(initPlan));
        LOG.debug(getOptMode() + " OPT: forced 'set data partitioner' - result=" + initPlan);
    }
    return blockwise;
}

Example 99 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptimizerConstrained method rewriteSetDegreeOfParallelism.

// /////
// REWRITE set degree of parallelism
// /
@Override
protected void rewriteSetDegreeOfParallelism(OptNode n, double M, boolean flagNested) {
    // constraint awareness
    if (n.getK() > 0 && ConfigurationManager.isParallelParFor()) {
        // set parfor degree of parallelism
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
        pfpb.setDegreeOfParallelism(n.getK());
        // distribute remaining parallelism
        int remainParforK = getRemainingParallelismParFor(n.getK(), n.getK());
        int remainOpsK = getRemainingParallelismOps(_lkmaxCP, n.getK());
        rAssignRemainingParallelism(n, remainParforK, remainOpsK);
        LOG.debug(getOptMode() + " OPT: forced 'set degree of parallelism' - result=(see EXPLAIN)");
    } else
        super.rewriteSetDegreeOfParallelism(n, M, flagNested);
}

Example 100 with ParForProgramBlock

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock in project systemml by apache.

the class OptimizerConstrained method rewriteSetFusedDataPartitioningExecution.

// /////
// REWRITE set fused data partitioning / execution
// /
protected void rewriteSetFusedDataPartitioningExecution(OptNode pn, double M, boolean flagLIX, HashMap<String, PartitionFormat> partitionedMatrices, LocalVariableMap vars, PExecMode emode) {
    if (emode == PExecMode.REMOTE_MR_DP || emode == PExecMode.REMOTE_SPARK_DP) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
        // partitioned matrix
        if (partitionedMatrices.size() <= 0) {
            LOG.debug(getOptMode() + " OPT: unable to force 'set fused data partitioning and execution' - result=" + false);
            return;
        }
        String moVarname = partitionedMatrices.keySet().iterator().next();
        PartitionFormat moDpf = partitionedMatrices.get(moVarname);
        MatrixObject mo = (MatrixObject) vars.get(moVarname);
        if (rIsAccessByIterationVariable(pn, moVarname, pfpb.getIterVar()) && ((moDpf == PartitionFormat.ROW_WISE && mo.getNumRows() == _N) || (moDpf == PartitionFormat.COLUMN_WISE && mo.getNumColumns() == _N) || (moDpf._dpf == PDataPartitionFormat.ROW_BLOCK_WISE_N && mo.getNumRows() <= _N * moDpf._N) || (moDpf._dpf == PDataPartitionFormat.COLUMN_BLOCK_WISE_N && mo.getNumColumns() <= _N * moDpf._N))) {
            int k = (int) Math.min(_N, _rk2);
            if (emode == PExecMode.REMOTE_MR_DP) {
                pn.addParam(ParamType.DATA_PARTITIONER, "REMOTE_MR(fused)");
                // set fused exec type
                pfpb.setExecMode(PExecMode.REMOTE_MR_DP);
            } else {
                pn.addParam(ParamType.DATA_PARTITIONER, "REMOTE_SPARK(fused)");
                // set fused exec type
                pfpb.setExecMode(PExecMode.REMOTE_SPARK_DP);
            }
            pn.setK(k);
            pfpb.setDataPartitioner(PDataPartitioner.NONE);
            pfpb.enableColocatedPartitionedMatrix(moVarname);
            pfpb.setDegreeOfParallelism(k);
        }
        LOG.debug(getOptMode() + " OPT: force 'set fused data partitioning and execution' - result=" + true);
    } else
        super.rewriteSetFusedDataPartitioningExecution(pn, M, flagLIX, partitionedMatrices, vars);
}