Examples with PartitionFormat org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PartitionFormat used on opensource projects

Example 11 with PartitionFormat

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PartitionFormat in project incubator-systemml by apache.

the class ParForStatementBlock method determineAccessPattern.

private PartitionFormat determineAccessPattern(IndexedIdentifier dat) {
    boolean isSpark = OptimizerUtils.isSparkExecutionMode();
    int blksz = ConfigurationManager.getBlocksize();
    PartitionFormat dpf = null;
    // 1) get all bounds expressions for index access
    Expression rowL = dat.getRowLowerBound();
    Expression rowU = dat.getRowUpperBound();
    Expression colL = dat.getColLowerBound();
    Expression colU = dat.getColUpperBound();
    boolean allRows = (rowL == null && rowU == null);
    boolean allCols = (colL == null && colU == null);
    try {
        // COLUMN_WISE if all rows and access to single column
        if (allRows && colL != null && colL.equals(colU)) {
            dpf = PartitionFormat.COLUMN_WISE;
        } else // ROW_WISE if all cols and access to single row
        if (allCols && rowL != null && rowL.equals(rowU)) {
            dpf = PartitionFormat.ROW_WISE;
        } else // COLUMN_BLOCK_WISE
        if (isSpark && allRows && colL != colU) {
            LinearFunction l1 = getLinearFunction(colL, true);
            LinearFunction l2 = getLinearFunction(colU, true);
            dpf = !isAlignedBlocking(l1, l2, blksz) ? PartitionFormat.NONE : new PartitionFormat(PDataPartitionFormat.COLUMN_BLOCK_WISE_N, (int) l1._b[0]);
        } else // ROW_BLOCK_WISE
        if (isSpark && allCols && rowL != rowU) {
            LinearFunction l1 = getLinearFunction(rowL, true);
            LinearFunction l2 = getLinearFunction(rowU, true);
            dpf = !isAlignedBlocking(l1, l2, blksz) ? PartitionFormat.NONE : new PartitionFormat(PDataPartitionFormat.ROW_BLOCK_WISE_N, (int) l1._b[0]);
        } else
            // NONE otherwise (conservative)
            dpf = PartitionFormat.NONE;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    return dpf;
}

Example 12 with PartitionFormat

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PartitionFormat in project incubator-systemml by apache.

the class ParForStatementBlock method determineDataPartitionFormat.

/**
 * Determines the PDataPartitioningFormat for read-only parent variables according
 * to the access pattern of that variable within the parfor statement block.
 * Row-wise or column wise partitioning is only suggested if we see pure row-wise or
 * column-wise access patterns.
 *
 * @param var variables
 * @return partition format
 */
public PartitionFormat determineDataPartitionFormat(String var) {
    PartitionFormat dpf = null;
    List<PartitionFormat> dpfc = new LinkedList<>();
    try {
        // determine partitioning candidates
        ParForStatement dpfs = (ParForStatement) _statements.get(0);
        rDeterminePartitioningCandidates(var, dpfs.getBody(), dpfc);
        // determine final solution
        for (PartitionFormat tmp : dpfc) dpf = // if no consensus
        (dpf != null && !dpf.equals(tmp)) ? PartitionFormat.NONE : tmp;
        if (dpf == null)
            dpf = PartitionFormat.NONE;
    } catch (LanguageException e) {
        LOG.trace("Unable to determine partitioning candidates.", e);
        dpf = PartitionFormat.NONE;
    }
    return dpf;
}

Example 13 with PartitionFormat

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PartitionFormat in project incubator-systemml by apache.

the class RemoteParForColocatedNLineInputFormat method getSplits.

@Override
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    InputSplit[] tmp = super.getSplits(job, numSplits);
    // get partitioning information
    MatrixCharacteristics mc = MRJobConfiguration.getPartitionedMatrixSize(job);
    PDataPartitionFormat dpf = MRJobConfiguration.getPartitioningFormat(job);
    PartitionFormat pf = new PartitionFormat(dpf, -1);
    int blen = (int) (pf.isRowwise() ? pf.getNumRows(mc) : pf.getNumColumns(mc));
    String fname = MRJobConfiguration.getPartitioningFilename(job);
    // create wrapper splits
    InputSplit[] ret = new InputSplit[tmp.length];
    for (int i = 0; i < tmp.length; i++) {
        // check for robustness of subsequent cast
        if (tmp[i] instanceof FileSplit)
            ret[i] = new RemoteParForColocatedFileSplit((FileSplit) tmp[i], fname, blen);
        else
            ret[i] = tmp[i];
    }
    return ret;
}

Example 14 with PartitionFormat

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PartitionFormat in project incubator-systemml by apache.

the class OptimizerConstrained method rewriteSetFusedDataPartitioningExecution.

// /////
// REWRITE set fused data partitioning / execution
// /
protected void rewriteSetFusedDataPartitioningExecution(OptNode pn, double M, boolean flagLIX, HashMap<String, PartitionFormat> partitionedMatrices, LocalVariableMap vars, PExecMode emode) {
    if (emode == PExecMode.REMOTE_MR_DP || emode == PExecMode.REMOTE_SPARK_DP) {
        ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
        // partitioned matrix
        if (partitionedMatrices.size() <= 0) {
            LOG.debug(getOptMode() + " OPT: unable to force 'set fused data partitioning and execution' - result=" + false);
            return;
        }
        String moVarname = partitionedMatrices.keySet().iterator().next();
        PartitionFormat moDpf = partitionedMatrices.get(moVarname);
        MatrixObject mo = (MatrixObject) vars.get(moVarname);
        if (rIsAccessByIterationVariable(pn, moVarname, pfpb.getIterVar()) && ((moDpf == PartitionFormat.ROW_WISE && mo.getNumRows() == _N) || (moDpf == PartitionFormat.COLUMN_WISE && mo.getNumColumns() == _N) || (moDpf._dpf == PDataPartitionFormat.ROW_BLOCK_WISE_N && mo.getNumRows() <= _N * moDpf._N) || (moDpf._dpf == PDataPartitionFormat.COLUMN_BLOCK_WISE_N && mo.getNumColumns() <= _N * moDpf._N))) {
            int k = (int) Math.min(_N, _rk2);
            if (emode == PExecMode.REMOTE_MR_DP) {
                pn.addParam(ParamType.DATA_PARTITIONER, "REMOTE_MR(fused)");
                // set fused exec type
                pfpb.setExecMode(PExecMode.REMOTE_MR_DP);
            } else {
                pn.addParam(ParamType.DATA_PARTITIONER, "REMOTE_SPARK(fused)");
                // set fused exec type
                pfpb.setExecMode(PExecMode.REMOTE_SPARK_DP);
            }
            pn.setK(k);
            pfpb.setDataPartitioner(PDataPartitioner.NONE);
            pfpb.enableColocatedPartitionedMatrix(moVarname);
            pfpb.setDegreeOfParallelism(k);
        }
        LOG.debug(getOptMode() + " OPT: force 'set fused data partitioning and execution' - result=" + true);
    } else
        super.rewriteSetFusedDataPartitioningExecution(pn, M, flagLIX, partitionedMatrices, vars);
}

Example 15 with PartitionFormat

use of org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PartitionFormat in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteSetTranposeSparseVectorOperations.

// /////
// REWRITE transpose sparse vector operations
// /
protected void rewriteSetTranposeSparseVectorOperations(OptNode pn, HashMap<String, PartitionFormat> partitionedMatrices, LocalVariableMap vars) {
    // assertions (warnings of corrupt optimizer decisions)
    if (pn.getNodeType() != NodeType.PARFOR)
        LOG.warn(getOptMode() + " OPT: Transpose sparse vector operations is only applicable for a ParFor node.");
    boolean apply = false;
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
    if (pfpb.getExecMode() == PExecMode.REMOTE_MR_DP && // general applicable
    partitionedMatrices.size() == 1) {
        String moVarname = partitionedMatrices.keySet().iterator().next();
        PartitionFormat moDpf = partitionedMatrices.get(moVarname);
        Data dat = vars.get(moVarname);
        if (dat != null && dat instanceof MatrixObject && moDpf == PartitionFormat.COLUMN_WISE && // check for sparse matrix
        ((MatrixObject) dat).getSparsity() <= MatrixBlock.SPARSITY_TURN_POINT && // tranpose-safe
        rIsTransposeSafePartition(pn, moVarname)) {
            pfpb.setTransposeSparseColumnVector(true);
            apply = true;
        }
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'set transpose sparse vector operations' - result=" + apply);
}