Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 6 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ResultVariables method getFrameBlock.

/**
 * Obtain the frame represented by the given output variable.
 * Calling this method avoids unnecessary output conversions.
 *
 * @param varname output variable name
 * @return frame as a frame block
 */
public FrameBlock getFrameBlock(String varname) {
    Data dat = _out.get(varname);
    if (dat == null)
        throw new DMLException("Non-existent output variable: " + varname);
    // basic checks for data type
    if (!(dat instanceof FrameObject))
        throw new DMLException("Expected frame result '" + varname + "' not a frame.");
    // convert output matrix to double array
    FrameObject fo = (FrameObject) dat;
    FrameBlock fb = fo.acquireRead();
    fo.release();
    return fb;
}

Example 7 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method computeTotalSizeResultVariables.

private static double computeTotalSizeResultVariables(ArrayList<ResultVar> retVars, LocalVariableMap vars, int k) {
    double sum = 1;
    for (ResultVar var : retVars) {
        Data dat = vars.get(var._name);
        if (!(dat instanceof MatrixObject))
            continue;
        MatrixObject mo = (MatrixObject) dat;
        if (mo.getNnz() == 0)
            sum += OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), 1.0);
        else {
            // Every worker will consume memory for (MatrixSize/k + nnz) data.
            // This is applicable only when there is non-zero nnz.
            sum += (k + 1) * (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), Math.min((1.0 / k) + mo.getSparsity(), 1.0)));
        }
    }
    return sum;
}

Example 8 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method hasLargeTotalResults.

/**
 * Heuristically compute total result sizes, if larger than local mem budget assumed to be large.
 *
 * @param pn internal representation of a plan alternative for program blocks and instructions
 * @param resultVars list of result variables
 * @param vars local variable map
 * @param checkSize ?
 * @return true if result sizes larger than local memory budget
 */
protected boolean hasLargeTotalResults(OptNode pn, ArrayList<ResultVar> resultVars, LocalVariableMap vars, boolean checkSize) {
    double totalSize = 0;
    // get num tasks according to task partitioning
    PTaskPartitioner tp = PTaskPartitioner.valueOf(pn.getParam(ParamType.TASK_PARTITIONER));
    int k = pn.getK();
    long W = estimateNumTasks(tp, _N, k);
    for (ResultVar var : resultVars) {
        // Potential unknowns: for local result var of child parfor (but we're only interested in top level)
        // Potential scalars: for disabled dependency analysis and unbounded scoping
        Data dat = vars.get(var._name);
        if (dat != null && dat instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) dat;
            long rows = mo.getNumRows();
            long cols = mo.getNumColumns();
            long nnz = mo.getNnz();
            if (// w/ compare
            nnz > 0) {
                totalSize += W * OptimizerUtils.estimateSizeExactSparsity(rows, cols, 1.0);
            } else // in total at most as dimensions (due to disjoint results)
            {
                totalSize += OptimizerUtils.estimateSizeExactSparsity(rows, cols, 1.0);
            }
        }
    }
    // heuristic:  large if >= local mem budget
    return (totalSize >= _lm);
}

Example 9 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method getNewRIXMemoryEstimate.

/**
 * TODO consolidate mem estimation with Indexing Hop
 *
 * NOTE: Using the dimensions without sparsity is a conservative worst-case consideration.
 *
 * @param n internal representation of a plan alternative for program blocks and instructions
 * @param varName variable name
 * @param dpf data partition format
 * @param vars local variable map
 * @return memory estimate
 */
protected double getNewRIXMemoryEstimate(OptNode n, String varName, PartitionFormat dpf, LocalVariableMap vars) {
    double mem = -1;
    // not all intermediates need to be known on optimize
    Data dat = vars.get(varName);
    if (dat != null) {
        MatrixObject mo = (MatrixObject) dat;
        // those are worst-case (dense) estimates
        switch(dpf._dpf) {
            case COLUMN_WISE:
                mem = OptimizerUtils.estimateSize(mo.getNumRows(), 1);
                break;
            case ROW_WISE:
                mem = OptimizerUtils.estimateSize(1, mo.getNumColumns());
                break;
            case COLUMN_BLOCK_WISE_N:
                mem = OptimizerUtils.estimateSize(mo.getNumRows(), dpf._N);
                break;
            case ROW_BLOCK_WISE_N:
                mem = OptimizerUtils.estimateSize(dpf._N, mo.getNumColumns());
                break;
            default:
        }
    }
    return mem;
}

Example 10 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteSetInPlaceResultIndexing.

// /////
// REWRITE set in-place result indexing
// /
protected void rewriteSetInPlaceResultIndexing(OptNode pn, double M, LocalVariableMap vars, HashSet<ResultVar> inPlaceResultVars, ExecutionContext ec) {
    // assertions (warnings of corrupt optimizer decisions)
    if (pn.getNodeType() != NodeType.PARFOR)
        LOG.warn(getOptMode() + " OPT: Set in-place result update is only applicable for a ParFor node.");
    boolean apply = false;
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
    // note currently we decide for all result vars jointly, i.e.,
    // only if all fit pinned in remaining budget, we apply this rewrite.
    ArrayList<ResultVar> retVars = pfpb.getResultVariables();
    // compute total sum of pinned result variable memory
    double sum = computeTotalSizeResultVariables(retVars, vars, pfpb.getDegreeOfParallelism());
    // NOTE: currently this rule is too conservative (the result variable is assumed to be dense and
    // most importantly counted twice if this is part of the maximum operation)
    double totalMem = Math.max((M + sum), rComputeSumMemoryIntermediates(pn, new HashSet<ResultVar>()));
    // optimization decision
    if (// basic correctness constraint
    rHasOnlyInPlaceSafeLeftIndexing(pn, retVars)) {
        // result update in-place for MR/Spark (w/ remote memory constraint)
        if ((pfpb.getExecMode() == PExecMode.REMOTE_MR_DP || pfpb.getExecMode() == PExecMode.REMOTE_MR || pfpb.getExecMode() == PExecMode.REMOTE_SPARK_DP || pfpb.getExecMode() == PExecMode.REMOTE_SPARK) && totalMem < _rm) {
            apply = true;
        } else // result update in-place for CP (w/ local memory constraint)
        if (pfpb.getExecMode() == PExecMode.LOCAL && totalMem * pfpb.getDegreeOfParallelism() < _lm && // no forced mr/spark execution
        pn.isCPOnly()) {
            apply = true;
        }
    }
    // modify result variable meta data, if rewrite applied
    if (apply) {
        // will be serialized and transfered via symbol table
        for (ResultVar var : retVars) {
            Data dat = vars.get(var._name);
            if (dat instanceof MatrixObject)
                ((MatrixObject) dat).setUpdateType(UpdateType.INPLACE_PINNED);
        }
        inPlaceResultVars.addAll(retVars);
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'set in-place result indexing' - result=" + apply + " (" + Arrays.toString(inPlaceResultVars.toArray(new ResultVar[0])) + ", M=" + toMB(totalMem) + ")");
}