Examples with MatrixObject org.apache.sysml.runtime.controlprogram.caching.MatrixObject used on opensource projects

Example 16 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class OptimizerRuleBased method hasOnlyInMemoryResults.

protected boolean hasOnlyInMemoryResults(OptNode n, ArrayList<String> resultVars, LocalVariableMap vars, boolean inLocal) throws DMLRuntimeException {
    boolean ret = true;
    if (n.isLeaf()) {
        String opName = n.getParam(ParamType.OPSTRING);
        //check opstring and exec type
        if (opName.equals(LeftIndexingOp.OPSTRING)) {
            LeftIndexingOp hop = (LeftIndexingOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
            //check agains set of varname
            String varName = hop.getInput().get(0).getName();
            if (resultVars.contains(varName) && vars.keySet().contains(varName)) {
                //dims of result vars must be known at this point in time
                MatrixObject mo = (MatrixObject) vars.get(hop.getInput().get(0).getName());
                long rows = mo.getNumRows();
                long cols = mo.getNumColumns();
                double memBudget = inLocal ? OptimizerUtils.getLocalMemBudget() : OptimizerUtils.getRemoteMemBudgetMap();
                ret &= isInMemoryResultMerge(rows, cols, memBudget);
            }
        }
    } else {
        for (OptNode c : n.getChilds()) ret &= hasOnlyInMemoryResults(c, resultVars, vars, inLocal);
    }
    return ret;
}

Example 17 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class OptimizerRuleBased method hasLargeTotalResults.

/**
	 * Heuristically compute total result sizes, if larger than local mem budget assumed to be large.
	 * 
	 * @param pn internal representation of a plan alternative for program blocks and instructions
	 * @param resultVars list of result variables
	 * @param vars local variable map
	 * @param checkSize ?
	 * @return true if result sizes larger than local memory budget
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
protected boolean hasLargeTotalResults(OptNode pn, ArrayList<String> resultVars, LocalVariableMap vars, boolean checkSize) throws DMLRuntimeException {
    double totalSize = 0;
    //get num tasks according to task partitioning 
    PTaskPartitioner tp = PTaskPartitioner.valueOf(pn.getParam(ParamType.TASK_PARTITIONER));
    int k = pn.getK();
    long W = estimateNumTasks(tp, _N, k);
    for (String var : resultVars) {
        //Potential unknowns: for local result var of child parfor (but we're only interested in top level)
        //Potential scalars: for disabled dependency analysis and unbounded scoping			
        Data dat = vars.get(var);
        if (dat != null && dat instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) vars.get(var);
            long rows = mo.getNumRows();
            long cols = mo.getNumColumns();
            long nnz = mo.getNnz();
            if (//w/ compare
            nnz > 0) {
                totalSize += W * OptimizerUtils.estimateSizeExactSparsity(rows, cols, 1.0);
            } else //in total at most as dimensions (due to disjoint results)
            {
                totalSize += OptimizerUtils.estimateSizeExactSparsity(rows, cols, 1.0);
            }
        }
    }
    //heuristic:  large if >= local mem budget 
    return (totalSize >= _lm);
}

Example 18 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class OptimizerRuleBased method hasResultMRLeftIndexing.

protected boolean hasResultMRLeftIndexing(OptNode n, ArrayList<String> resultVars, LocalVariableMap vars, boolean checkSize) throws DMLRuntimeException {
    boolean ret = false;
    if (n.isLeaf()) {
        String opName = n.getParam(ParamType.OPSTRING);
        //check opstring and exec type
        if (opName != null && opName.equals(LeftIndexingOp.OPSTRING) && n.getExecType() == getRemoteExecType()) {
            LeftIndexingOp hop = (LeftIndexingOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
            //check agains set of varname
            String varName = hop.getInput().get(0).getName();
            if (resultVars.contains(varName)) {
                ret = true;
                if (checkSize && vars.keySet().contains(varName)) {
                    //dims of result vars must be known at this point in time
                    MatrixObject mo = (MatrixObject) vars.get(hop.getInput().get(0).getName());
                    long rows = mo.getNumRows();
                    long cols = mo.getNumColumns();
                    ret = !isInMemoryResultMerge(rows, cols, OptimizerUtils.getRemoteMemBudgetMap(false));
                }
            }
        }
    } else {
        for (OptNode c : n.getChilds()) ret |= hasResultMRLeftIndexing(c, resultVars, vars, checkSize);
    }
    return ret;
}

Example 19 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class PreparedScript method setMatrix.

/**
 * Binds a matrix object to a registered input variable.
 * If reuse requested, then the input is guaranteed to be
 * preserved over multiple <code>executeScript</code> calls.
 *
 * @param varname input variable name
 * @param matrix matrix represented as a MatrixBlock
 * @param reuse if {@code true}, preserve value over multiple {@code executeScript} calls
 */
public void setMatrix(String varname, MatrixBlock matrix, boolean reuse) {
    if (!_inVarnames.contains(varname))
        throw new DMLException("Unspecified input variable: " + varname);
    int blocksize = ConfigurationManager.getBlocksize();
    // create new matrix object
    MatrixCharacteristics mc = new MatrixCharacteristics(matrix.getNumRows(), matrix.getNumColumns(), blocksize, blocksize);
    MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
    MatrixObject mo = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), meta);
    mo.acquireModify(matrix);
    mo.release();
    // put create matrix wrapper into symbol table
    _vars.put(varname, mo);
    if (reuse) {
        // prevent cleanup
        mo.enableCleanup(false);
        _inVarReuse.put(varname, mo);
    }
}

Example 20 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class MLContextConversionUtil method binaryBlocksToMatrixObject.

private static MatrixObject binaryBlocksToMatrixObject(JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocks, MatrixMetadata matrixMetadata, boolean copy) {
    MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    JavaPairRDD<MatrixIndexes, MatrixBlock> javaPairRdd = SparkUtils.copyBinaryBlockMatrix(binaryBlocks, copy);
    MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
    matrixObject.setRDDHandle(new RDDObject(javaPairRdd));
    return matrixObject;
}