Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 1 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ExecutionContext method pinVariables.

/**
	 * Pin a given list of variables i.e., set the "clean up" state in 
	 * corresponding matrix objects, so that the cached data inside these
	 * objects is not cleared and the corresponding HDFS files are not 
	 * deleted (through rmvar instructions). 
	 * 
	 * This is necessary for: function input variables, parfor result variables, 
	 * parfor shared inputs that are passed to functions.
	 * 
	 * The function returns the OLD "clean up" state of matrix objects.
	 * 
	 * @param varList variable list
	 * @return map of old cleanup state of matrix objects
	 */
public HashMap<String, Boolean> pinVariables(ArrayList<String> varList) {
    //2-pass approach since multiple vars might refer to same matrix object
    HashMap<String, Boolean> varsState = new HashMap<String, Boolean>();
    //step 1) get current information
    for (String var : varList) {
        Data dat = _variables.get(var);
        if (dat instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) dat;
            varsState.put(var, mo.isCleanupEnabled());
        //System.out.println("pre-pin "+var+" ("+mo.isCleanupEnabled()+")");
        }
    }
    //step 2) pin variables
    for (String var : varList) {
        Data dat = _variables.get(var);
        if (dat instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) dat;
            mo.enableCleanup(false);
        //System.out.println("pin "+var);
        }
    }
    return varsState;
}

Example 2 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class InterProceduralAnalysis method populateLocalVariableMapForFunctionCall.

private void populateLocalVariableMapForFunctionCall(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callvars, LocalVariableMap vars, Set<Long> inputSafeNNZ, Integer numCalls) throws HopsException {
    ArrayList<DataIdentifier> inputVars = fstmt.getInputParams();
    ArrayList<Hop> inputOps = fop.getInput();
    for (int i = 0; i < inputVars.size(); i++) {
        //create mapping between input hops and vars
        DataIdentifier dat = inputVars.get(i);
        Hop input = inputOps.get(i);
        if (input.getDataType() == DataType.MATRIX) {
            //propagate matrix characteristics
            MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
            MatrixCharacteristics mc = new MatrixCharacteristics(input.getDim1(), input.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), inputSafeNNZ.contains(input.getHopID()) ? input.getNnz() : -1);
            MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, null, null);
            mo.setMetaData(meta);
            vars.put(dat.getName(), mo);
        } else if (input.getDataType() == DataType.SCALAR) {
            //(for multiple calls, literal equivalence already checked)
            if (input instanceof LiteralOp) {
                vars.put(dat.getName(), ScalarObjectFactory.createScalarObject(input.getValueType(), (LiteralOp) input));
            } else //and input scalar is existing variable in symbol table
            if (PROPAGATE_SCALAR_VARS_INTO_FUN && numCalls != null && numCalls == 1 && input instanceof DataOp) {
                Data scalar = callvars.get(input.getName());
                if (scalar != null && scalar instanceof ScalarObject) {
                    vars.put(dat.getName(), scalar);
                }
            }
        }
    }
}

Example 3 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ResultVariables method getFrame.

/**
	 * Obtain the frame represented by the given output variable.
	 * 
	 * @param varname output variable name
	 * @return frame as a two-dimensional string array
	 * @throws DMLException if DMLException occurs
	 */
public String[][] getFrame(String varname) throws DMLException {
    if (!_out.containsKey(varname))
        throw new DMLException("Non-existent output variable: " + varname);
    Data dat = _out.get(varname);
    //basic checks for data type	
    if (!(dat instanceof FrameObject))
        throw new DMLException("Expected frame result '" + varname + "' not a frame.");
    //convert output matrix to double array	
    FrameObject fo = (FrameObject) dat;
    FrameBlock frame = fo.acquireRead();
    String[][] ret = DataConverter.convertToStringFrame(frame);
    fo.release();
    return ret;
}

Example 4 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method determineFlagCellFormatWoCompare.

protected boolean determineFlagCellFormatWoCompare(ArrayList<String> resultVars, LocalVariableMap vars) {
    boolean ret = true;
    for (String rVar : resultVars) {
        Data dat = vars.get(rVar);
        if (dat == null || !(dat instanceof MatrixObject)) {
            ret = false;
            break;
        } else {
            MatrixObject mo = (MatrixObject) dat;
            MatrixFormatMetaData meta = (MatrixFormatMetaData) mo.getMetaData();
            OutputInfo oi = meta.getOutputInfo();
            long nnz = meta.getMatrixCharacteristics().getNonZeros();
            if (oi == OutputInfo.BinaryBlockOutputInfo || nnz != 0) {
                ret = false;
                break;
            }
        }
    }
    return ret;
}

Example 5 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method hasLargeTotalResults.

/**
	 * Heuristically compute total result sizes, if larger than local mem budget assumed to be large.
	 * 
	 * @param pn internal representation of a plan alternative for program blocks and instructions
	 * @param resultVars list of result variables
	 * @param vars local variable map
	 * @param checkSize ?
	 * @return true if result sizes larger than local memory budget
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
protected boolean hasLargeTotalResults(OptNode pn, ArrayList<String> resultVars, LocalVariableMap vars, boolean checkSize) throws DMLRuntimeException {
    double totalSize = 0;
    //get num tasks according to task partitioning 
    PTaskPartitioner tp = PTaskPartitioner.valueOf(pn.getParam(ParamType.TASK_PARTITIONER));
    int k = pn.getK();
    long W = estimateNumTasks(tp, _N, k);
    for (String var : resultVars) {
        //Potential unknowns: for local result var of child parfor (but we're only interested in top level)
        //Potential scalars: for disabled dependency analysis and unbounded scoping			
        Data dat = vars.get(var);
        if (dat != null && dat instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) vars.get(var);
            long rows = mo.getNumRows();
            long cols = mo.getNumColumns();
            long nnz = mo.getNnz();
            if (//w/ compare
            nnz > 0) {
                totalSize += W * OptimizerUtils.estimateSizeExactSparsity(rows, cols, 1.0);
            } else //in total at most as dimensions (due to disjoint results)
            {
                totalSize += OptimizerUtils.estimateSizeExactSparsity(rows, cols, 1.0);
            }
        }
    }
    //heuristic:  large if >= local mem budget 
    return (totalSize >= _lm);
}