Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 41 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class MRJobInstruction method extractInputMatrices.

/**
 * Extracts input variables with MATRIX data type, and stores references to
 * corresponding matrix objects in <code>inputMatrices</code>. Also, stores
 * the data types in <code>inputDataTypes</code>.
 *
 * @param ec execution context
 * @return array of matrix objects
 */
public MatrixObject[] extractInputMatrices(ExecutionContext ec) {
    ArrayList<MatrixObject> inputmat = new ArrayList<>();
    inputDataTypes = new DataType[inputVars.length];
    for (int i = 0; i < inputVars.length; i++) {
        Data d = ec.getVariable(inputVars[i]);
        inputDataTypes[i] = d.getDataType();
        if (d.getDataType() == DataType.MATRIX) {
            inputmat.add((MatrixObject) d);
        } else if (d.getDataType() == DataType.FRAME) {
            // FIXME conversion from frame to matrix object (meta data only) to adhere to
            // the given matrix-based mr job submission framework
            FrameObject fo = (FrameObject) d;
            MatrixObject mo = new MatrixObject(fo.getValueType(), fo.getFileName(), fo.getMetaData());
            mo.setFileFormatProperties(fo.getFileFormatProperties());
            inputmat.add(mo);
        }
    }
    inputMatrices = inputmat.toArray(new MatrixObject[inputmat.size()]);
    // populate auxiliary data structures
    populateInputs();
    return inputMatrices;
}

Example 42 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ParWorker method pinResultVariables.

protected void pinResultVariables() {
    for (String var : _resultVars) {
        Data dat = _ec.getVariable(var);
        if (dat instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) dat;
            mo.enableCleanup(false);
        }
    }
}

Example 43 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method computeTotalSizeResultVariables.

private double computeTotalSizeResultVariables(ArrayList<String> retVars, LocalVariableMap vars, int k) {
    double sum = 1;
    for (String var : retVars) {
        Data dat = vars.get(var);
        if (dat instanceof MatrixObject) {
            MatrixObject mo = (MatrixObject) dat;
            double nnz = mo.getNnz();
            if (nnz == 0.0)
                sum += OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), 1.0);
            else {
                double sp = mo.getSparsity();
                sum += (k + 1) * (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), // Every worker will consume memory for (MatrixSize/k + nnz) data.
                Math.min((1.0 / k) + sp, 1.0)));
            // This is applicable only when there is non-zero nnz. 
            }
        }
    }
    return sum;
}

Example 44 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteSetInPlaceResultIndexing.

///////
//REWRITE set in-place result indexing
///
protected void rewriteSetInPlaceResultIndexing(OptNode pn, double M, LocalVariableMap vars, HashSet<String> inPlaceResultVars, ExecutionContext ec) throws DMLRuntimeException {
    //assertions (warnings of corrupt optimizer decisions)
    if (pn.getNodeType() != NodeType.PARFOR)
        LOG.warn(getOptMode() + " OPT: Set in-place result update is only applicable for a ParFor node.");
    boolean apply = false;
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
    //note currently we decide for all result vars jointly, i.e.,
    //only if all fit pinned in remaining budget, we apply this rewrite.
    ArrayList<String> retVars = pfpb.getResultVariables();
    //compute total sum of pinned result variable memory
    double sum = computeTotalSizeResultVariables(retVars, vars, pfpb.getDegreeOfParallelism());
    //NOTE: currently this rule is too conservative (the result variable is assumed to be dense and
    //most importantly counted twice if this is part of the maximum operation)
    HashMap<String, ArrayList<UIPCandidateHop>> uipCandHopHM = new HashMap<String, ArrayList<UIPCandidateHop>>();
    double totalMem = Math.max((M + sum), rComputeSumMemoryIntermediates(pn, new HashSet<String>(), uipCandHopHM));
    //optimization decision
    if (//basic correctness constraint
    rHasOnlyInPlaceSafeLeftIndexing(pn, retVars)) {
        //result update in-place for MR/Spark (w/ remote memory constraint)
        if ((pfpb.getExecMode() == PExecMode.REMOTE_MR_DP || pfpb.getExecMode() == PExecMode.REMOTE_MR || pfpb.getExecMode() == PExecMode.REMOTE_SPARK_DP || pfpb.getExecMode() == PExecMode.REMOTE_SPARK) && totalMem < _rm) {
            apply = true;
        } else //result update in-place for CP (w/ local memory constraint)
        if (pfpb.getExecMode() == PExecMode.LOCAL && totalMem * pfpb.getDegreeOfParallelism() < _lm && //no forced mr/spark execution  
        pn.isCPOnly()) {
            apply = true;
        }
    }
    if (APPLY_REWRITE_UPDATE_INPLACE_INTERMEDIATE && LOG.isDebugEnabled())
        listUIPRes.remove();
    //modify result variable meta data, if rewrite applied
    if (apply) {
        //will be serialized and transfered via symbol table 
        for (String var : retVars) {
            Data dat = vars.get(var);
            if (dat instanceof MatrixObject)
                ((MatrixObject) dat).setUpdateType(UpdateType.INPLACE_PINNED);
        }
        inPlaceResultVars.addAll(retVars);
        if (APPLY_REWRITE_UPDATE_INPLACE_INTERMEDIATE) {
            isUpdateInPlaceApplicable(pn, uipCandHopHM);
            boolean bAnyUIPApplicable = false;
            for (Entry<String, ArrayList<UIPCandidateHop>> entry : uipCandHopHM.entrySet()) {
                ArrayList<UIPCandidateHop> uipCandHopList = entry.getValue();
                if (uipCandHopList != null) {
                    for (UIPCandidateHop uipCandHop : uipCandHopList) if (uipCandHop.isIntermediate() && uipCandHop.isLoopApplicable() && uipCandHop.isUpdateInPlace()) {
                        uipCandHop.getHop().setUpdateType(UpdateType.INPLACE_PINNED);
                        bAnyUIPApplicable = true;
                        if (LOG.isDebugEnabled())
                            listUIPRes.get().add(uipCandHop.getHop().getName());
                    }
                }
            }
            if (bAnyUIPApplicable)
                try {
                    //Recompile this block recursively if there is any update in place applicable.
                    LocalVariableMap localVaraibleMap = (LocalVariableMap) ec.getVariables().clone();
                    Recompiler.recompileProgramBlockHierarchy(pfpb.getChildBlocks(), localVaraibleMap, 0L, true);
                } catch (Exception ex) {
                    throw new DMLRuntimeException(ex);
                }
        }
    }
    if (APPLY_REWRITE_UPDATE_INPLACE_INTERMEDIATE && LOG.isTraceEnabled()) {
        LOG.trace("UpdateInPlace = " + apply + " for lines between " + pn.getBeginLine() + " and " + pn.getEndLine() + " for " + uipCandHopHM.size() + " intermediate matrix objects:" + uipCandHopHM.keySet().toString());
        for (Entry<String, ArrayList<UIPCandidateHop>> entry : uipCandHopHM.entrySet()) {
            ArrayList<UIPCandidateHop> uipCandHopList = entry.getValue();
            if (uipCandHopList != null) {
                for (UIPCandidateHop uipCandHop : uipCandHopList) {
                    if (uipCandHop.getHop() != null) {
                        LOG.trace("Matrix Object: Name: " + uipCandHop.getHop().getName() + "<" + uipCandHop.getHop().getBeginLine() + "," + uipCandHop.getHop().getEndLine() + ">, InLoop:" + uipCandHop.isLoopApplicable() + ", UIPApplicable:" + uipCandHop.isUpdateInPlace() + ", HopUIPApplicable:" + uipCandHop.getHop().getUpdateType());
                        LOG.trace("Explain Candidate HOP after recompile");
                        LOG.trace(Explain.explain(uipCandHop.getHop()));
                    } else {
                        LOG.trace("Matrix Object: Name: " + uipCandHop.getLixHop().getName() + "<" + uipCandHop.getLixHop().getBeginLine() + "," + uipCandHop.getLixHop().getEndLine() + ">, InLoop:" + uipCandHop.isLoopApplicable() + ", Not an Intermediate matrix object");
                    }
                }
            }
        }
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'set in-place result indexing' - result=" + apply + " (" + ProgramConverter.serializeStringCollection(inPlaceResultVars) + ", M=" + toMB(totalMem) + ")");
}

Example 45 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ResultVariables method getMatrix.

/**
	 * Obtain the matrix represented by the given output variable.
	 * 
	 * @param varname output variable name
	 * @return matrix as a two-dimensional double array
	 * @throws DMLException if DMLException occurs
	 */
public double[][] getMatrix(String varname) throws DMLException {
    if (!_out.containsKey(varname))
        throw new DMLException("Non-existent output variable: " + varname);
    double[][] ret = null;
    Data dat = _out.get(varname);
    //basic checks for data type	
    if (!(dat instanceof MatrixObject))
        throw new DMLException("Expected matrix result '" + varname + "' not a matrix.");
    //convert output matrix to double array	
    MatrixObject mo = (MatrixObject) dat;
    MatrixBlock mb = mo.acquireRead();
    ret = DataConverter.convertToDoubleMatrix(mb);
    mo.release();
    return ret;
}