Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 11 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteRemoveUnnecessaryCompareMatrix.

// /////
// REWRITE remove compare matrix (for result merge, needs to be invoked before setting result merge)
// /
protected void rewriteRemoveUnnecessaryCompareMatrix(OptNode n, ExecutionContext ec) {
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
    ArrayList<ResultVar> cleanedVars = new ArrayList<>();
    ArrayList<ResultVar> resultVars = pfpb.getResultVariables();
    String itervar = pfpb.getIterVar();
    for (ResultVar rvar : resultVars) {
        Data dat = ec.getVariable(rvar._name);
        if (// subject to result merge with compare
        dat instanceof MatrixObject && ((MatrixObject) dat).getNnz() != 0 && // guaranteed no conditional indexing
        n.hasOnlySimpleChilds() && // guaranteed full matrix replace
        rContainsResultFullReplace(n, rvar._name, itervar, (MatrixObject) dat) && // && !pfsb.variablesRead().containsVariable(rvar)                  //never read variable in loop body
        !// never read variable in loop body
        rIsReadInRightIndexing(n, rvar._name) && ((MatrixObject) dat).getNumRows() <= Integer.MAX_VALUE && ((MatrixObject) dat).getNumColumns() <= Integer.MAX_VALUE) {
            // replace existing matrix object with empty matrix
            MatrixObject mo = (MatrixObject) dat;
            ec.cleanupCacheableData(mo);
            ec.setMatrixOutput(rvar._name, new MatrixBlock((int) mo.getNumRows(), (int) mo.getNumColumns(), false), null);
            // keep track of cleaned result variables
            cleanedVars.add(rvar);
        }
    }
    _numEvaluatedPlans++;
    LOG.debug(getOptMode() + " OPT: rewrite 'remove unnecessary compare matrix' - result=" + (!cleanedVars.isEmpty()) + " (" + ProgramConverter.serializeResultVariables(cleanedVars) + ")");
}

Example 12 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method determineFlagCellFormatWoCompare.

protected boolean determineFlagCellFormatWoCompare(ArrayList<ResultVar> resultVars, LocalVariableMap vars) {
    boolean ret = true;
    for (ResultVar rVar : resultVars) {
        Data dat = vars.get(rVar._name);
        if (dat == null || !(dat instanceof MatrixObject)) {
            ret = false;
            break;
        } else {
            MatrixObject mo = (MatrixObject) dat;
            MetaDataFormat meta = (MetaDataFormat) mo.getMetaData();
            OutputInfo oi = meta.getOutputInfo();
            long nnz = meta.getMatrixCharacteristics().getNonZeros();
            if (oi == OutputInfo.BinaryBlockOutputInfo || nnz != 0) {
                ret = false;
                break;
            }
        }
    }
    return ret;
}

Example 13 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteSetSparkEagerRDDCaching.

// /////
// REWRITE set spark eager rdd caching
// /
protected void rewriteSetSparkEagerRDDCaching(OptNode n, LocalVariableMap vars) {
    // get program blocks of root parfor
    Object[] progobj = OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID());
    ParForStatementBlock pfsb = (ParForStatementBlock) progobj[0];
    ParForProgramBlock pfpb = (ParForProgramBlock) progobj[1];
    ArrayList<String> ret = new ArrayList<>();
    if (// spark exec mode
    OptimizerUtils.isSparkExecutionMode() && // local parfor
    n.getExecType() == ExecType.CP && // at least 2 iterations
    _N > 1) {
        Set<String> cand = pfsb.variablesRead().getVariableNames();
        Collection<String> rpVars = pfpb.getSparkRepartitionVariables();
        for (String var : cand) {
            Data dat = vars.get(var);
            if (dat != null && dat instanceof MatrixObject && ((MatrixObject) dat).getRDDHandle() != null) {
                MatrixObject mo = (MatrixObject) dat;
                MatrixCharacteristics mc = mo.getMatrixCharacteristics();
                RDDObject rdd = mo.getRDDHandle();
                if (// not a repartition var
                (rpVars == null || !rpVars.contains(var)) && // is cached rdd
                rdd.rHasCheckpointRDDChilds() && // is out-of-core dataset
                _lm / n.getK() < OptimizerUtils.estimateSizeExactSparsity(mc)) {
                    ret.add(var);
                }
            }
        }
        // apply rewrite to parfor pb
        if (!ret.isEmpty()) {
            pfpb.setSparkEagerCacheVariables(ret);
        }
    }
    _numEvaluatedPlans++;
    LOG.debug(getOptMode() + " OPT: rewrite 'set spark eager rdd caching' - result=" + ret.size() + " (" + ProgramConverter.serializeStringCollection(ret) + ")");
}

Example 14 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralValueTypeCastScalarRead.

private static LiteralOp replaceLiteralValueTypeCastScalarRead(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // as.double/as.integer/as.boolean over scalar read - literal replacement
    if (c instanceof UnaryOp && (((UnaryOp) c).getOp() == OpOp1.CAST_AS_DOUBLE || ((UnaryOp) c).getOp() == OpOp1.CAST_AS_INT || ((UnaryOp) c).getOp() == OpOp1.CAST_AS_BOOLEAN) && c.getInput().get(0) instanceof DataOp && c.getDataType() == DataType.SCALAR) {
        Data dat = vars.get(c.getInput().get(0).getName());
        if (// required for selective constant propagation
        dat != null) {
            ScalarObject sdat = (ScalarObject) dat;
            UnaryOp cast = (UnaryOp) c;
            switch(cast.getOp()) {
                case CAST_AS_INT:
                    ret = new LiteralOp(sdat.getLongValue());
                    break;
                case CAST_AS_DOUBLE:
                    ret = new LiteralOp(sdat.getDoubleValue());
                    break;
                case CAST_AS_BOOLEAN:
                    ret = new LiteralOp(sdat.getBooleanValue());
                    break;
                default:
            }
        }
    }
    return ret;
}

Example 15 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralDataTypeCastMatrixRead.

private static LiteralOp replaceLiteralDataTypeCastMatrixRead(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // as.scalar/matrix read - literal replacement
    if (c instanceof UnaryOp && ((UnaryOp) c).getOp() == OpOp1.CAST_AS_SCALAR && c.getInput().get(0) instanceof DataOp && c.getInput().get(0).getDataType() == DataType.MATRIX) {
        Data dat = vars.get(c.getInput().get(0).getName());
        if (// required for selective constant propagation
        dat != null) {
            // cast as scalar (see VariableCPInstruction)
            MatrixObject mo = (MatrixObject) dat;
            MatrixBlock mBlock = mo.acquireRead();
            if (mBlock.getNumRows() != 1 || mBlock.getNumColumns() != 1)
                throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix of dimension (" + mBlock.getNumRows() + " x " + mBlock.getNumColumns() + ") to scalar.");
            double value = mBlock.getValue(0, 0);
            mo.release();
            // literal substitution (always double)
            ret = new LiteralOp(value);
        }
    }
    return ret;
}