Examples with MatrixObject org.apache.sysml.runtime.controlprogram.caching.MatrixObject used on opensource projects

Example 66 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class InterProceduralAnalysis method createOutputMatrix.

private static MatrixObject createOutputMatrix(long dim1, long dim2, long nnz) {
    MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
    MatrixCharacteristics mc = new MatrixCharacteristics(dim1, dim2, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), nnz);
    MetaDataFormat meta = new MetaDataFormat(mc, null, null);
    moOut.setMetaData(meta);
    return moOut;
}

Example 67 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class InterProceduralAnalysis method extractFunctionCallUnknownReturnStatistics.

private static void extractFunctionCallUnknownReturnStatistics(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callVars) {
    ArrayList<DataIdentifier> foutputOps = fstmt.getOutputParams();
    String[] outputVars = fop.getOutputVariableNames();
    String fkey = fop.getFunctionKey();
    try {
        for (int i = 0; i < foutputOps.size(); i++) {
            DataIdentifier di = foutputOps.get(i);
            // name in calling program
            String pvarname = outputVars[i];
            if (di.getDataType() == DataType.MATRIX) {
                MatrixObject moOut = createOutputMatrix(-1, -1, -1);
                callVars.put(pvarname, moOut);
            }
        }
    } catch (Exception ex) {
        throw new HopsException("Failed to extract output statistics of function " + fkey + ".", ex);
    }
}

Example 68 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralFullUnaryAggregate.

private static LiteralOp replaceLiteralFullUnaryAggregate(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // full unary aggregate w/ matrix less than 10^6 cells
    if (c instanceof AggUnaryOp && isReplaceableUnaryAggregate((AggUnaryOp) c) && c.getInput().get(0) instanceof DataOp && vars.keySet().contains(c.getInput().get(0).getName())) {
        Hop data = c.getInput().get(0);
        MatrixObject mo = (MatrixObject) vars.get(data.getName());
        // dimensions might not have been updated during recompile
        if (mo.getNumRows() * mo.getNumColumns() < REPLACE_LITERALS_MAX_MATRIX_SIZE) {
            MatrixBlock mBlock = mo.acquireRead();
            double value = replaceUnaryAggregate((AggUnaryOp) c, mBlock);
            mo.release();
            // literal substitution (always double)
            ret = new LiteralOp(value);
        }
    }
    return ret;
}

Example 69 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralDataTypeCastMatrixRead.

private static LiteralOp replaceLiteralDataTypeCastMatrixRead(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // as.scalar/matrix read - literal replacement
    if (c instanceof UnaryOp && ((UnaryOp) c).getOp() == OpOp1.CAST_AS_SCALAR && c.getInput().get(0) instanceof DataOp && c.getInput().get(0).getDataType() == DataType.MATRIX) {
        Data dat = vars.get(c.getInput().get(0).getName());
        if (// required for selective constant propagation
        dat != null) {
            // cast as scalar (see VariableCPInstruction)
            MatrixObject mo = (MatrixObject) dat;
            MatrixBlock mBlock = mo.acquireRead();
            if (mBlock.getNumRows() != 1 || mBlock.getNumColumns() != 1)
                throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix of dimension (" + mBlock.getNumRows() + " x " + mBlock.getNumColumns() + ") to scalar.");
            double value = mBlock.getValue(0, 0);
            mo.release();
            // literal substitution (always double)
            ret = new LiteralOp(value);
        }
    }
    return ret;
}

Example 70 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class Recompiler method checkCPReblock.

/**
 * Returns true iff (1) all instruction are reblock instructions and (2) all
 * individual reblock operations fit in the current memory budget.
 *
 * @param inst instruction
 * @param inputs the inputs
 * @return return true if and only if all instructions are reblock instructions and all
 * individual reblock oeprations fir in the current memory budget.
 * @throws IOException if IOException occurs
 */
public static boolean checkCPReblock(MRJobInstruction inst, MatrixObject[] inputs) throws IOException {
    boolean ret = true;
    boolean localMode = InfrastructureAnalyzer.isLocalMode();
    // check only shuffle inst
    String rdInst = inst.getIv_randInstructions();
    String rrInst = inst.getIv_recordReaderInstructions();
    String mapInst = inst.getIv_instructionsInMapper();
    String aggInst = inst.getIv_aggInstructions();
    String otherInst = inst.getIv_otherInstructions();
    if ((rdInst != null && rdInst.length() > 0) || (rrInst != null && rrInst.length() > 0) || (mapInst != null && mapInst.length() > 0) || (aggInst != null && aggInst.length() > 0) || (otherInst != null && otherInst.length() > 0)) {
        ret = false;
    }
    // check only reblock inst
    if (ret) {
        String shuffleInst = inst.getIv_shuffleInstructions();
        String[] instParts = shuffleInst.split(Lop.INSTRUCTION_DELIMITOR);
        for (String rblk : instParts) if (!InstructionUtils.getOpCode(rblk).equals(ReBlock.OPCODE) && !InstructionUtils.getOpCode(rblk).equals(CSVReBlock.OPCODE)) {
            ret = false;
            break;
        }
    }
    // counter-productive because any export from CP would reintroduce the empty blocks)
    if (ret) {
        String shuffleInst = inst.getIv_shuffleInstructions();
        String[] instParts = shuffleInst.split(Lop.INSTRUCTION_DELIMITOR);
        for (String rblk : instParts) if (InstructionUtils.getOpCode(rblk).equals(ReBlock.OPCODE) && // no output of empty blocks
        rblk.endsWith("false")) {
            ret = false;
            break;
        }
    }
    // check recompile memory budget
    if (ret) {
        for (MatrixObject mo : inputs) {
            long rows = mo.getNumRows();
            long cols = mo.getNumColumns();
            // however, we do a conservative check with the CSV filesize
            if (rows == -1 || cols == -1) {
                Path path = new Path(mo.getFileName());
                long size = MapReduceTool.getFilesizeOnHDFS(path);
                if (size > CP_CSV_REBLOCK_UNKNOWN_THRESHOLD_SIZE || CP_CSV_REBLOCK_UNKNOWN_THRESHOLD_SIZE > OptimizerUtils.getLocalMemBudget()) {
                    ret = false;
                    break;
                }
            } else // default case (known dimensions)
            {
                long nnz = mo.getNnz();
                double sp = OptimizerUtils.getSparsity(rows, cols, nnz);
                double mem = MatrixBlock.estimateSizeInMemory(rows, cols, sp);
                if (!OptimizerUtils.isValidCPDimensions(rows, cols) || !OptimizerUtils.isValidCPMatrixSize(rows, cols, sp) || mem >= OptimizerUtils.getLocalMemBudget()) {
                    ret = false;
                    break;
                }
            }
        }
    }
    // NOTE: this does not apply to local mode because there text read single-threaded as well
    if (ret && !localMode) {
        for (MatrixObject mo : inputs) {
            MetaDataFormat iimd = (MetaDataFormat) mo.getMetaData();
            if ((iimd.getInputInfo() == InputInfo.TextCellInputInfo || iimd.getInputInfo() == InputInfo.MatrixMarketInputInfo || iimd.getInputInfo() == InputInfo.CSVInputInfo || iimd.getInputInfo() == InputInfo.BinaryCellInputInfo) && !mo.isDirty()) {
                // get file size on hdfs (as indicator for estimated read time)
                Path path = new Path(mo.getFileName());
                long fileSize = MapReduceTool.getFilesizeOnHDFS(path);
                // compute cp reblock size threshold based on available parallelism
                long cpThreshold = CP_REBLOCK_THRESHOLD_SIZE * OptimizerUtils.getParallelTextReadParallelism();
                if (fileSize > cpThreshold) {
                    ret = false;
                    break;
                }
            }
        }
    }
    return ret;
}