Examples with MetaDataFormat org.apache.sysml.runtime.matrix.MetaDataFormat used on opensource projects

Example 36 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class Recompiler method createOutputMatrix.

private static MatrixObject createOutputMatrix(long dim1, long dim2, long nnz) {
    MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
    int blksz = ConfigurationManager.getBlocksize();
    MatrixCharacteristics mc = new MatrixCharacteristics(dim1, dim2, blksz, blksz, nnz);
    MetaDataFormat meta = new MetaDataFormat(mc, null, null);
    moOut.setMetaData(meta);
    return moOut;
}

Example 37 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class Recompiler method extractDAGOutputStatistics.

public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars, boolean overwrite) {
    if (// for all writes to symbol table
    hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTWRITE) {
        String varName = hop.getName();
        if (// not existing so far
        !vars.keySet().contains(varName) || overwrite) {
            // extract matrix sizes for size propagation
            if (hop.getDataType() == DataType.MATRIX) {
                MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
                MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), hop.getNnz());
                MetaDataFormat meta = new MetaDataFormat(mc, null, null);
                mo.setMetaData(meta);
                vars.put(varName, mo);
            } else // extract scalar constants for second constant propagation
            if (hop.getDataType() == DataType.SCALAR) {
                // extract literal assignments
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof LiteralOp) {
                    ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp) hop.getInput().get(0));
                    if (constant != null)
                        vars.put(varName, constant);
                } else // extract constant variable assignments
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof DataOp) {
                    DataOp dop = (DataOp) hop.getInput().get(0);
                    String dopvarname = dop.getName();
                    if (dop.isRead() && vars.keySet().contains(dopvarname)) {
                        ScalarObject constant = (ScalarObject) vars.get(dopvarname);
                        // no clone because constant
                        vars.put(varName, constant);
                    }
                } else // extract ncol/nrow variable assignments
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof UnaryOp && (((UnaryOp) hop.getInput().get(0)).getOp() == OpOp1.NROW || ((UnaryOp) hop.getInput().get(0)).getOp() == OpOp1.NCOL)) {
                    UnaryOp uop = (UnaryOp) hop.getInput().get(0);
                    if (uop.getOp() == OpOp1.NROW && uop.getInput().get(0).getDim1() > 0)
                        vars.put(varName, new IntObject(uop.getInput().get(0).getDim1()));
                    else if (uop.getOp() == OpOp1.NCOL && uop.getInput().get(0).getDim2() > 0)
                        vars.put(varName, new IntObject(uop.getInput().get(0).getDim2()));
                } else // remove other updated scalars
                {
                    // we need to remove other updated scalars in order to ensure result
                    // correctness of recompilation w/o being too conservative
                    vars.remove(varName);
                }
            }
        } else // already existing: take largest
        {
            Data dat = vars.get(varName);
            if (dat instanceof MatrixObject) {
                MatrixObject mo = (MatrixObject) dat;
                MatrixCharacteristics mc = mo.getMatrixCharacteristics();
                if (OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros() >= 0) ? ((double) mc.getNonZeros()) / mc.getRows() / mc.getCols() : 1.0) < OptimizerUtils.estimateSize(hop.getDim1(), hop.getDim2())) {
                    // update statistics if necessary
                    mc.setDimension(hop.getDim1(), hop.getDim2());
                    mc.setNonZeros(hop.getNnz());
                }
            } else // scalar (just overwrite)
            {
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof LiteralOp) {
                    ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp) hop.getInput().get(0));
                    if (constant != null)
                        vars.put(varName, constant);
                }
            }
        }
    }
}

Example 38 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class CacheableData method readBlobFromHDFS.

protected T readBlobFromHDFS(String fname) throws IOException {
    MetaDataFormat iimd = (MetaDataFormat) _metaData;
    MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
    return readBlobFromHDFS(fname, mc.getRows(), mc.getCols());
}

Example 39 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class FrameObject method readBlobFromHDFS.

@Override
protected FrameBlock readBlobFromHDFS(String fname, long rlen, long clen) throws IOException {
    MetaDataFormat iimd = (MetaDataFormat) _metaData;
    MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
    // handle missing schema if necessary
    ValueType[] lschema = (_schema != null) ? _schema : UtilFunctions.nCopies(clen >= 1 ? (int) clen : 1, ValueType.STRING);
    // read the frame block
    FrameBlock data = null;
    try {
        FrameReader reader = FrameReaderFactory.createFrameReader(iimd.getInputInfo(), getFileFormatProperties());
        data = reader.readFrameFromHDFS(fname, lschema, mc.getRows(), mc.getCols());
    } catch (DMLRuntimeException ex) {
        throw new IOException(ex);
    }
    // sanity check correct output
    if (data == null)
        throw new IOException("Unable to load frame from file: " + fname);
    return data;
}

Example 40 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class FrameObject method readBlobFromRDD.

@Override
protected FrameBlock readBlobFromRDD(RDDObject rdd, MutableBoolean status) throws IOException {
    // note: the read of a frame block from an RDD might trigger
    // lazy evaluation of pending transformations.
    RDDObject lrdd = rdd;
    // prepare return status (by default only collect)
    status.setValue(false);
    MetaDataFormat iimd = (MetaDataFormat) _metaData;
    MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
    int rlen = (int) mc.getRows();
    int clen = (int) mc.getCols();
    // handle missing schema if necessary
    ValueType[] lschema = (_schema != null) ? _schema : UtilFunctions.nCopies(clen >= 1 ? (int) clen : 1, ValueType.STRING);
    FrameBlock fb = null;
    try {
        // prevent unnecessary collect through rdd checkpoint
        if (rdd.allowsShortCircuitCollect()) {
            lrdd = (RDDObject) rdd.getLineageChilds().get(0);
        }
        // collect frame block from binary block RDD
        fb = SparkExecutionContext.toFrameBlock(lrdd, lschema, rlen, clen);
    } catch (DMLRuntimeException ex) {
        throw new IOException(ex);
    }
    // sanity check correct output
    if (fb == null)
        throw new IOException("Unable to load frame from rdd.");
    return fb;
}