Examples with OutputInfo org.apache.sysml.runtime.matrix.data.OutputInfo used on opensource projects

Example 56 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project systemml by apache.

the class FrameObject method writeBlobToHDFS.

@Override
protected void writeBlobToHDFS(String fname, String ofmt, int rep, FileFormatProperties fprop) throws IOException, DMLRuntimeException {
    OutputInfo oinfo = OutputInfo.stringToOutputInfo(ofmt);
    FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo, fprop);
    writer.writeFrameToHDFS(_data, fname, getNumRows(), getNumColumns());
}

Example 57 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project systemml by apache.

the class MatrixObject method writeBlobToHDFS.

/**
 * Writes in-memory matrix to HDFS in a specified format.
 */
@Override
protected void writeBlobToHDFS(String fname, String ofmt, int rep, FileFormatProperties fprop) throws IOException, DMLRuntimeException {
    long begin = 0;
    if (LOG.isTraceEnabled()) {
        LOG.trace(" Writing matrix to HDFS...  " + hashCode() + "  Path: " + fname + ", Format: " + (ofmt != null ? ofmt : "inferred from metadata"));
        begin = System.currentTimeMillis();
    }
    MetaDataFormat iimd = (MetaDataFormat) _metaData;
    if (_data != null) {
        // Get the dimension information from the metadata stored within MatrixObject
        MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
        // Write the matrix to HDFS in requested format
        OutputInfo oinfo = (ofmt != null ? OutputInfo.stringToOutputInfo(ofmt) : InputInfo.getMatchingOutputInfo(iimd.getInputInfo()));
        // note: this is only required if singlenode (due to binarycell default)
        if (oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE && (mc.getRowsPerBlock() != ConfigurationManager.getBlocksize() || mc.getColsPerBlock() != ConfigurationManager.getBlocksize())) {
            DataConverter.writeMatrixToHDFS(_data, fname, oinfo, new MatrixCharacteristics(mc.getRows(), mc.getCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), mc.getNonZeros()), rep, fprop);
        } else {
            DataConverter.writeMatrixToHDFS(_data, fname, oinfo, mc, rep, fprop);
        }
        if (LOG.isTraceEnabled())
            LOG.trace("Writing matrix to HDFS (" + fname + ") - COMPLETED... " + (System.currentTimeMillis() - begin) + " msec.");
    } else if (LOG.isTraceEnabled()) {
        LOG.trace("Writing matrix to HDFS (" + fname + ") - NOTHING TO WRITE (_data == null).");
    }
    if (DMLScript.STATISTICS)
        CacheStatistics.incrementHDFSWrites();
}

Example 58 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project systemml by apache.

the class MatrixObject method writeBlobFromRDDtoHDFS.

@Override
protected void writeBlobFromRDDtoHDFS(RDDObject rdd, String fname, String outputFormat) throws IOException, DMLRuntimeException {
    // prepare output info
    MetaDataFormat iimd = (MetaDataFormat) _metaData;
    OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo(outputFormat) : InputInfo.getMatchingOutputInfo(iimd.getInputInfo()));
    // note: the write of an RDD to HDFS might trigger
    // lazy evaluation of pending transformations.
    long newnnz = SparkExecutionContext.writeRDDtoHDFS(rdd, fname, oinfo);
    _metaData.getMatrixCharacteristics().setNonZeros(newnnz);
}

Example 59 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project systemml by apache.

the class ResultMergeLocalFile method createNewMatrixObject.

private MatrixObject createNewMatrixObject(MatrixObject output, ArrayList<MatrixObject> inMO) {
    MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
    MatrixObject moNew = new MatrixObject(_output.getValueType(), _outputFName);
    // create deep copy of metadata obj
    MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
    OutputInfo oiOld = metadata.getOutputInfo();
    InputInfo iiOld = metadata.getInputInfo();
    MatrixCharacteristics mc = new MatrixCharacteristics(mcOld);
    mc.setNonZeros(_isAccum ? -1 : computeNonZeros(output, inMO));
    MetaDataFormat meta = new MetaDataFormat(mc, oiOld, iiOld);
    moNew.setMetaData(meta);
    return moNew;
}

Example 60 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project systemml by apache.

the class CacheableData method isEqualOutputFormat.

protected boolean isEqualOutputFormat(String outputFormat) {
    boolean ret = true;
    if (outputFormat != null) {
        try {
            MetaDataFormat iimd = (MetaDataFormat) _metaData;
            OutputInfo oi1 = InputInfo.getMatchingOutputInfo(iimd.getInputInfo());
            OutputInfo oi2 = OutputInfo.stringToOutputInfo(outputFormat);
            if (oi1 != oi2)
                ret = false;
        } catch (Exception ex) {
            ret = false;
        }
    }
    return ret;
}