Examples with WriterTextCSV org.apache.sysml.runtime.io.WriterTextCSV used on opensource projects

Example 1 with WriterTextCSV

use of org.apache.sysml.runtime.io.WriterTextCSV in project incubator-systemml by apache.

the class VariableCPInstruction method writeCSVFile.

/**
 * Helper function to write CSV files to HDFS.
 *
 * @param ec execution context
 * @param fname file name
 */
private void writeCSVFile(ExecutionContext ec, String fname) {
    MatrixObject mo = ec.getMatrixObject(getInput1().getName());
    String outFmt = "csv";
    if (mo.isDirty()) {
        // there exist data computed in CP that is not backed up on HDFS
        // i.e., it is either in-memory or in evicted space
        mo.exportData(fname, outFmt, _formatProperties);
    } else {
        try {
            OutputInfo oi = ((MetaDataFormat) mo.getMetaData()).getOutputInfo();
            MatrixCharacteristics mc = ((MetaDataFormat) mo.getMetaData()).getMatrixCharacteristics();
            if (oi == OutputInfo.CSVOutputInfo) {
                WriterTextCSV writer = new WriterTextCSV((CSVFileFormatProperties) _formatProperties);
                writer.addHeaderToCSV(mo.getFileName(), fname, mc.getRows(), mc.getCols());
            } else if (oi == OutputInfo.BinaryBlockOutputInfo || oi == OutputInfo.TextCellOutputInfo) {
                mo.exportData(fname, outFmt, _formatProperties);
            } else {
                throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into CSV format.");
            }
            // Write Metadata file
            MapReduceTool.writeMetaDataFile(fname + ".mtd", mo.getValueType(), mc, OutputInfo.CSVOutputInfo, _formatProperties);
        } catch (IOException e) {
            throw new DMLRuntimeException(e);
        }
    }
}

Example 2 with WriterTextCSV

use of org.apache.sysml.runtime.io.WriterTextCSV in project systemml by apache.

the class VariableCPInstruction method writeCSVFile.

/**
 * Helper function to write CSV files to HDFS.
 *
 * @param ec execution context
 * @param fname file name
 */
private void writeCSVFile(ExecutionContext ec, String fname) {
    MatrixObject mo = ec.getMatrixObject(getInput1().getName());
    String outFmt = "csv";
    if (mo.isDirty()) {
        // there exist data computed in CP that is not backed up on HDFS
        // i.e., it is either in-memory or in evicted space
        mo.exportData(fname, outFmt, _formatProperties);
    } else {
        try {
            OutputInfo oi = ((MetaDataFormat) mo.getMetaData()).getOutputInfo();
            MatrixCharacteristics mc = ((MetaDataFormat) mo.getMetaData()).getMatrixCharacteristics();
            if (oi == OutputInfo.CSVOutputInfo) {
                WriterTextCSV writer = new WriterTextCSV((CSVFileFormatProperties) _formatProperties);
                writer.addHeaderToCSV(mo.getFileName(), fname, mc.getRows(), mc.getCols());
            } else if (oi == OutputInfo.BinaryBlockOutputInfo || oi == OutputInfo.TextCellOutputInfo) {
                mo.exportData(fname, outFmt, _formatProperties);
            } else {
                throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into CSV format.");
            }
            // Write Metadata file
            MapReduceTool.writeMetaDataFile(fname + ".mtd", mo.getValueType(), mc, OutputInfo.CSVOutputInfo, _formatProperties);
        } catch (IOException e) {
            throw new DMLRuntimeException(e);
        }
    }
}