Examples with FrameWriter org.apache.sysml.runtime.io.FrameWriter used on opensource projects

Example 6 with FrameWriter

use of org.apache.sysml.runtime.io.FrameWriter in project incubator-systemml by apache.

the class JMLCInputStreamReadTest method runJMLCInputStreamReadTest.

private void runJMLCInputStreamReadTest(DataType dt, boolean sparse, String format, boolean metaData) throws IOException {
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    loadTestConfiguration(config);
    // generate inputs
    OutputInfo oinfo = format.equals("csv") ? OutputInfo.CSVOutputInfo : OutputInfo.TextCellOutputInfo;
    double[][] data = TestUtils.round(getRandomMatrix(rows, cols, 0.51, 7.49, sparse ? sparsity2 : sparsity1, 7));
    Connection conn = new Connection();
    try {
        if (dt == DataType.MATRIX) {
            // write input matrix
            MatrixBlock mb = DataConverter.convertToMatrixBlock(data);
            MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(oinfo);
            writer.writeMatrixToHDFS(mb, output("X"), rows, cols, -1, -1, -1);
            // read matrix from input stream
            FileInputStream fis = new FileInputStream(output("X"));
            double[][] data2 = conn.convertToDoubleMatrix(fis, rows, cols, format);
            fis.close();
            // compare matrix result
            TestUtils.compareMatrices(data, data2, rows, cols, 0);
        } else if (dt == DataType.FRAME) {
            // write input frame
            String[][] fdata = FrameTransformTest.createFrameData(data, "V");
            // test quoted tokens w/ inner quotes
            fdata[3][1] = "\"ab\"\"cdef\"";
            if (format.equals("csv"))
                // test delimiter and space tokens
                fdata[7][2] = "\"a,bc def\"";
            FrameBlock fb = DataConverter.convertToFrameBlock(fdata);
            if (metaData) {
                fb.setColumnNames(IntStream.range(0, cols).mapToObj(i -> "CC" + i).collect(Collectors.toList()).toArray(new String[0]));
            }
            FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
            writer.writeFrameToHDFS(fb, output("X"), rows, cols);
            // read frame from input stream
            FileInputStream fis = new FileInputStream(output("X"));
            String[][] fdata2 = conn.convertToStringFrame(fis, rows, cols, format);
            fis.close();
            // compare frame result
            TestUtils.compareFrames(fdata, fdata2, rows, cols);
        } else {
            throw new IOException("Unsupported data type: " + dt.name());
        }
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        MapReduceTool.deleteFileIfExistOnHDFS(output("X"));
        IOUtilFunctions.closeSilently(conn);
    }
}

Example 7 with FrameWriter

use of org.apache.sysml.runtime.io.FrameWriter in project incubator-systemml by apache.

the class FrameMatrixCastingTest method writeMatrixOrFrameInput.

private static void writeMatrixOrFrameInput(String fname, double[][] A, int rows, int cols, DataType dt, ValueType vt) throws IOException {
    int blksize = ConfigurationManager.getBlocksize();
    // write input data
    if (dt == DataType.FRAME) {
        FrameBlock fb = DataConverter.convertToFrameBlock(DataConverter.convertToMatrixBlock(A), vt);
        FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.BinaryBlockOutputInfo);
        writer.writeFrameToHDFS(fb, fname, rows, cols);
    } else {
        MatrixBlock mb = DataConverter.convertToMatrixBlock(A);
        MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.BinaryBlockOutputInfo);
        writer.writeMatrixToHDFS(mb, fname, (long) rows, (long) cols, blksize, blksize, -1);
    }
    // write meta data
    MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, blksize, blksize);
    MapReduceTool.writeMetaDataFile(fname + ".mtd", vt, null, dt, mc, OutputInfo.BinaryBlockOutputInfo);
}

Example 8 with FrameWriter

use of org.apache.sysml.runtime.io.FrameWriter in project incubator-systemml by apache.

the class FrameMatrixReblockTest method writeFrameInput.

private static void writeFrameInput(String fname, String ofmt, double[][] frame, int rows, int cols) throws IOException {
    MatrixBlock mb = DataConverter.convertToMatrixBlock(frame);
    FrameBlock fb = DataConverter.convertToFrameBlock(mb);
    // write input data
    FrameWriter writer = FrameWriterFactory.createFrameWriter(InputInfo.getMatchingOutputInfo(InputInfo.stringExternalToInputInfo(ofmt)));
    writer.writeFrameToHDFS(fb, fname, rows, cols);
}

Example 9 with FrameWriter

use of org.apache.sysml.runtime.io.FrameWriter in project incubator-systemml by apache.

the class FrameReadWriteTest method writeAndVerifyData.

void writeAndVerifyData(OutputInfo oinfo, FrameBlock frame1, FrameBlock frame2, CSVFileFormatProperties fprop) throws IOException {
    String fname1 = SCRIPT_DIR + TEST_DIR + "/frameData1";
    String fname2 = SCRIPT_DIR + TEST_DIR + "/frameData2";
    // Create reader/writer
    FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo, fprop);
    FrameReader reader = FrameReaderFactory.createFrameReader(OutputInfo.getMatchingInputInfo(oinfo), fprop);
    // Write frame data to disk
    writer.writeFrameToHDFS(frame1, fname1, frame1.getNumRows(), frame1.getNumColumns());
    writer.writeFrameToHDFS(frame2, fname2, frame2.getNumRows(), frame2.getNumColumns());
    // Read frame data from disk
    FrameBlock frame1Read = reader.readFrameFromHDFS(fname1, frame1.getSchema(), frame1.getNumRows(), frame1.getNumColumns());
    FrameBlock frame2Read = reader.readFrameFromHDFS(fname2, frame2.getSchema(), frame2.getNumRows(), frame2.getNumColumns());
    // Verify that data read with original frames
    verifyFrameData(frame1, frame1Read);
    verifyFrameData(frame2, frame2Read);
    MapReduceTool.deleteFileIfExistOnHDFS(fname1);
    MapReduceTool.deleteFileIfExistOnHDFS(fname2);
}

Example 10 with FrameWriter

use of org.apache.sysml.runtime.io.FrameWriter in project incubator-systemml by apache.

the class FrameObject method writeBlobToHDFS.

@Override
protected void writeBlobToHDFS(String fname, String ofmt, int rep, FileFormatProperties fprop) throws IOException, DMLRuntimeException {
    OutputInfo oinfo = OutputInfo.stringToOutputInfo(ofmt);
    FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo, fprop);
    writer.writeFrameToHDFS(_data, fname, getNumRows(), getNumColumns());
}