Examples with MatrixWriter org.apache.sysml.runtime.io.MatrixWriter used on opensource projects

Example 1 with MatrixWriter

use of org.apache.sysml.runtime.io.MatrixWriter in project incubator-systemml by apache.

the class DataConverter method writeMatrixToHDFS.

public static void writeMatrixToHDFS(MatrixBlock mat, String dir, OutputInfo outputinfo, MatrixCharacteristics mc, int replication, FileFormatProperties formatProperties) throws IOException {
    try {
        MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(outputinfo, replication, formatProperties);
        writer.writeMatrixToHDFS(mat, dir, mc.getRows(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock(), mc.getNonZeros());
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 2 with MatrixWriter

use of org.apache.sysml.runtime.io.MatrixWriter in project incubator-systemml by apache.

the class FrameConverterTest method runMatrixConverterAndVerify.

/**
 * @param schema
 * @param A
 * @param type
 * @param iinfo
 * @param oinfo
 * @param instType
 */
private void runMatrixConverterAndVerify(ValueType[] schema, double[][] A, ConvType type, InputInfo iinfo, OutputInfo oinfo) throws IOException {
    try {
        MatrixCharacteristics mcMatrix = new MatrixCharacteristics(rows, schema.length, 1000, 1000, 0);
        MatrixCharacteristics mcFrame = new MatrixCharacteristics(rows, schema.length, -1, -1, -1);
        MatrixBlock matrixBlock1 = null;
        FrameBlock frame1 = null;
        if (type == ConvType.MAT2BIN) {
            // initialize the matrix (dense) data.
            matrixBlock1 = new MatrixBlock(rows, schema.length, false);
            matrixBlock1.init(A, rows, schema.length);
            // write matrix data to hdfs
            MatrixWriter matWriter = MatrixWriterFactory.createMatrixWriter(oinfo);
            matWriter.writeMatrixToHDFS(matrixBlock1, input("A"), rows, schema.length, mcMatrix.getRowsPerBlock(), mcMatrix.getColsPerBlock(), mcMatrix.getNonZeros());
        } else {
            // initialize the frame data.
            frame1 = new FrameBlock(schema);
            initFrameData(frame1, A, schema);
            // write frame data to hdfs
            FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
            writer.writeFrameToHDFS(frame1, input("A"), rows, schema.length);
        }
        // run converter under test
        runConverter(type, mcFrame, mcMatrix, Arrays.asList(schema), input("A"), output("B"));
        if (type == ConvType.MAT2BIN) {
            // read frame data from hdfs
            FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
            FrameBlock frame2 = reader.readFrameFromHDFS(output("B"), rows, schema.length);
            // verify input and output frame/matrix
            verifyFrameMatrixData(frame2, matrixBlock1);
        } else {
            // read matrix data from hdfs
            MatrixReader matReader = MatrixReaderFactory.createMatrixReader(iinfo);
            MatrixBlock matrixBlock2 = matReader.readMatrixFromHDFS(output("B"), rows, schema.length, mcMatrix.getRowsPerBlock(), mcMatrix.getColsPerBlock(), mcMatrix.getNonZeros());
            // verify input and output frame/matrix
            verifyFrameMatrixData(frame1, matrixBlock2);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        MapReduceTool.deleteFileIfExistOnHDFS(input("A"));
        MapReduceTool.deleteFileIfExistOnHDFS(output("B"));
    }
}

Example 3 with MatrixWriter

use of org.apache.sysml.runtime.io.MatrixWriter in project systemml by apache.

the class FrameConverterTest method runMatrixConverterAndVerify.

/**
 * @param schema
 * @param A
 * @param type
 * @param iinfo
 * @param oinfo
 * @param instType
 */
private void runMatrixConverterAndVerify(ValueType[] schema, double[][] A, ConvType type, InputInfo iinfo, OutputInfo oinfo) throws IOException {
    try {
        MatrixCharacteristics mcMatrix = new MatrixCharacteristics(rows, schema.length, 1000, 1000, 0);
        MatrixCharacteristics mcFrame = new MatrixCharacteristics(rows, schema.length, -1, -1, -1);
        MatrixBlock matrixBlock1 = null;
        FrameBlock frame1 = null;
        if (type == ConvType.MAT2BIN) {
            // initialize the matrix (dense) data.
            matrixBlock1 = new MatrixBlock(rows, schema.length, false);
            matrixBlock1.init(A, rows, schema.length);
            // write matrix data to hdfs
            MatrixWriter matWriter = MatrixWriterFactory.createMatrixWriter(oinfo);
            matWriter.writeMatrixToHDFS(matrixBlock1, input("A"), rows, schema.length, mcMatrix.getRowsPerBlock(), mcMatrix.getColsPerBlock(), mcMatrix.getNonZeros());
        } else {
            // initialize the frame data.
            frame1 = new FrameBlock(schema);
            initFrameData(frame1, A, schema);
            // write frame data to hdfs
            FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
            writer.writeFrameToHDFS(frame1, input("A"), rows, schema.length);
        }
        // run converter under test
        runConverter(type, mcFrame, mcMatrix, Arrays.asList(schema), input("A"), output("B"));
        if (type == ConvType.MAT2BIN) {
            // read frame data from hdfs
            FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
            FrameBlock frame2 = reader.readFrameFromHDFS(output("B"), rows, schema.length);
            // verify input and output frame/matrix
            verifyFrameMatrixData(frame2, matrixBlock1);
        } else {
            // read matrix data from hdfs
            MatrixReader matReader = MatrixReaderFactory.createMatrixReader(iinfo);
            MatrixBlock matrixBlock2 = matReader.readMatrixFromHDFS(output("B"), rows, schema.length, mcMatrix.getRowsPerBlock(), mcMatrix.getColsPerBlock(), mcMatrix.getNonZeros());
            // verify input and output frame/matrix
            verifyFrameMatrixData(frame1, matrixBlock2);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        MapReduceTool.deleteFileIfExistOnHDFS(input("A"));
        MapReduceTool.deleteFileIfExistOnHDFS(output("B"));
    }
}

Example 4 with MatrixWriter

use of org.apache.sysml.runtime.io.MatrixWriter in project incubator-systemml by apache.

the class TransformReadMetaTest method runTransformReadMetaTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 * @throws IOException 
	 * @throws DMLRuntimeException 
	 */
private void runTransformReadMetaTest(RUNTIME_PLATFORM rt, String ofmt, String delim) throws IOException, DMLRuntimeException {
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String testname = delim.equals(",") ? TEST_NAME1 : TEST_NAME2;
        getAndLoadTestConfiguration(testname);
        //generate input data
        double[][] X = DataConverter.convertToDoubleMatrix(MatrixBlock.seqOperations(0.5, rows / 2, 0.5).appendOperations(MatrixBlock.seqOperations(0.5, rows / 2, 0.5), new MatrixBlock()));
        MatrixBlock mbX = DataConverter.convertToMatrixBlock(X);
        CSVFileFormatProperties fprops = new CSVFileFormatProperties(false, delim, false);
        MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo, 1, fprops);
        writer.writeMatrixToHDFS(mbX, input("X"), rows, 2, -1, -1, -1);
        //read specs transform X and Y
        String specX = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR + TEST_DIR + SPEC_X);
        fullDMLScriptName = SCRIPT_DIR + TEST_DIR + testname + ".dml";
        programArgs = new String[] { "-args", input("X"), specX, output("M1"), output("M"), ofmt, delim };
        //run test
        runTest(true, false, null, -1);
        //compare meta data frames
        InputInfo iinfo = InputInfo.stringExternalToInputInfo(ofmt);
        FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
        FrameBlock mExpected = TfMetaUtils.readTransformMetaDataFromFile(specX, output("M1"), delim);
        FrameBlock mRet = reader.readFrameFromHDFS(output("M"), rows, 2);
        for (int i = 0; i < rows; i++) for (int j = 0; j < 2; j++) {
            Assert.assertTrue("Wrong result: " + mRet.get(i, j) + ".", UtilFunctions.compareTo(ValueType.STRING, mExpected.get(i, j), mRet.get(i, j)) == 0);
        }
    } catch (Exception ex) {
        throw new IOException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 5 with MatrixWriter

use of org.apache.sysml.runtime.io.MatrixWriter in project incubator-systemml by apache.

the class TransformAndApplyTest method runTransformAndApplyTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 * @throws IOException 
	 * @throws DMLRuntimeException 
	 */
private void runTransformAndApplyTest(RUNTIME_PLATFORM rt, String ofmt) throws IOException, DMLRuntimeException {
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        getAndLoadTestConfiguration(TEST_NAME1);
        //generate input data
        double[][] X = DataConverter.convertToDoubleMatrix(MatrixBlock.seqOperations(0.5, rows / 2, 0.5).appendOperations(MatrixBlock.seqOperations(0.5, rows / 2, 0.5), new MatrixBlock()));
        double[][] Y = DataConverter.convertToDoubleMatrix(MatrixBlock.seqOperations(rows / 2, 0.5, -0.5));
        //write inputs
        MatrixBlock mbX = DataConverter.convertToMatrixBlock(X);
        MatrixBlock mbY = DataConverter.convertToMatrixBlock(Y);
        MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo);
        writer.writeMatrixToHDFS(mbX, input("X"), rows, 2, -1, -1, -1);
        writer.writeMatrixToHDFS(mbY, input("Y"), rows, 1, -1, -1, -1);
        //read specs transform X and Y
        String specX = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR + TEST_DIR + SPEC_X);
        String specY = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR + TEST_DIR + SPEC_Y);
        fullDMLScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME1 + ".dml";
        programArgs = new String[] { "-args", input("X"), input("Y"), specX, specY, output("M1"), output("M2"), output("R1"), output("R2") };
        //run test
        runTest(true, false, null, -1);
        //compare matrices (values recoded to identical codes)
        HashMap<CellIndex, Double> dml1 = readDMLMatrixFromHDFS("R1");
        HashMap<CellIndex, Double> dml2 = readDMLMatrixFromHDFS("R2");
        double[][] R1 = TestUtils.convertHashMapToDoubleArray(dml1);
        double[][] R2 = TestUtils.convertHashMapToDoubleArray(dml2);
        for (int i = 0; i < rows; i++) {
            Assert.assertEquals("Output values don't match: " + R1[i][0] + " vs " + R2[i][0], new Double(R1[i][0]), new Double(R2[rows - i - 1][0]));
        }
    } catch (Exception ex) {
        throw new IOException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}