Examples with MatrixReader org.apache.sysml.runtime.io.MatrixReader used on opensource projects

Example 11 with MatrixReader

use of org.apache.sysml.runtime.io.MatrixReader in project incubator-systemml by apache.

the class TransformFrameApplyTest method runTransformTest.

/**
	 * 
	 * @param rt
	 * @param ofmt
	 * @param dataset
	 */
private void runTransformTest(RUNTIME_PLATFORM rt, String ofmt, TransformType type) {
    //set runtime platform
    RUNTIME_PLATFORM rtold = rtplatform;
    boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    //set transform specification
    String SPEC = null;
    String DATASET = null;
    switch(type) {
        case RECODE:
            SPEC = SPEC1;
            DATASET = DATASET1;
            break;
        case DUMMY:
            SPEC = SPEC2;
            DATASET = DATASET1;
            break;
        case BIN:
            SPEC = SPEC3;
            DATASET = DATASET1;
            break;
        case IMPUTE:
            SPEC = SPEC4;
            DATASET = DATASET2;
            break;
        case OMIT:
            SPEC = SPEC5;
            DATASET = DATASET2;
            break;
    }
    if (!ofmt.equals("csv"))
        throw new RuntimeException("Unsupported test output format");
    try {
        getAndLoadTestConfiguration(TEST_NAME1);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
        programArgs = new String[] { "-nvargs", "DATA=" + HOME + "input/" + DATASET, "TFSPEC=" + HOME + "input/" + SPEC, "TFMTD=" + output("tfmtd"), "TFDATA=" + output("tfout"), "OFMT=" + ofmt };
        OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld;
        runTest(true, false, null, -1);
        fullDMLScriptName = HOME + TEST_NAME2 + ".dml";
        programArgs = new String[] { "-explain", "-nvargs", "DATA=" + HOME + "input/" + DATASET, "TFSPEC=" + HOME + "input/" + SPEC, // generated above
        "APPLYMTD=" + output("tfmtd"), "TFDATA=" + output("test_tfout"), "OFMT=" + ofmt };
        OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true;
        runTest(true, false, null, -1);
        //read both outputs and compare
        MatrixReader reader1 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo);
        MatrixBlock mb1 = reader1.readMatrixFromHDFS(output("tfout"), -1, -1, -1, -1, -1);
        MatrixReader reader2 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo);
        MatrixBlock mb2 = reader2.readMatrixFromHDFS(output("test_tfout"), -1, -1, -1, -1, -1);
        double[][] R1 = DataConverter.convertToDoubleMatrix(mb1);
        double[][] R2 = DataConverter.convertToDoubleMatrix(mb2);
        TestUtils.compareMatrices(R1, R2, R1.length, R1[0].length, 0);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = rtold;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld;
    }
}