Examples with FrameReaderTextCSVParallel org.apache.sysml.runtime.io.FrameReaderTextCSVParallel used on opensource projects

Example 1 with FrameReaderTextCSVParallel

use of org.apache.sysml.runtime.io.FrameReaderTextCSVParallel in project incubator-systemml by apache.

the class TransformCSVFrameEncodeReadTest method runTransformTest.

/**
	 * 
	 * @param rt
	 * @param ofmt
	 * @param dataset
	 */
private void runTransformTest(RUNTIME_PLATFORM rt, String ofmt, boolean subset, boolean parRead) {
    //set runtime platform
    RUNTIME_PLATFORM rtold = rtplatform;
    boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    if (!ofmt.equals("csv"))
        throw new RuntimeException("Unsupported test output format");
    try {
        getAndLoadTestConfiguration(TEST_NAME1);
        String HOME = SCRIPT_DIR + TEST_DIR;
        int nrows = subset ? 4 : 13;
        fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
        programArgs = new String[] { "-explain", "-stats", "-args", HOME + "input/" + DATASET, String.valueOf(nrows), output("R") };
        OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true;
        runTest(true, false, null, -1);
        //read input/output and compare
        FrameReader reader2 = parRead ? new FrameReaderTextCSVParallel(new CSVFileFormatProperties()) : new FrameReaderTextCSV(new CSVFileFormatProperties());
        FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L);
        System.out.println(DataConverter.toString(fb2));
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = rtold;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld;
    }
}