Examples with RUNTIME_PLATFORM org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM used on opensource projects

Example 51 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class LeftIndexingTest method runTestLeftIndexing.

private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod indexingMethod) {
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    RUNTIME_PLATFORM oldRTP = rtplatform;
    TestConfiguration config = getTestConfiguration("LeftIndexingTest");
    try {
        if (indexingMethod != null) {
            LeftIndexingOp.FORCED_LEFT_INDEXING = indexingMethod;
        }
        if (et == ExecType.SPARK) {
            rtplatform = RUNTIME_PLATFORM.SPARK;
        } else {
            // rtplatform = (et==ExecType.MR)? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.SINGLE_NODE;
            rtplatform = RUNTIME_PLATFORM.HYBRID;
        }
        if (rtplatform == RUNTIME_PLATFORM.SPARK)
            DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009;
        // long rowstart=2, rowend=4, colstart=9, colend=10;
        /*
	        Random rand=new Random(System.currentTimeMillis());
	        rowstart=(long)(rand.nextDouble()*((double)rows))+1;
	        rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart;
	        colstart=(long)(rand.nextDouble()*((double)cols))+1;
	        colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart;
	        */
        config.addVariable("rowstart", rowstart);
        config.addVariable("rowend", rowend);
        config.addVariable("colstart", colstart);
        config.addVariable("colend", colend);
        loadTestConfiguration(config);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String LI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = LI_HOME + "LeftIndexingTest" + ".dml";
        programArgs = new String[] { "-args", input("A"), Long.toString(rows), Long.toString(cols), Long.toString(rowstart), Long.toString(rowend), Long.toString(colstart), Long.toString(colend), output("AB"), output("AC"), output("AD"), input("B"), input("C"), input("D"), Long.toString(rowend - rowstart + 1), Long.toString(colend - colstart + 1), Long.toString(cols - colstart + 1) };
        fullRScriptName = LI_HOME + "LeftIndexingTest" + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir();
        // rand.nextDouble();
        double sparsity = 1.0;
        double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("A", A, true);
        // rand.nextDouble();
        sparsity = 0.1;
        double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("B", B, true);
        // rand.nextDouble();
        sparsity = 0.5;
        double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("C", C, true);
        // rand.nextDouble();
        sparsity = 0.01;
        double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("D", D, true);
        /*
			 * Expected number of jobs:
			 * Reblock - 1 job 
			 * While loop iteration - 10 jobs
			 * Final output write - 1 job
			 */
        // boolean exceptionExpected = false;
        // int expectedNumberOfJobs = 12;
        // runTest(exceptionExpected, null, expectedNumberOfJobs);
        boolean exceptionExpected = false;
        int expectedNumberOfJobs = -1;
        runTest(true, exceptionExpected, null, expectedNumberOfJobs);
    } finally {
        rtplatform = oldRTP;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        LeftIndexingOp.FORCED_LEFT_INDEXING = null;
    }
    runRScript(true);
    for (String file : config.getOutputFiles()) {
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(file);
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS(file);
        // System.out.println(file+"-DML: "+dmlfile);
        // System.out.println(file+"-R: "+rfile);
        TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
    }
}

Example 52 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class LeftIndexingUpdateInPlaceTest method runLeftIndexingUpdateInPlaceTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param vectorM2
 */
public void runLeftIndexingUpdateInPlaceTest(boolean sparseM1, boolean sparseM2, boolean vectorM2, boolean emptyM2) {
    RUNTIME_PLATFORM oldRTP = rtplatform;
    rtplatform = RUNTIME_PLATFORM.HYBRID;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        double spM1 = sparseM1 ? sparsity1 : sparsity2;
        double spM2 = emptyM2 ? 0 : (sparseM2 ? sparsity1 : sparsity2);
        int colsM2 = vectorM2 ? cols3 : cols2;
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), input("B"), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate input data sets
        double[][] A = getRandomMatrix(rows1, cols1, -1, 1, spM1, 1234);
        writeInputMatrixWithMTD("A", A, true);
        double[][] B = getRandomMatrix(rows1, colsM2, -1, 1, spM2, 5678);
        writeInputMatrixWithMTD("B", B, true);
        // run dml and r script
        runTest(true, false, null, 1);
        runRScript(true);
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, 0, "DML", "R");
        checkDMLMetaDataFile("R", new MatrixCharacteristics(rows1, cols1, 1, 1));
    } finally {
        rtplatform = oldRTP;
    }
}

Example 53 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ToStringTest method toStringTestHelper.

protected void toStringTestHelper(RUNTIME_PLATFORM platform, String testName, String expectedOutput) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = platform;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        // Create and load test configuration
        getAndLoadTestConfiguration(testName);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testName + ".dml";
        programArgs = new String[] { "-args", output(OUTPUT_NAME) };
        // Run DML and R scripts
        runTest(true, false, null, -1);
        // Compare output strings
        String output = TestUtils.readDMLString(output(OUTPUT_NAME));
        TestUtils.compareScalars(expectedOutput, output);
    } finally {
        // Reset settings
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 54 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ZeroRowsColsMatrixTest method runEmptyMatrixTest.

private void runEmptyMatrixTest(String testname, boolean rewrites, boolean emptyRet, ExecType et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    try {
        String TEST_NAME = testname;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "recompile_runtime", "-args", String.valueOf(dim), String.valueOf(emptyRet).toUpperCase(), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = getRCmd(String.valueOf(dim), String.valueOf(emptyRet).toUpperCase(), expectedDir());
        // run Tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        // check meta data
        if (!testname.equals(TEST_NAME4))
            checkDMLMetaDataFile("R", new MatrixCharacteristics(dim, 3, 1000, 1000));
    } finally {
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 55 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ParForAccumulatorResultMergeTest method runParForAccumulatorResultMergeTest.

private void runParForAccumulatorResultMergeTest(String test, boolean init, boolean sparse, ExecType et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case CP:
            rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
            break;
        case MR:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
        default:
            throw new RuntimeException("Unsupported exec type: " + et.name());
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (et == ExecType.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String TEST_NAME = test;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", String.valueOf(rows), String.valueOf(cols), String.valueOf(init).toUpperCase(), String.valueOf(sparse).toUpperCase(), output("R") };
        fullRScriptName = HOME + TEST_NAME.substring(0, TEST_NAME.length() - 1) + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + String.valueOf(rows) + " " + String.valueOf(cols) + " " + String.valueOf(init).toUpperCase() + " " + String.valueOf(sparse).toUpperCase() + " " + expectedDir();
        // run tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}