Examples with RUNTIME_PLATFORM org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM used on opensource projects

Example 21 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class AppendMatrixTest method commonAppendTest.

/**
 * @param platform
 * @param rows
 * @param cols1
 * @param cols2
 * @param sparse
 */
public void commonAppendTest(RUNTIME_PLATFORM platform, int rows, int cols1, int cols2, boolean sparse, AppendMethod forcedAppendMethod) {
    TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
    RUNTIME_PLATFORM prevPlfm = rtplatform;
    double sparsity = (sparse) ? sparsity2 : sparsity1;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    try {
        if (forcedAppendMethod != null) {
            BinaryOp.FORCED_APPEND_METHOD = forcedAppendMethod;
        }
        rtplatform = platform;
        if (rtplatform == RUNTIME_PLATFORM.SPARK)
            DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        config.addVariable("rows", rows);
        config.addVariable("cols", cols1);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), Long.toString(rows), Long.toString(cols1), input("B"), Long.toString(cols2), output("C") };
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        Random rand = new Random(System.currentTimeMillis());
        double[][] A = getRandomMatrix(rows, cols1, min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("A", A, true);
        sparsity = rand.nextDouble();
        double[][] B = getRandomMatrix(rows, cols2, min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("B", B, true);
        boolean exceptionExpected = false;
        int expectedCompiledMRJobs = (rtplatform == RUNTIME_PLATFORM.HADOOP) ? 2 : 1;
        int expectedExecutedMRJobs = (rtplatform == RUNTIME_PLATFORM.HADOOP) ? 2 : 0;
        runTest(true, exceptionExpected, null, expectedCompiledMRJobs);
        runRScript(true);
        Assert.assertEquals("Wrong number of executed MR jobs.", expectedExecutedMRJobs, Statistics.getNoOfExecutedMRJobs());
        for (String file : config.getOutputFiles()) {
            HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(file);
            HashMap<CellIndex, Double> rfile = readRMatrixFromFS(file);
            TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
        }
    } finally {
        // reset execution platform
        rtplatform = prevPlfm;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        BinaryOp.FORCED_APPEND_METHOD = null;
    }
}

Example 22 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class BinUaggChainTest method runBinUaggTest.

// ----------------------
/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runBinUaggTest(String testname, boolean singleBlock, boolean sparse, ExecType instType) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String TEST_NAME = testname;
        getAndLoadTestConfiguration(TEST_NAME);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual datasets
        double[][] A = getRandomMatrix(rows, singleBlock ? cols1 : cols2, -1, 1, sparse ? sparsity2 : sparsity1, 7);
        writeInputMatrixWithMTD("A", A, true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        // check compiled/executed jobs
        if (rtplatform != RUNTIME_PLATFORM.SPARK) {
            int expectedNumCompiled = (singleBlock) ? 1 : 3;
            int expectedNumExecuted = (singleBlock) ? 1 : 3;
            checkNumCompiledMRJobs(expectedNumCompiled);
            checkNumExecutedMRJobs(expectedNumExecuted);
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 23 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class CentralMomentTest method runCentralMomentTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runCentralMomentTest(int order, boolean sparse, ExecType et) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        getAndLoadTestConfiguration(TEST_NAME);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), Integer.toString(order), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + order + " " + expectedDir();
        // generate actual dataset (always dense because values <=0 invalid)
        double sparsitya = sparse ? sparsity2 : sparsity1;
        double[][] A = getRandomMatrix(rows, 1, 1, maxVal, sparsitya, 7);
        writeInputMatrixWithMTD("A", A, true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 24 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class RightIndexingMatrixTest method runRightIndexingTest.

public void runRightIndexingTest(ExecType et, boolean sparse, long rl, long ru, long cl, long cu) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        double sparsity = sparse ? sparsity2 : sparsity1;
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        config.addVariable("rowstart", rl);
        config.addVariable("rowend", ru);
        config.addVariable("colstart", cl);
        config.addVariable("colend", cu);
        loadTestConfiguration(config);
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), Long.toString(rows), Long.toString(cols), Long.toString(rl), Long.toString(ru), Long.toString(cl), Long.toString(cu), output("B"), output("C"), output("D") };
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rl + " " + ru + " " + cl + " " + cu + " " + expectedDir();
        double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("A", A, true);
        // run tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare results
        for (String file : config.getOutputFiles()) {
            HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(file);
            HashMap<CellIndex, Double> rfile = readRMatrixFromFS(file);
            TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 25 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class RowBatchRightIndexingTest method runRightIndexingTest.

/**
 * @param et
 * @param sparse
 */
public void runRightIndexingTest(ExecType et, boolean sparse) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        double sparsity = sparse ? sparsity2 : sparsity1;
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), output("B") };
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 23);
        writeInputMatrixWithMTD("A", A, true);
        // run tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare output aggregate
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}