Examples with RUNTIME_PLATFORM org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM used on opensource projects

Example 61 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class MultipleReadsIPATest method runMultipleReadsTest.

/**
 * @param condition
 * @param branchRemoval
 * @param IPA
 */
private void runMultipleReadsTest(ExecType et, boolean IPA) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("X1"), Integer.toString(rows1), Integer.toString(cols1), input("X2"), Integer.toString(rows2), Integer.toString(cols2), output("X") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        rtplatform = (et == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
        OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
        double[][] X1 = getRandomMatrix(rows1, cols1, -1, 1, 1.0d, 7);
        writeInputMatrix("X1", X1, true);
        double[][] X2 = getRandomMatrix(rows2, cols2, -1, 1, 1.0d, 7);
        writeInputMatrix("X2", X2, true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("X");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("X");
        TestUtils.compareMatrices(dmlfile, rfile, 0, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
        OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
    }
}

Example 62 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class MultipleOrderByColsTest method runOrderTest.

private void runOrderTest(String testname, boolean sparse, boolean desc, boolean ixret, ExecType et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String TEST_NAME = testname;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-stats", "-args", input("A"), String.valueOf(desc).toUpperCase(), String.valueOf(ixret).toUpperCase(), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + String.valueOf(desc).toUpperCase() + " " + String.valueOf(ixret).toUpperCase() + " " + expectedDir();
        // with rounding for duplicates
        double sparsity = (sparse) ? sparsity2 : sparsity1;
        double[][] A = TestUtils.round(getRandomMatrix(rows, cols, -10, 10, sparsity, 7));
        writeInputMatrixWithMTD("A", A, true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        // check for applied rewrite
        if (testname.equals(TEST_NAME2) && !ixret)
            Assert.assertTrue(Statistics.getCPHeavyHitterCount("rsort") == 1);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 63 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class Conv2DBackwardDataTest method runConv2DTest.

/**
 * @param et
 * @param sparse
 */
public void runConv2DTest(ExecType et, int imgSize, int numImg, int numChannels, int numFilters, int filterSize, int stride, int pad, boolean sparse1, boolean sparse2) {
    RUNTIME_PLATFORM oldRTP = rtplatform;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        if (et == ExecType.SPARK) {
            rtplatform = RUNTIME_PLATFORM.SPARK;
        } else {
            rtplatform = (et == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.SINGLE_NODE;
        }
        if (rtplatform == RUNTIME_PLATFORM.SPARK)
            DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        loadTestConfiguration(config);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        String sparseVal1 = ("" + sparse1).toUpperCase();
        String sparseVal2 = ("" + sparse2).toUpperCase();
        long P = ConvolutionUtils.getP(imgSize, filterSize, stride, pad);
        programArgs = new String[] { "-explain", "-args", "" + imgSize, "" + numImg, "" + numChannels, "" + numFilters, "" + filterSize, "" + stride, "" + pad, "" + P, "" + P, output("B"), sparseVal1, sparseVal2 };
        boolean exceptionExpected = false;
        int expectedNumberOfJobs = -1;
        runTest(true, exceptionExpected, null, expectedNumberOfJobs);
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + imgSize + " " + numImg + " " + numChannels + " " + numFilters + " " + filterSize + " " + stride + " " + pad + " " + P + " " + P + " " + expectedDir() + " " + sparseVal1 + " " + sparseVal2;
        // Run comparison R script
        runRScript(true);
        HashMap<CellIndex, Double> bHM = readRMatrixFromFS("B");
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        TestUtils.compareMatrices(dmlfile, bHM, epsilon, "B-DML", "NumPy");
    } finally {
        rtplatform = oldRTP;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 64 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class PoolTest method runPoolTest.

public void runPoolTest(ExecType et, int imgSize, int numImg, int numChannels, int stride, int pad, int poolSize1, int poolSize2, String poolMode, boolean sparse) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String sparseVal = String.valueOf(sparse).toUpperCase();
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", String.valueOf(imgSize), String.valueOf(numImg), String.valueOf(numChannels), String.valueOf(poolSize1), String.valueOf(poolSize2), String.valueOf(stride), String.valueOf(pad), poolMode, output("B"), sparseVal };
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + imgSize + " " + numImg + " " + numChannels + " " + poolSize1 + " " + poolSize2 + " " + stride + " " + pad + " " + expectedDir() + " " + sparseVal + " " + poolMode;
        // run scripts
        runTest(true, false, null, -1);
        runRScript(true);
        // compare results
        HashMap<CellIndex, Double> bHM = readRMatrixFromFS("B");
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        TestUtils.compareMatrices(dmlfile, bHM, epsilon, "B-DML", "NumPy");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 65 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ABATernaryAggregateTest method runTernaryAggregateTest.

private void runTernaryAggregateTest(String testname, boolean sparse, boolean vectors, boolean rewrites, ExecType et) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean rewritesOld = OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES;
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = rewrites;
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-stats", "-args", input("A"), output("R") };
        fullRScriptName = HOME + testname + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual dataset
        double sparsity = sparse ? sparsity2 : sparsity1;
        double[][] A = getRandomMatrix(vectors ? rows * cols : rows, vectors ? 1 : cols, 0, 1, sparsity, 17);
        writeInputMatrixWithMTD("A", A, true);
        // run test cases
        runTest(true, false, null, -1);
        runRScript(true);
        // compare output matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        // check for rewritten patterns in statistics output
        if (rewrites && et != ExecType.MR) {
            String opcode = ((et == ExecType.SPARK) ? Instruction.SP_INST_PREFIX : "") + (((testname.equals(TEST_NAME1) || testname.equals(TEST_NAME3) || vectors) ? "tak+*" : "tack+*"));
            Assert.assertTrue(Statistics.getCPHeavyHitterOpCodes().contains(opcode));
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = rewritesOld;
    }
}