Examples with RUNTIME_PLATFORM org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM used on opensource projects

Example 56 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ParForBlockwiseDataPartitioningTest method runParForDataPartitioningTest.

private void runParForDataPartitioningTest(String testname, PDataPartitioner partitioner, PExecMode mode, boolean sparse) {
    RUNTIME_PLATFORM oldRT = rtplatform;
    boolean oldUseSparkConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    boolean oldDynRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
    // run always in spark execution mode
    DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        CompilerConfig.FLAG_DYN_RECOMPILE = false;
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-stats", "-args", input("V"), partitioner.name(), mode.name(), output("R") };
        fullRScriptName = HOME + testname + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate input data
        int lrows = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME3) ? rows : cols;
        int lcols = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME3) ? cols : rows;
        double lsparsity = sparse ? sparsity2 : sparsity1;
        double[][] V = getRandomMatrix(lrows, lcols, 0, 1, lsparsity, System.nanoTime());
        writeInputMatrixWithMTD("V", V, true);
        // run test
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
        // test for correct plan
        boolean pos = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME2);
        Assert.assertEquals(pos, heavyHittersContainsSubString("ParFor-DPSP") || heavyHittersContainsSubString("ParFor-DPESP"));
    } finally {
        rtplatform = oldRT;
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldUseSparkConfig;
        CompilerConfig.FLAG_DYN_RECOMPILE = oldDynRecompile;
    }
}

Example 57 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ParForColwiseDataPartitioningTest method runParForDataPartitioningTest.

/**
 * @param outer execution mode of outer parfor loop
 * @param inner execution mode of inner parfor loop
 * @param instType execution mode of instructions
 */
private void runParForDataPartitioningTest(PDataPartitioner partitioner, PExecMode mode, boolean small, boolean sparse, boolean multiParts) {
    RUNTIME_PLATFORM oldRT = rtplatform;
    boolean oldUseSparkConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (partitioner == PDataPartitioner.REMOTE_SPARK || mode == PExecMode.REMOTE_SPARK) {
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
    }
    try {
        // inst exec type, influenced via rows
        int rows = small ? rows1 : rows2;
        int cols = small ? cols1 : cols2;
        // script
        int scriptNum = -1;
        switch(partitioner) {
            case NONE:
                scriptNum = 1;
                break;
            case LOCAL:
                if (mode == PExecMode.LOCAL)
                    scriptNum = 2;
                else
                    scriptNum = 3;
            case REMOTE_MR:
                if (mode == PExecMode.LOCAL) {
                    if (!multiParts)
                        scriptNum = 4;
                    else
                        scriptNum = 6;
                } else
                    scriptNum = 5;
                break;
            case REMOTE_SPARK:
                if (mode == PExecMode.LOCAL) {
                    if (!multiParts)
                        scriptNum = 7;
                    else
                        scriptNum = 9;
                } else
                    scriptNum = 8;
                break;
            default:
        }
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        loadTestConfiguration(config);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml";
        programArgs = new String[] { "-args", input("V"), Integer.toString(rows), Integer.toString(cols), output("R") };
        fullRScriptName = HOME + TEST_NAME + (multiParts ? "6" : "") + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        long seed = System.nanoTime();
        double sparsity = -1;
        if (sparse)
            sparsity = sparsity2;
        else
            sparsity = sparsity1;
        double[][] V = getRandomMatrix(rows, cols, 0, 1, sparsity, seed);
        writeInputMatrix("V", V, true);
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
    } finally {
        rtplatform = oldRT;
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldUseSparkConfig;
    }
}

Example 58 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class PiggybackingTest1 method testDistCacheBug_append.

@Test
public void testDistCacheBug_append() {
    RUNTIME_PLATFORM rtold = rtplatform;
    rtplatform = RUNTIME_PLATFORM.HADOOP;
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    loadTestConfiguration(config);
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + TEST_NAME + "_append.dml";
    String OUT_FILE = output(config.getOutputFiles()[1]);
    programArgs = new String[] { "-args", OUT_FILE };
    boolean exceptionExpected = false;
    int numMRJobs = 4;
    runTest(true, exceptionExpected, null, numMRJobs);
    Double expected = 1120.0;
    Double output = TestUtils.readDMLScalarFromHDFS(OUT_FILE).get(new CellIndex(1, 1));
    Assert.assertEquals("Values not equal: " + output + "!=" + expected, output, expected);
    rtplatform = rtold;
}

Example 59 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class WeightedDivMatrixMultTest method runWeightedDivMMTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runWeightedDivMMTest(String testname, boolean sparse, boolean rewrites, boolean rep, ExecType instType) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    boolean forceOld = QuaternaryOp.FORCE_REPLICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    QuaternaryOp.FORCE_REPLICATION = rep;
    try {
        boolean basic = testname.equals(TEST_NAME3);
        boolean left = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME4) || testname.equals(TEST_NAME6) || testname.equals(TEST_NAME8) || testname.equals(TEST_NAME10) || testname.equals(TEST_NAME12) || testname.equals(TEST_NAME13);
        double sparsity = (sparse) ? spSparse : spDense;
        String TEST_NAME = testname;
        String TEST_CACHE_DIR = TEST_CACHE_ENABLED ? TEST_CACHE_DIR = TEST_NAME + "_" + sparsity + "/" : "";
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config, TEST_CACHE_DIR);
        // This is for running the junit test the new way, i.e., construct the arguments directly
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-stats", "-explain", "runtime", "-args", input("W"), input("U"), input("V"), output("R"), Double.toString(div_eps) };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir() + " " + div_eps;
        int rows = sparse ? rows2 : rows1;
        int cols = sparse ? cols2 : cols1;
        int rank = sparse ? rank2 : rank1;
        // generate actual dataset
        double[][] W = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
        writeInputMatrixWithMTD("W", W, TestUtils.computeNNZ(W), true);
        double[][] U = getRandomMatrix(rows, rank, 0, 1, 1.0, 713);
        writeInputMatrixWithMTD("U", U, TestUtils.computeNNZ(U), true);
        double[][] V = getRandomMatrix(cols, rank, 0, 1, 1.0, 812);
        writeInputMatrixWithMTD("V", V, TestUtils.computeNNZ(V), true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        checkDMLMetaDataFile("R", new MatrixCharacteristics(left ? cols : rows, basic ? cols : rank, 1, 1));
        // check statistics for right operator in cp and spark
        if (instType == ExecType.CP && rewrites) {
            Assert.assertTrue("Missing opcode wdivmm", Statistics.getCPHeavyHitterOpCodes().contains(WeightedDivMM.OPCODE_CP));
        } else if (instType == ExecType.SPARK && rewrites) {
            boolean reduce = rep || testname.equals(TEST_NAME8) || testname.equals(TEST_NAME9);
            String opcode = Instruction.SP_INST_PREFIX + ((reduce) ? WeightedDivMMR.OPCODE : WeightedDivMM.OPCODE);
            Assert.assertTrue("Missing opcode sp_wdivmm", Statistics.getCPHeavyHitterOpCodes().contains(opcode));
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
        QuaternaryOp.FORCE_REPLICATION = forceOld;
    }
}

Example 60 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class WeightedSquaredLossTest method runMLUnaryBuiltinTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runMLUnaryBuiltinTest(String testname, boolean sparse, boolean rewrites, boolean rep, ExecType instType) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    boolean forceOld = QuaternaryOp.FORCE_REPLICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    QuaternaryOp.FORCE_REPLICATION = rep;
    try {
        double sparsity = (sparse) ? spSparse : spDense;
        String TEST_NAME = testname;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = TEST_NAME + "_" + sparsity + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        // This is for running the junit test the new way, i.e., construct the arguments directly
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-stats", "-explain", "runtime", "-args", input("X"), input("U"), input("V"), input("W"), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual dataset
        double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
        writeInputMatrixWithMTD("X", X, true);
        double[][] U = getRandomMatrix(rows, rank, 0, 1, 1.0, 213);
        writeInputMatrixWithMTD("U", U, true);
        double[][] V = getRandomMatrix(cols, rank, 0, 1, 1.0, 312);
        writeInputMatrixWithMTD("V", V, true);
        if (!TEST_NAME.equals(TEST_NAME3)) {
            double[][] W = getRandomMatrix(rows, cols, 0, 1, sparsity, 1467);
            writeInputMatrixWithMTD("W", W, true);
        }
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        checkDMLMetaDataFile("R", new MatrixCharacteristics(1, 1, 1, 1));
        // check statistics for right operator in cp
        if (instType == ExecType.CP && rewrites)
            Assert.assertTrue("Rewrite not applied.", Statistics.getCPHeavyHitterOpCodes().contains(WeightedSquaredLoss.OPCODE_CP));
        else if (instType == ExecType.SPARK && rewrites) {
            boolean noWeights = testname.equals(TEST_NAME3) || testname.equals(TEST_NAME6) || testname.equals(TEST_NAME7);
            String opcode = Instruction.SP_INST_PREFIX + ((rep || !noWeights) ? WeightedSquaredLossR.OPCODE : WeightedSquaredLoss.OPCODE);
            Assert.assertTrue("Rewrite not applied.", Statistics.getCPHeavyHitterOpCodes().contains(opcode));
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
        QuaternaryOp.FORCE_REPLICATION = forceOld;
    }
}