Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 56 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ScalarFunctionTest method runScalarFunctionTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runScalarFunctionTest(String testname) {
    String TEST_NAME = testname;
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    loadTestConfiguration(config);
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + TEST_NAME + ".dml";
    programArgs = new String[] { "-args", output("R") };
    fullRScriptName = HOME + TEST_NAME + ".R";
    rCmd = getRCmd(expectedDir());
    // run Tests
    runTest(true, false, null, -1);
    runRScript(true);
    // compare matrices
    HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
    HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
    TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    // check meta data
    checkDMLMetaDataFile("R", new MatrixCharacteristics(1, 1, 1, 1));
}

Example 57 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ZeroRowsColsMatrixTest method runEmptyMatrixTest.

private void runEmptyMatrixTest(String testname, boolean rewrites, boolean emptyRet, ExecType et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    try {
        String TEST_NAME = testname;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "recompile_runtime", "-args", String.valueOf(dim), String.valueOf(emptyRet).toUpperCase(), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = getRCmd(String.valueOf(dim), String.valueOf(emptyRet).toUpperCase(), expectedDir());
        // run Tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        // check meta data
        if (!testname.equals(TEST_NAME4))
            checkDMLMetaDataFile("R", new MatrixCharacteristics(dim, 3, 1000, 1000));
    } finally {
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 58 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ParForAccumulatorResultMergeTest method runParForAccumulatorResultMergeTest.

private void runParForAccumulatorResultMergeTest(String test, boolean init, boolean sparse, ExecType et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case CP:
            rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
            break;
        case MR:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
        default:
            throw new RuntimeException("Unsupported exec type: " + et.name());
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (et == ExecType.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String TEST_NAME = test;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", String.valueOf(rows), String.valueOf(cols), String.valueOf(init).toUpperCase(), String.valueOf(sparse).toUpperCase(), output("R") };
        fullRScriptName = HOME + TEST_NAME.substring(0, TEST_NAME.length() - 1) + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + String.valueOf(rows) + " " + String.valueOf(cols) + " " + String.valueOf(init).toUpperCase() + " " + String.valueOf(sparse).toUpperCase() + " " + expectedDir();
        // run tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 59 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ParForBlockwiseDataPartitioningTest method runParForDataPartitioningTest.

private void runParForDataPartitioningTest(String testname, PDataPartitioner partitioner, PExecMode mode, boolean sparse) {
    RUNTIME_PLATFORM oldRT = rtplatform;
    boolean oldUseSparkConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    boolean oldDynRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
    // run always in spark execution mode
    DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        CompilerConfig.FLAG_DYN_RECOMPILE = false;
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-stats", "-args", input("V"), partitioner.name(), mode.name(), output("R") };
        fullRScriptName = HOME + testname + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate input data
        int lrows = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME3) ? rows : cols;
        int lcols = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME3) ? cols : rows;
        double lsparsity = sparse ? sparsity2 : sparsity1;
        double[][] V = getRandomMatrix(lrows, lcols, 0, 1, lsparsity, System.nanoTime());
        writeInputMatrixWithMTD("V", V, true);
        // run test
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
        // test for correct plan
        boolean pos = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME2);
        Assert.assertEquals(pos, heavyHittersContainsSubString("ParFor-DPSP") || heavyHittersContainsSubString("ParFor-DPESP"));
    } finally {
        rtplatform = oldRT;
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldUseSparkConfig;
        CompilerConfig.FLAG_DYN_RECOMPILE = oldDynRecompile;
    }
}

Example 60 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ParForColwiseDataPartitioningTest method runParForDataPartitioningTest.

/**
 * @param outer execution mode of outer parfor loop
 * @param inner execution mode of inner parfor loop
 * @param instType execution mode of instructions
 */
private void runParForDataPartitioningTest(PDataPartitioner partitioner, PExecMode mode, boolean small, boolean sparse, boolean multiParts) {
    RUNTIME_PLATFORM oldRT = rtplatform;
    boolean oldUseSparkConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (partitioner == PDataPartitioner.REMOTE_SPARK || mode == PExecMode.REMOTE_SPARK) {
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
    }
    try {
        // inst exec type, influenced via rows
        int rows = small ? rows1 : rows2;
        int cols = small ? cols1 : cols2;
        // script
        int scriptNum = -1;
        switch(partitioner) {
            case NONE:
                scriptNum = 1;
                break;
            case LOCAL:
                if (mode == PExecMode.LOCAL)
                    scriptNum = 2;
                else
                    scriptNum = 3;
            case REMOTE_MR:
                if (mode == PExecMode.LOCAL) {
                    if (!multiParts)
                        scriptNum = 4;
                    else
                        scriptNum = 6;
                } else
                    scriptNum = 5;
                break;
            case REMOTE_SPARK:
                if (mode == PExecMode.LOCAL) {
                    if (!multiParts)
                        scriptNum = 7;
                    else
                        scriptNum = 9;
                } else
                    scriptNum = 8;
                break;
            default:
        }
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        loadTestConfiguration(config);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml";
        programArgs = new String[] { "-args", input("V"), Integer.toString(rows), Integer.toString(cols), output("R") };
        fullRScriptName = HOME + TEST_NAME + (multiParts ? "6" : "") + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        long seed = System.nanoTime();
        double sparsity = -1;
        if (sparse)
            sparsity = sparsity2;
        else
            sparsity = sparsity1;
        double[][] V = getRandomMatrix(rows, cols, 0, 1, sparsity, seed);
        writeInputMatrix("V", V, true);
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
    } finally {
        rtplatform = oldRT;
        DMLScript.USE_LOCAL_SPARK_CONFIG = oldUseSparkConfig;
    }
}