Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 31 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class BinUaggChainTest method runBinUaggTest.

// ----------------------
/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runBinUaggTest(String testname, boolean singleBlock, boolean sparse, ExecType instType) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String TEST_NAME = testname;
        getAndLoadTestConfiguration(TEST_NAME);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual datasets
        double[][] A = getRandomMatrix(rows, singleBlock ? cols1 : cols2, -1, 1, sparse ? sparsity2 : sparsity1, 7);
        writeInputMatrixWithMTD("A", A, true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        // check compiled/executed jobs
        if (rtplatform != RUNTIME_PLATFORM.SPARK) {
            int expectedNumCompiled = (singleBlock) ? 1 : 3;
            int expectedNumExecuted = (singleBlock) ? 1 : 3;
            checkNumCompiledMRJobs(expectedNumCompiled);
            checkNumExecutedMRJobs(expectedNumExecuted);
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 32 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class CentralMomentTest method runCentralMomentTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runCentralMomentTest(int order, boolean sparse, ExecType et) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        getAndLoadTestConfiguration(TEST_NAME);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), Integer.toString(order), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + order + " " + expectedDir();
        // generate actual dataset (always dense because values <=0 invalid)
        double sparsitya = sparse ? sparsity2 : sparsity1;
        double[][] A = getRandomMatrix(rows, 1, 1, maxVal, sparsitya, 7);
        writeInputMatrixWithMTD("A", A, true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 33 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class RightIndexingMatrixTest method runRightIndexingTest.

public void runRightIndexingTest(ExecType et, boolean sparse, long rl, long ru, long cl, long cu) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        double sparsity = sparse ? sparsity2 : sparsity1;
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        config.addVariable("rowstart", rl);
        config.addVariable("rowend", ru);
        config.addVariable("colstart", cl);
        config.addVariable("colend", cu);
        loadTestConfiguration(config);
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), Long.toString(rows), Long.toString(cols), Long.toString(rl), Long.toString(ru), Long.toString(cl), Long.toString(cu), output("B"), output("C"), output("D") };
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rl + " " + ru + " " + cl + " " + cu + " " + expectedDir();
        double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("A", A, true);
        // run tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare results
        for (String file : config.getOutputFiles()) {
            HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(file);
            HashMap<CellIndex, Double> rfile = readRMatrixFromFS(file);
            TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 34 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class RowBatchRightIndexingTest method runRightIndexingTest.

/**
 * @param et
 * @param sparse
 */
public void runRightIndexingTest(ExecType et, boolean sparse) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        double sparsity = sparse ? sparsity2 : sparsity1;
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), output("B") };
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 23);
        writeInputMatrixWithMTD("A", A, true);
        // run tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare output aggregate
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 35 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ScalarIOTest method testDoubleScalarWrite.

@Test
public void testDoubleScalarWrite() {
    Double double_scalar = 464.55;
    fullDMLScriptName = HOME + "ScalarWrite.dml";
    programArgs = new String[] { "-args", String.valueOf(double_scalar), output("a.scalar") };
    runTest(true, false, null, -1);
    Double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue();
    Assert.assertEquals("Values not equal: " + double_scalar + "!=" + double_out_scalar, double_scalar, double_out_scalar);
    // Invoke the DML script that does computations and then writes scalar to HDFS
    fullDMLScriptName = HOME + "ScalarComputeWrite.dml";
    runTest(true, false, null, -1);
    double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue();
    Assert.assertEquals("Computation test for Integers failed: Values not equal: " + double_scalar + "!=" + double_out_scalar, double_scalar, double_out_scalar);
}