Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 51 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class LeftIndexingSparseSparseTest method runLeftIndexingSparseSparseTest.

/**
 * @param type
 */
public void runLeftIndexingSparseSparseTest(LixType type, ExecType et, LeftIndexingOp.LeftIndexingMethod indexingMethod) {
    // setup range (column lower/upper)
    int cl = -1;
    switch(type) {
        case LEFT_ALIGNED:
            cl = 1;
            break;
        case LEFT2_ALIGNED:
            cl = 2;
            break;
        case RIGHT_ALIGNED:
            cl = cols1 - cols2 + 1;
            break;
        case RIGHT2_ALIGNED:
            cl = cols1 - cols2;
            break;
        case CENTERED:
            cl = (cols1 - cols2) / 2;
            break;
    }
    int cu = cl + cols2 - 1;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    RUNTIME_PLATFORM oldRTP = rtplatform;
    try {
        if (indexingMethod != null) {
            LeftIndexingOp.FORCED_LEFT_INDEXING = indexingMethod;
        }
        if (et == ExecType.SPARK) {
            rtplatform = RUNTIME_PLATFORM.SPARK;
        } else {
            // rtplatform = (et==ExecType.MR)? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.SINGLE_NODE;
            rtplatform = RUNTIME_PLATFORM.HYBRID;
        }
        if (rtplatform == RUNTIME_PLATFORM.SPARK)
            DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = type.toString() + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), input("B"), String.valueOf(cl), String.valueOf(cu), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + cl + " " + cu + " " + expectedDir();
        // generate input data sets
        double[][] A = getRandomMatrix(rows1, cols1, -1, 1, sparsity1, 1234);
        writeInputMatrixWithMTD("A", A, true);
        double[][] B = getRandomMatrix(rows2, cols2, -1, 1, sparsity2, 5678);
        writeInputMatrixWithMTD("B", B, true);
        // REBLOCK
        runTest(true, false, null, 1);
    } finally {
        rtplatform = oldRTP;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        LeftIndexingOp.FORCED_LEFT_INDEXING = null;
    }
    runRScript(true);
    HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
    HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
    TestUtils.compareMatrices(dmlfile, rfile, 0, "DML", "R");
    checkDMLMetaDataFile("R", new MatrixCharacteristics(rows1, cols1, 1, 1));
}

Example 52 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class LeftIndexingTest method runTestLeftIndexing.

private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod indexingMethod) {
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    RUNTIME_PLATFORM oldRTP = rtplatform;
    TestConfiguration config = getTestConfiguration("LeftIndexingTest");
    try {
        if (indexingMethod != null) {
            LeftIndexingOp.FORCED_LEFT_INDEXING = indexingMethod;
        }
        if (et == ExecType.SPARK) {
            rtplatform = RUNTIME_PLATFORM.SPARK;
        } else {
            // rtplatform = (et==ExecType.MR)? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.SINGLE_NODE;
            rtplatform = RUNTIME_PLATFORM.HYBRID;
        }
        if (rtplatform == RUNTIME_PLATFORM.SPARK)
            DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009;
        // long rowstart=2, rowend=4, colstart=9, colend=10;
        /*
	        Random rand=new Random(System.currentTimeMillis());
	        rowstart=(long)(rand.nextDouble()*((double)rows))+1;
	        rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart;
	        colstart=(long)(rand.nextDouble()*((double)cols))+1;
	        colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart;
	        */
        config.addVariable("rowstart", rowstart);
        config.addVariable("rowend", rowend);
        config.addVariable("colstart", colstart);
        config.addVariable("colend", colend);
        loadTestConfiguration(config);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String LI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = LI_HOME + "LeftIndexingTest" + ".dml";
        programArgs = new String[] { "-args", input("A"), Long.toString(rows), Long.toString(cols), Long.toString(rowstart), Long.toString(rowend), Long.toString(colstart), Long.toString(colend), output("AB"), output("AC"), output("AD"), input("B"), input("C"), input("D"), Long.toString(rowend - rowstart + 1), Long.toString(colend - colstart + 1), Long.toString(cols - colstart + 1) };
        fullRScriptName = LI_HOME + "LeftIndexingTest" + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir();
        // rand.nextDouble();
        double sparsity = 1.0;
        double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("A", A, true);
        // rand.nextDouble();
        sparsity = 0.1;
        double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("B", B, true);
        // rand.nextDouble();
        sparsity = 0.5;
        double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("C", C, true);
        // rand.nextDouble();
        sparsity = 0.01;
        double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("D", D, true);
        /*
			 * Expected number of jobs:
			 * Reblock - 1 job 
			 * While loop iteration - 10 jobs
			 * Final output write - 1 job
			 */
        // boolean exceptionExpected = false;
        // int expectedNumberOfJobs = 12;
        // runTest(exceptionExpected, null, expectedNumberOfJobs);
        boolean exceptionExpected = false;
        int expectedNumberOfJobs = -1;
        runTest(true, exceptionExpected, null, expectedNumberOfJobs);
    } finally {
        rtplatform = oldRTP;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        LeftIndexingOp.FORCED_LEFT_INDEXING = null;
    }
    runRScript(true);
    for (String file : config.getOutputFiles()) {
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(file);
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS(file);
        // System.out.println(file+"-DML: "+dmlfile);
        // System.out.println(file+"-R: "+rfile);
        TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
    }
}

Example 53 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class LeftIndexingUpdateInPlaceTest method runLeftIndexingUpdateInPlaceTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param vectorM2
 */
public void runLeftIndexingUpdateInPlaceTest(boolean sparseM1, boolean sparseM2, boolean vectorM2, boolean emptyM2) {
    RUNTIME_PLATFORM oldRTP = rtplatform;
    rtplatform = RUNTIME_PLATFORM.HYBRID;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        double spM1 = sparseM1 ? sparsity1 : sparsity2;
        double spM2 = emptyM2 ? 0 : (sparseM2 ? sparsity1 : sparsity2);
        int colsM2 = vectorM2 ? cols3 : cols2;
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), input("B"), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate input data sets
        double[][] A = getRandomMatrix(rows1, cols1, -1, 1, spM1, 1234);
        writeInputMatrixWithMTD("A", A, true);
        double[][] B = getRandomMatrix(rows1, colsM2, -1, 1, spM2, 5678);
        writeInputMatrixWithMTD("B", B, true);
        // run dml and r script
        runTest(true, false, null, 1);
        runRScript(true);
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, 0, "DML", "R");
        checkDMLMetaDataFile("R", new MatrixCharacteristics(rows1, cols1, 1, 1));
    } finally {
        rtplatform = oldRTP;
    }
}

Example 54 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class RewriteMergeBlocksTest method testRewriteMerge.

private void testRewriteMerge(String testname, boolean expectedMerge) {
    TestConfiguration config = getTestConfiguration(testname);
    loadTestConfiguration(config);
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + testname + ".dml";
    programArgs = new String[] { "-explain", "-stats", "-args", output("R") };
    fullRScriptName = HOME + testname + ".R";
    rCmd = getRCmd(expectedDir());
    runTest(true, false, null, -1);
    runRScript(true);
    // compare outputs and check for compiled mmchain as proof for merge blocks
    HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
    HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
    TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    Assert.assertTrue(expectedMerge == heavyHittersContainsSubString("mmchain"));
}

Example 55 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class RewriteSlicedMatrixMultTest method testRewriteSlicedMatrixMult.

/**
 * @param condition
 * @param branchRemoval
 * @param IPA
 */
private void testRewriteSlicedMatrixMult(String testname, boolean sparse, boolean rewrites) {
    boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-stats", "-args", input("A"), input("B"), output("R") };
        fullRScriptName = HOME + testname + ".R";
        rCmd = getRCmd(inputDir(), expectedDir());
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
        double sparsity = sparse ? sparsity2 : sparsity1;
        double[][] A = getRandomMatrix(dim1, dim2, -1, 1, sparsity, 7);
        writeInputMatrixWithMTD("A", A, true);
        double[][] B = getRandomMatrix(dim2, dim3, -1, 1, sparsity, 3);
        writeInputMatrixWithMTD("B", B, true);
        // run performance tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
    }
}