Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 61 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ParForReplaceThreadIDRecompileTest method runThreadIDReplaceTest.

/**
 * @param TEST_NAME
 * @param recompile
 */
private void runThreadIDReplaceTest(String TEST_NAME, boolean recompile) {
    boolean flag = CompilerConfig.FLAG_DYN_RECOMPILE;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
        // This is for running the junit test the new way, i.e., construct the arguments directly
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        double[][] A = new double[][] { { 2.0 }, { 3.0 } };
        writeInputMatrixWithMTD("A", A, false);
        runTest(true, false, null, -1);
        // compare matrices
        HashMap<CellIndex, Double> dmlout = readDMLMatrixFromHDFS("B");
        Assert.assertTrue(dmlout.size() >= 1);
    } finally {
        CompilerConfig.FLAG_DYN_RECOMPILE = flag;
    }
}

Example 62 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class PiggybackingTest1 method testDistCacheBug_append.

@Test
public void testDistCacheBug_append() {
    RUNTIME_PLATFORM rtold = rtplatform;
    rtplatform = RUNTIME_PLATFORM.HADOOP;
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    loadTestConfiguration(config);
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + TEST_NAME + "_append.dml";
    String OUT_FILE = output(config.getOutputFiles()[1]);
    programArgs = new String[] { "-args", OUT_FILE };
    boolean exceptionExpected = false;
    int numMRJobs = 4;
    runTest(true, exceptionExpected, null, numMRJobs);
    Double expected = 1120.0;
    Double output = TestUtils.readDMLScalarFromHDFS(OUT_FILE).get(new CellIndex(1, 1));
    Assert.assertEquals("Values not equal: " + output + "!=" + expected, output, expected);
    rtplatform = rtold;
}

Example 63 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class WeightedDivMatrixMultTest method runWeightedDivMMTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runWeightedDivMMTest(String testname, boolean sparse, boolean rewrites, boolean rep, ExecType instType) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    boolean forceOld = QuaternaryOp.FORCE_REPLICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    QuaternaryOp.FORCE_REPLICATION = rep;
    try {
        boolean basic = testname.equals(TEST_NAME3);
        boolean left = testname.equals(TEST_NAME1) || testname.equals(TEST_NAME4) || testname.equals(TEST_NAME6) || testname.equals(TEST_NAME8) || testname.equals(TEST_NAME10) || testname.equals(TEST_NAME12) || testname.equals(TEST_NAME13);
        double sparsity = (sparse) ? spSparse : spDense;
        String TEST_NAME = testname;
        String TEST_CACHE_DIR = TEST_CACHE_ENABLED ? TEST_CACHE_DIR = TEST_NAME + "_" + sparsity + "/" : "";
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config, TEST_CACHE_DIR);
        // This is for running the junit test the new way, i.e., construct the arguments directly
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-stats", "-explain", "runtime", "-args", input("W"), input("U"), input("V"), output("R"), Double.toString(div_eps) };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir() + " " + div_eps;
        int rows = sparse ? rows2 : rows1;
        int cols = sparse ? cols2 : cols1;
        int rank = sparse ? rank2 : rank1;
        // generate actual dataset
        double[][] W = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
        writeInputMatrixWithMTD("W", W, TestUtils.computeNNZ(W), true);
        double[][] U = getRandomMatrix(rows, rank, 0, 1, 1.0, 713);
        writeInputMatrixWithMTD("U", U, TestUtils.computeNNZ(U), true);
        double[][] V = getRandomMatrix(cols, rank, 0, 1, 1.0, 812);
        writeInputMatrixWithMTD("V", V, TestUtils.computeNNZ(V), true);
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        checkDMLMetaDataFile("R", new MatrixCharacteristics(left ? cols : rows, basic ? cols : rank, 1, 1));
        // check statistics for right operator in cp and spark
        if (instType == ExecType.CP && rewrites) {
            Assert.assertTrue("Missing opcode wdivmm", Statistics.getCPHeavyHitterOpCodes().contains(WeightedDivMM.OPCODE_CP));
        } else if (instType == ExecType.SPARK && rewrites) {
            boolean reduce = rep || testname.equals(TEST_NAME8) || testname.equals(TEST_NAME9);
            String opcode = Instruction.SP_INST_PREFIX + ((reduce) ? WeightedDivMMR.OPCODE : WeightedDivMM.OPCODE);
            Assert.assertTrue("Missing opcode sp_wdivmm", Statistics.getCPHeavyHitterOpCodes().contains(opcode));
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
        QuaternaryOp.FORCE_REPLICATION = forceOld;
    }
}

Example 64 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class WeightedSquaredLossTest method runMLUnaryBuiltinTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runMLUnaryBuiltinTest(String testname, boolean sparse, boolean rewrites, boolean rep, ExecType instType) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    boolean forceOld = QuaternaryOp.FORCE_REPLICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    QuaternaryOp.FORCE_REPLICATION = rep;
    try {
        double sparsity = (sparse) ? spSparse : spDense;
        String TEST_NAME = testname;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = TEST_NAME + "_" + sparsity + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        // This is for running the junit test the new way, i.e., construct the arguments directly
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-stats", "-explain", "runtime", "-args", input("X"), input("U"), input("V"), input("W"), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual dataset
        double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
        writeInputMatrixWithMTD("X", X, true);
        double[][] U = getRandomMatrix(rows, rank, 0, 1, 1.0, 213);
        writeInputMatrixWithMTD("U", U, true);
        double[][] V = getRandomMatrix(cols, rank, 0, 1, 1.0, 312);
        writeInputMatrixWithMTD("V", V, true);
        if (!TEST_NAME.equals(TEST_NAME3)) {
            double[][] W = getRandomMatrix(rows, cols, 0, 1, sparsity, 1467);
            writeInputMatrixWithMTD("W", W, true);
        }
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        checkDMLMetaDataFile("R", new MatrixCharacteristics(1, 1, 1, 1));
        // check statistics for right operator in cp
        if (instType == ExecType.CP && rewrites)
            Assert.assertTrue("Rewrite not applied.", Statistics.getCPHeavyHitterOpCodes().contains(WeightedSquaredLoss.OPCODE_CP));
        else if (instType == ExecType.SPARK && rewrites) {
            boolean noWeights = testname.equals(TEST_NAME3) || testname.equals(TEST_NAME6) || testname.equals(TEST_NAME7);
            String opcode = Instruction.SP_INST_PREFIX + ((rep || !noWeights) ? WeightedSquaredLossR.OPCODE : WeightedSquaredLoss.OPCODE);
            Assert.assertTrue("Rewrite not applied.", Statistics.getCPHeavyHitterOpCodes().contains(opcode));
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
        QuaternaryOp.FORCE_REPLICATION = forceOld;
    }
}

Example 65 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FunctionRecompileTest method runFunctionTest.

private void runFunctionTest(boolean recompile, boolean IPA) {
    boolean oldFlagRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
    boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME1);
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        loadTestConfiguration(config);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
        programArgs = new String[] { "-args", input("V"), Integer.toString(rows), Integer.toString(cols), output("R") };
        fullRScriptName = HOME + TEST_NAME1 + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        long seed = System.nanoTime();
        double[][] V = getRandomMatrix(rows, cols, 0, 1, sparsity, seed);
        writeInputMatrix("V", V, true);
        CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
        OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        // note: change from previous version due to fix in op selection (unknown size XtX and mapmult)
        // CHECK compiled MR jobs
        int expectNumCompiled = -1;
        if (// reblock (with recompile right indexing); before: 3 reblock, GMR,GMR
        IPA)
            // reblock (with recompile right indexing); before: 3 reblock, GMR,GMR
            expectNumCompiled = 1;
        else
            // reblock, GMR,GMR,GMR,GMR (last two should piggybacked)
            expectNumCompiled = 5;
        Assert.assertEquals("Unexpected number of compiled MR jobs.", expectNumCompiled, Statistics.getNoOfCompiledMRJobs());
        // CHECK executed MR jobs
        int expectNumExecuted = -1;
        if (recompile)
            expectNumExecuted = 0;
        else if (// reblock (with recompile right indexing); before: 21 reblock, 10*(GMR,GMR)
        IPA)
            // reblock (with recompile right indexing); before: 21 reblock, 10*(GMR,GMR)
            expectNumExecuted = 1;
        else
            // reblock, 10*(GMR,GMR,GMR, GMR) (last two should piggybacked)
            expectNumExecuted = 41;
        Assert.assertEquals("Unexpected number of executed MR jobs.", expectNumExecuted, Statistics.getNoOfExecutedMRJobs());
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
    } finally {
        CompilerConfig.FLAG_DYN_RECOMPILE = oldFlagRecompile;
        OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
    }
}