Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 21 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ParForCorrelationTestLarge method runParForCorrelationTest.

/**
 * @param outer execution mode of outer parfor loop
 * @param inner execution mode of inner parfor loop
 * @param instType execution mode of instructions
 */
private void runParForCorrelationTest(PExecMode outer, PExecMode inner) {
    // script
    int scriptNum = -1;
    if (inner == PExecMode.REMOTE_MR)
        scriptNum = 2;
    else if (outer == PExecMode.REMOTE_MR)
        scriptNum = 3;
    else if (outer == PExecMode.LOCAL)
        scriptNum = 1;
    else
        // optimized
        scriptNum = 4;
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    config.addVariable("rows", rows);
    config.addVariable("cols", cols);
    loadTestConfiguration(config);
    /* This is for running the junit test the new way, i.e., construct the arguments directly */
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml";
    programArgs = new String[] { "-args", input("V"), Integer.toString(rows), Integer.toString(cols), output("PearsonR") };
    fullRScriptName = HOME + TEST_NAME + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
    long seed = System.nanoTime();
    double[][] V = getRandomMatrix(rows, cols, minVal, maxVal, 1.0, seed);
    writeInputMatrix("V", V, true);
    boolean exceptionExpected = false;
    runTest(true, exceptionExpected, null, -1);
    runRScript(true);
    // compare matrices
    HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("PearsonR");
    HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout");
    TestUtils.compareMatrices(dmlfile, rfile, eps, "PearsonR-DML", "PearsonR-R");
}

Example 22 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ParForNaiveBayesTest method runParForNaiveBayesTest.

/**
 * @param outer
 * @param instType
 * @param smallMem
 * @param sparse
 */
private void runParForNaiveBayesTest(PExecMode outer, ExecType instType, boolean smallMem, boolean sparse) {
    int cols = (instType == ExecType.MR) ? cols2 : cols1;
    // inst exec type, influenced via rows
    RUNTIME_PLATFORM oldPlatform = rtplatform;
    rtplatform = (instType == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    // determine the script
    int scriptNum = -1;
    if (// constrained opt
    outer == PExecMode.LOCAL)
        // constrained opt
        scriptNum = 1;
    else if (// constrained opt
    outer == PExecMode.REMOTE_MR)
        // constrained opt
        scriptNum = 2;
    else if (// constrained opt
    outer == PExecMode.REMOTE_MR_DP)
        // constrained opt
        scriptNum = 3;
    else
        // opt
        scriptNum = 4;
    // invocation arguments
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    config.addVariable("rows", rows);
    config.addVariable("cols", cols);
    loadTestConfiguration(config);
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml";
    programArgs = new String[] { "-args", input("D"), input("C"), Integer.toString((int) maxVal), output("class_prior"), output("class_conditionals") };
    fullRScriptName = HOME + TEST_NAME + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + Integer.toString((int) maxVal) + " " + expectedDir();
    // input data
    double[][] D = getRandomMatrix(rows, cols, -1, 1, sparse ? sparsity2 : sparsity1, 7);
    double[][] C = TestUtils.round(getRandomMatrix(rows, 1, minVal, maxVal, 1.0, 3));
    MatrixCharacteristics mc1 = new MatrixCharacteristics(rows, cols, -1, -1);
    writeInputMatrixWithMTD("D", D, true, mc1);
    MatrixCharacteristics mc2 = new MatrixCharacteristics(rows, 1, -1, -1);
    writeInputMatrixWithMTD("C", C, true, mc2);
    // set memory budget (to test automatic opt for remote_mr_dp)
    long oldmem = InfrastructureAnalyzer.getLocalMaxMemory();
    if (smallMem) {
        long mem = 1024 * 1024 * 8;
        InfrastructureAnalyzer.setLocalMaxMemory(mem);
    }
    try {
        // run the testcase (DML and R)
        runTest(true, false, null, -1);
        runRScript(true);
        // compare output matrices
        for (String out : new String[] { "class_prior", "class_conditionals" }) {
            HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(out);
            HashMap<CellIndex, Double> rfile = readRMatrixFromFS(out);
            TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        }
    } finally {
        rtplatform = oldPlatform;
        InfrastructureAnalyzer.setLocalMaxMemory(oldmem);
    }
}

Example 23 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class AggregateInfTest method runInfAggregateOperationTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runInfAggregateOperationTest(boolean pos, boolean sparse, ExecType instType) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = (instType == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    try {
        double sparsity = (sparse) ? sparsity1 : sparsity2;
        getAndLoadTestConfiguration(TEST_NAME);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual dataset
        double[][] A = getRandomMatrix(rows, cols, -0.05, 1, sparsity, 7);
        double infval = pos ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
        A[7][7] = infval;
        writeInputMatrixWithMTD("A", A, false);
        // run test
        runTest(true, false, null, -1);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> compfile = new HashMap<CellIndex, Double>();
        compfile.put(new CellIndex(1, 1), infval);
        TestUtils.compareMatrices(dmlfile, compfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
    }
}

Example 24 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ColSumsSqTest method testColSumsSquared.

/**
 * Test the column sums of squared values function, "colSums(X^2)",
 * on dense/sparse matrices/vectors with rewrites/no rewrites on
 * the CP/Spark/MR platforms.
 *
 * @param testName The name of this test case.
 * @param sparse Whether or not the matrix/vector should be sparse.
 * @param vector Boolean value choosing between a vector and a matrix.
 * @param rewrites Whether or not to employ algebraic rewrites.
 * @param platform Selection between CP/Spark/MR platforms.
 */
private void testColSumsSquared(String testName, boolean sparse, boolean vector, boolean rewrites, ExecType platform) {
    // Configure settings for this test case
    boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(platform) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        // Create and load test configuration
        getAndLoadTestConfiguration(testName);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testName + ".dml";
        programArgs = new String[] { "-explain", "-stats", "-args", input(INPUT_NAME), output(OUTPUT_NAME) };
        fullRScriptName = HOME + testName + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // Generate data
        double sparsity = sparse ? sparsity2 : sparsity1;
        int columns = vector ? 1 : cols;
        double[][] X = getRandomMatrix(rows, columns, -1, 1, sparsity, 7);
        writeInputMatrixWithMTD(INPUT_NAME, X, true);
        // Run DML and R scripts
        runTest(true, false, null, -1);
        runRScript(true);
        // Compare output matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(OUTPUT_NAME);
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS(OUTPUT_NAME);
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        // occurred for matrix cases and not for vector cases.
        if (rewrites && (platform == ExecType.SPARK || platform == ExecType.CP)) {
            String prefix = (platform == ExecType.SPARK) ? Instruction.SP_INST_PREFIX : "";
            String opcode = prefix + op;
            boolean rewriteApplied = Statistics.getCPHeavyHitterOpCodes().contains(opcode);
            if (vector)
                Assert.assertFalse("Rewrite applied to vector case.", rewriteApplied);
            else
                Assert.assertTrue("Rewrite not applied to matrix case.", rewriteApplied);
        }
    } finally {
        // Reset settings
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 25 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FullGroupedAggregateMatrixTest method runGroupedAggregateOperationTest.

/**
 * @param testname
 * @param type
 * @param sparse
 * @param instType
 */
@SuppressWarnings("rawtypes")
private void runGroupedAggregateOperationTest(String testname, OpType type, boolean sparse, ExecType instType, int numCols) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        // determine script and function name
        String TEST_NAME = testname;
        int fn = type.ordinal();
        double sparsity = (sparse) ? sparsity1 : sparsity2;
        String TEST_CACHE_DIR = TEST_CACHE_ENABLED ? TEST_NAME + type.ordinal() + "_" + sparsity + "_" + numCols + "/" : "";
        boolean exceptionExpected = !TEST_NAME.equals(TEST_NAME1);
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config, TEST_CACHE_DIR);
        // This is for running the junit test the new way, i.e., construct the arguments directly
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), input("B"), String.valueOf(fn), String.valueOf(numGroups), output("C") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + fn + " " + expectedDir();
        // generate actual dataset
        double[][] A = getRandomMatrix(rows, numCols, -0.05, 1, sparsity, 7);
        writeInputMatrix("A", A, true);
        MatrixCharacteristics mc1 = new MatrixCharacteristics(rows, numCols, 1000, 1000);
        MapReduceTool.writeMetaDataFile(input("A.mtd"), ValueType.DOUBLE, mc1, OutputInfo.TextCellOutputInfo);
        double[][] B = TestUtils.round(getRandomMatrix(rows, 1, 1, numGroups, 1.0, 3));
        writeInputMatrix("B", B, true);
        MatrixCharacteristics mc2 = new MatrixCharacteristics(rows, 1, 1000, 1000);
        MapReduceTool.writeMetaDataFile(input("B.mtd"), ValueType.DOUBLE, mc2, OutputInfo.TextCellOutputInfo);
        // run tests
        Class cla = (exceptionExpected ? DMLException.class : null);
        runTest(true, exceptionExpected, cla, -1);
        // compare matrices
        if (!exceptionExpected) {
            // run R script for comparison
            runRScript(true);
            // compare output matrices
            HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
            HashMap<CellIndex, Double> rfile = readRMatrixFromFS("C");
            TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
            // check dml output meta data
            checkDMLMetaDataFile("C", new MatrixCharacteristics(numGroups, numCols, 1, 1));
        }
    } catch (IOException ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}