Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 91 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FullMatrixVectorRowCellwiseOperationTest method runMatrixVectorCellwiseOperationTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runMatrixVectorCellwiseOperationTest(OpType type, SparsityType sparseM1, SparsityType sparseM2, ExecType instType) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String TEST_NAME = null;
        switch(type) {
            case ADDITION:
                TEST_NAME = TEST_NAME1;
                break;
            case SUBTRACTION:
                TEST_NAME = TEST_NAME2;
                break;
            case MULTIPLICATION:
                TEST_NAME = TEST_NAME3;
                break;
            case DIVISION:
                TEST_NAME = TEST_NAME4;
                break;
        }
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        // get sparsity
        double lsparsity1 = 1.0, lsparsity2 = 1.0;
        switch(sparseM1) {
            case DENSE:
                lsparsity1 = sparsity1;
                break;
            case SPARSE:
                lsparsity1 = sparsity2;
                break;
            case EMPTY:
                lsparsity1 = 0.0;
                break;
        }
        switch(sparseM2) {
            case DENSE:
                lsparsity2 = sparsity1;
                break;
            case SPARSE:
                lsparsity2 = sparsity2;
                break;
            case EMPTY:
                lsparsity2 = 0.0;
                break;
        }
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED && (type != OpType.DIVISION)) {
            TEST_CACHE_DIR = type.ordinal() + "_" + lsparsity1 + "_" + lsparsity2 + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "recompile_runtime", "-args", input("A"), input("B"), output("C") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual dataset
        double[][] A = getRandomMatrix(rows, cols, 0, (lsparsity1 == 0) ? 0 : 1, lsparsity1, 7);
        writeInputMatrixWithMTD("A", A, true);
        double[][] B = getRandomMatrix(1, cols, 0, (lsparsity2 == 0) ? 0 : 1, lsparsity2, 3);
        writeInputMatrixWithMTD("B", B, true);
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        if (!(type == OpType.DIVISION)) {
            runRScript(true);
            // compare matrices
            HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
            HashMap<CellIndex, Double> rfile = readRMatrixFromFS("C");
            TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        } else {
            // For division, IEEE 754 defines x/0.0 as INFINITY and 0.0/0.0 as NaN.
            // Java handles this correctly while R always returns 1.0E308 in those cases.
            // Hence, we directly write the expected results.
            double[][] C = new double[rows][cols];
            for (int i = 0; i < rows; i++) for (int j = 0; j < cols; j++) C[i][j] = A[i][j] / B[0][j];
            writeExpectedMatrix("C", C);
            compareResults();
        }
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 92 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FullIntegerDivisionTest method runIntegerDivisionTest.

/**
 * @param type
 * @param dt1
 * @param dt2
 * @param sparse
 * @param instType
 */
private void runIntegerDivisionTest(OpType type, DataType dt1, DataType dt2, boolean sparse, ExecType instType) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = (instType == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    double sparsity = sparse ? sparsity2 : sparsity1;
    String TEST_CACHE_DIR = "";
    if (TEST_CACHE_ENABLED) {
        double sparsityLeft = 1.0;
        if (dt1 == DataType.MATRIX) {
            sparsityLeft = sparsity;
        }
        double sparsityRight = 1.0;
        if (dt2 == DataType.MATRIX) {
            sparsityRight = sparsity;
        }
        TEST_CACHE_DIR = type.name() + sparsityLeft + "_" + sparsityRight + "/";
    }
    try {
        String TEST_NAME = null;
        switch(type) {
            case MOD:
                TEST_NAME = TEST_NAME1;
                break;
            case DIV:
                TEST_NAME = TEST_NAME2;
                break;
        }
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), input("B"), output("C") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        if (dt1 == DataType.SCALAR && dt2 == DataType.SCALAR) {
            // Clear OUT folder to prevent access denied errors running DML script
            // for tests testModSSDense, testDivSSDense, testModSSSparse, testDivSSSparse
            // due to setOutAndExpectedDeletionDisabled(true).
            TestUtils.clearDirectory(outputDir());
        }
        // generate dataset A
        if (dt1 == DataType.MATRIX) {
            double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 7);
            MatrixCharacteristics mcA = new MatrixCharacteristics(rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, (long) (rows * cols * sparsity));
            writeInputMatrixWithMTD("A", A, true, mcA);
        } else {
            double[][] A = getRandomMatrix(1, 1, min, max, 1.0, 7);
            writeScalarInputMatrixWithMTD("A", A, true);
        }
        // generate dataset B
        if (dt2 == DataType.MATRIX) {
            MatrixCharacteristics mcB = new MatrixCharacteristics(rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, (long) (rows * cols * sparsity));
            double[][] B = getRandomMatrix(rows, cols, min, max, sparsity, 3);
            writeInputMatrixWithMTD("B", B, true, mcB);
        } else {
            double[][] B = getRandomMatrix(1, 1, min, max, 1.0, 3);
            writeScalarInputMatrixWithMTD("B", B, true);
        }
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = null;
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("C");
        if (dt1 == DataType.SCALAR && dt2 == DataType.SCALAR)
            dmlfile = readScalarMatrixFromHDFS("C");
        else
            dmlfile = readDMLMatrixFromHDFS("C");
        // NaN and Infinity currently ignored because R's writeMM replaces them with 1.0E308
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R", true);
    } finally {
        rtplatform = platformOld;
    }
}

Example 93 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FullLogicalScalarRightTest method runLogicalTest.

private void runLogicalTest(Type type, boolean zero, boolean sparse, ExecType et) {
    String TEST_NAME = TEST_NAME1;
    int rows = rows1;
    int cols = cols1;
    double sparsity = sparse ? sparsity2 : sparsity1;
    double constant = zero ? 0 : 0.5;
    String TEST_CACHE_DIR = "";
    if (TEST_CACHE_ENABLED) {
        TEST_CACHE_DIR = type.ordinal() + "_" + constant + "_" + sparsity + "/";
    }
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = (et == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), Integer.toString(type.ordinal()), Double.toString(constant), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + type.ordinal() + " " + constant + " " + expectedDir();
        // generate actual dataset
        double[][] A = getRandomMatrix(rows, cols, -1, 1, sparsity, 7);
        writeInputMatrixWithMTD("A", A, true);
        // run tests
        runTest(true, false, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
    }
}

Example 94 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FullMatrixMultiplicationTest method runVectorMatrixMultiplicationTest.

private void runVectorMatrixMultiplicationTest(boolean sparseM1, boolean sparseM2, ExecType instType) {
    // setup exec type, rows, cols
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = (instType == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        double sparsityA = sparseM1 ? sparsity2 : sparsity1;
        double sparsityB = sparseM2 ? sparsity2 : sparsity1;
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = "vm" + String.valueOf(sparsityA) + "_" + String.valueOf(sparsityB) + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), Integer.toString(1), Integer.toString(colsA), input("B"), Integer.toString(rowsB), Integer.toString(colsB), output("C") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual dataset
        double[][] A = getRandomMatrix(1, colsA, 0, 1, sparsityA, 7);
        writeInputMatrix("A", A, true);
        double[][] B = getRandomMatrix(rowsB, colsB, 0, 1, sparsityB, 3);
        writeInputMatrix("B", B, true);
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("C");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
    }
}

Example 95 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FullMatrixMultiplicationTest method runMatrixMatrixMultiplicationTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runMatrixMatrixMultiplicationTest(boolean sparseM1, boolean sparseM2, ExecType instType) {
    // setup exec type, rows, cols
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = (instType == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        double sparsityA = sparseM1 ? sparsity2 : sparsity1;
        double sparsityB = sparseM2 ? sparsity2 : sparsity1;
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = "mm" + String.valueOf(sparsityA) + "_" + String.valueOf(sparsityB) + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), Integer.toString(rowsA), Integer.toString(colsA), input("B"), Integer.toString(rowsB), Integer.toString(colsB), output("C") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate actual dataset
        double[][] A = getRandomMatrix(rowsA, colsA, 0, 1, sparsityA, 7);
        writeInputMatrix("A", A, true);
        double[][] B = getRandomMatrix(rowsB, colsB, 0, 1, sparsityB, 3);
        writeInputMatrix("B", B, true);
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        // compare matrices
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("C");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
    }
}