Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 11 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class TestUtils method compareMatrices.

/**
 * Compares two matrices given as HashMaps. The matrix containing more nnz
 * is iterated and each cell value compared against the corresponding cell
 * in the smaller matrix, to ensure that all values are compared.<br/>
 * This method does not assert. Instead statistics are added to
 * AssertionBuffer, at the end of the test you should call
 * {@link TestUtils#displayAssertionBuffer()}.
 *
 * @param m1
 * @param m2
 * @param tolerance
 * @return True if matrices are identical regarding tolerance.
 */
public static boolean compareMatrices(HashMap<CellIndex, Double> m1, HashMap<CellIndex, Double> m2, double tolerance, String name1, String name2, boolean ignoreNaN) {
    HashMap<CellIndex, Double> first = m2;
    HashMap<CellIndex, Double> second = m1;
    String namefirst = name2;
    String namesecond = name1;
    boolean flag = true;
    /**
     * to ensure that always the matrix with more nnz is iterated
     */
    if (m1.size() > m2.size()) {
        first = m1;
        second = m2;
        namefirst = name1;
        namesecond = name2;
        flag = false;
    }
    int countErrorWithinTolerance = 0;
    int countIdentical = 0;
    double minerr = -1;
    double maxerr = 0;
    for (CellIndex index : first.keySet()) {
        Double v1 = first.get(index);
        Double v2 = second.get(index);
        if (v1 == null)
            v1 = 0.0;
        if (v2 == null)
            v2 = 0.0;
        if (Math.abs(v1 - v2) < minerr || minerr == -1)
            minerr = Math.abs(v1 - v2);
        if (Math.abs(v1 - v2) > maxerr)
            maxerr = Math.abs(v1 - v2);
        if (!compareCellValue(first.get(index), second.get(index), 0, ignoreNaN)) {
            if (!compareCellValue(first.get(index), second.get(index), tolerance, ignoreNaN)) {
                countErrorWithinTolerance++;
                if (!flag)
                    System.out.println(index + ": " + first.get(index) + " <--> " + second.get(index));
                else
                    System.out.println(index + ": " + second.get(index) + " <--> " + first.get(index));
            }
        } else {
            countIdentical++;
        }
    }
    String assertPrefix = (countErrorWithinTolerance == 0) ? "    " : "!  ";
    _AssertInfos.add(assertPrefix + name1 + "<->" + name2 + " # stored values in " + namefirst + ": " + first.size());
    _AssertInfos.add(assertPrefix + name1 + "<->" + name2 + " # stored values in " + namesecond + ": " + second.size());
    _AssertInfos.add(assertPrefix + name1 + "<->" + name2 + " identical values(z=0): " + countIdentical);
    _AssertInfos.add(assertPrefix + name1 + "<->" + name2 + " wrong values(z=" + tolerance + "): " + countErrorWithinTolerance);
    _AssertInfos.add(assertPrefix + name1 + "<->" + name2 + " min error: " + minerr);
    _AssertInfos.add(assertPrefix + name1 + "<->" + name2 + " max error: " + maxerr);
    if (countErrorWithinTolerance == 0)
        return true;
    _AssertOccured = true;
    return false;
}

Example 12 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class TestUtils method convertHashMapToDoubleArray.

/**
 * @param matrix
 * @param rows
 * @param cols
 * @return
 */
public static double[][] convertHashMapToDoubleArray(HashMap<CellIndex, Double> matrix, int rows, int cols) {
    double[][] ret_arr = new double[rows][cols];
    for (CellIndex ci : matrix.keySet()) {
        int i = ci.row - 1;
        int j = ci.column - 1;
        ret_arr[i][j] = matrix.get(ci);
    }
    return ret_arr;
}

Example 13 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ApplyTransformTest method testApplyTransform.

protected void testApplyTransform(ScriptType scriptType) {
    System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + X + ", " + missing_value_maps + ", " + binning_maps + ", " + dummy_coding_maps + ", " + normalization_maps + "} ------------");
    this.scriptType = scriptType;
    getAndLoadTestConfiguration(TEST_NAME);
    List<String> proArgs = new ArrayList<String>();
    if (scriptType == ScriptType.PYDML) {
        proArgs.add("-python");
    }
    proArgs.add("-stats");
    proArgs.add("-nvargs");
    proArgs.add("X=" + sourceDirectory + X);
    proArgs.add("missing_value_maps=" + (missing_value_maps.equals(" ") ? " " : sourceDirectory + missing_value_maps));
    proArgs.add("bin_defns=" + (binning_maps.equals(" ") ? " " : sourceDirectory + binning_maps));
    proArgs.add("dummy_code_maps=" + (dummy_coding_maps.equals(" ") ? " " : sourceDirectory + dummy_coding_maps));
    proArgs.add("normalization_maps=" + (normalization_maps.equals(" ") ? " " : sourceDirectory + normalization_maps));
    proArgs.add("transformed_X=" + output("transformed_X.mtx"));
    proArgs.add("Log=" + output("log.csv"));
    programArgs = proArgs.toArray(new String[proArgs.size()]);
    fullDMLScriptName = getScript();
    runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
    HashMap<CellIndex, Double> XDML = readDMLMatrixFromHDFS("transformed_X.mtx");
    Iterator<Map.Entry<CellIndex, Double>> iter = XDML.entrySet().iterator();
    while (iter.hasNext()) {
        Map.Entry<CellIndex, Double> elt = iter.next();
        int row = elt.getKey().row;
        int col = elt.getKey().column;
        double val = elt.getValue();
        System.out.println("[" + row + "," + col + "]->" + val);
    }
    boolean success = true;
    if (missing_value_maps != " " && normalization_maps != " ") {
        CellIndex cell;
        if (dummy_coding_maps != " ")
            cell = new CellIndex(3, 3);
        else
            cell = new CellIndex(3, 2);
        if (XDML.containsKey(cell)) {
            double val = XDML.get(cell).doubleValue();
            success = success && (Math.abs(val) < 0.0000001);
        }
    } else if (missing_value_maps != " ") {
        CellIndex cell;
        if (dummy_coding_maps != " ")
            cell = new CellIndex(3, 3);
        else
            cell = new CellIndex(3, 2);
        if (XDML.containsKey(cell)) {
            double val = XDML.get(cell).doubleValue();
            success = success && (Math.abs(-0.2 / 3 - val) < 0.0000001);
        } else
            success = false;
    } else if (normalization_maps != " ") {
        CellIndex cell;
        if (dummy_coding_maps != " ")
            cell = new CellIndex(3, 3);
        else
            cell = new CellIndex(3, 2);
        if (XDML.containsKey(cell)) {
            double val = XDML.get(cell).doubleValue();
            success = success && (Math.abs(0.2 / 3 - val) < 0.0000001);
        } else
            success = false;
    } else {
        CellIndex cell;
        if (dummy_coding_maps != " ")
            cell = new CellIndex(3, 3);
        else
            cell = new CellIndex(3, 2);
        if (XDML.containsKey(cell)) {
            double val = XDML.get(cell).doubleValue();
            success = success && (Math.abs(val) < 0.0000001);
        }
    }
    if (binning_maps != " ") {
        CellIndex cell1, cell2, cell3, cell4;
        if (dummy_coding_maps != " ") {
            cell1 = new CellIndex(1, 1);
            cell2 = new CellIndex(2, 1);
            cell3 = new CellIndex(3, 2);
            cell4 = new CellIndex(4, 2);
        } else {
            cell1 = new CellIndex(1, 1);
            cell2 = new CellIndex(2, 1);
            cell3 = new CellIndex(3, 1);
            cell4 = new CellIndex(4, 1);
        }
        if (!XDML.containsKey(cell1))
            success = false;
        else
            success = success && (XDML.get(cell1).doubleValue() == 1);
        if (!XDML.containsKey(cell2))
            success = false;
        else
            success = success && (XDML.get(cell2).doubleValue() == 1);
        if (!XDML.containsKey(cell3))
            success = false;
        else
            success = success && (dummy_coding_maps != " ") ? (XDML.get(cell3).doubleValue() == 1) : (XDML.get(cell3).doubleValue() == 2);
        if (!XDML.containsKey(cell4))
            success = false;
        else
            success = success && (dummy_coding_maps != " ") ? (XDML.get(cell4).doubleValue() == 1) : (XDML.get(cell4).doubleValue() == 2);
    }
}

Example 14 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class BivariateCategoricalCategoricallTest method testOddsRatio.

@Test
public void testOddsRatio() {
    TestConfiguration config = getTestConfiguration(TEST_ODDS_RATIO);
    config.addVariable("rows", rows);
    loadTestConfiguration(config);
    /* This is for running the junit test the new way, i.e., construct the arguments directly */
    String CC_HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = CC_HOME + TEST_ODDS_RATIO + ".dml";
    programArgs = new String[] { "-args", input("A"), Integer.toString(rows), input("B"), output("oddsRatio"), output("sigma"), output("leftConf"), output("rightConf"), output("sigmasAway") // output("chiSquared"),
    // output(degFreedom"),
    // output("pValue"),
    // output("cramersV")
    };
    fullRScriptName = CC_HOME + TEST_ODDS_RATIO + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
    // current test works only for 2x2 contingency tables => #categories must be 2
    int numCat = 2;
    double[][] A = getRandomMatrix(rows, 1, 1, numCat, 1, System.currentTimeMillis());
    double[][] B = getRandomMatrix(rows, 1, 1, numCat, 1, System.currentTimeMillis() + 1);
    TestUtils.round(A);
    TestUtils.round(B);
    writeInputMatrix("A", A, true);
    writeInputMatrix("B", B, true);
    runTest(true, false, null, -1);
    runRScript(true);
    for (String file : config.getOutputFiles()) {
        /* NOte that some files do not contain matrix, but just a single scalar value inside */
        HashMap<CellIndex, Double> dmlfile;
        HashMap<CellIndex, Double> rfile;
        if (file.endsWith(".scalar")) {
            file = file.replace(".scalar", "");
            dmlfile = readDMLScalarFromHDFS(file);
            rfile = readRScalarFromFS(file);
        } else {
            dmlfile = readDMLMatrixFromHDFS(file);
            rfile = readRMatrixFromFS(file);
        }
        TestUtils.compareMatrices(dmlfile, rfile, eps, file + "-DML", file + "-R");
    }
}

Example 15 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class BivariateOrdinalOrdinalTest method testOrdinalOrdinalWithWeights.

@Test
public void testOrdinalOrdinalWithWeights() {
    TestConfiguration config = getTestConfiguration(TEST_ORDINAL_ORDINAL_WEIGHTS);
    config.addVariable("rows", rows);
    loadTestConfiguration(config);
    /* This is for running the junit test the new way, i.e., construct the arguments directly */
    String OO_HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = OO_HOME + TEST_ORDINAL_ORDINAL_WEIGHTS + ".dml";
    programArgs = new String[] { "-args", input("A"), Integer.toString(rows), input("B"), input("WM"), output("Spearman") };
    fullRScriptName = OO_HOME + TEST_ORDINAL_ORDINAL_WEIGHTS + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
    double[][] A = getRandomMatrix(rows, 1, 1, ncatA, 1, System.currentTimeMillis());
    double[][] B = getRandomMatrix(rows, 1, 1, ncatB, 1, System.currentTimeMillis());
    double[][] WM = getRandomMatrix(rows, 1, 1, maxW, 1, System.currentTimeMillis());
    TestUtils.floor(A);
    TestUtils.floor(B);
    TestUtils.floor(WM);
    writeInputMatrix("A", A, true);
    writeInputMatrix("B", B, true);
    writeInputMatrix("WM", WM, true);
    runTest(true, false, null, -1);
    runRScript(true);
    for (String file : config.getOutputFiles()) {
        /* NOte that some files do not contain matrix, but just a single scalar value inside */
        HashMap<CellIndex, Double> dmlfile;
        HashMap<CellIndex, Double> rfile;
        if (file.endsWith(".scalar")) {
            file = file.replace(".scalar", "");
            dmlfile = readDMLScalarFromHDFS(file);
            rfile = readRScalarFromFS(file);
        } else {
            dmlfile = readDMLMatrixFromHDFS(file);
            rfile = readRMatrixFromFS(file);
        }
        TestUtils.compareMatrices(dmlfile, rfile, eps, file + "-DML", file + "-R");
    }
}