Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 6 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class FullPPredScalarRightTest method runPPredTest.

/**
	 * 
	 * @param type
	 * @param instType
	 * @param sparse
	 */
private void runPPredTest(Type type, boolean zero, boolean sparse, ExecType et) {
    String TEST_NAME = TEST_NAME1;
    int rows = rows1;
    int cols = cols1;
    double sparsity = sparse ? sparsity2 : sparsity1;
    double constant = zero ? 0 : 0.5;
    String TEST_CACHE_DIR = "";
    if (TEST_CACHE_ENABLED) {
        TEST_CACHE_DIR = type.ordinal() + "_" + constant + "_" + sparsity + "/";
    }
    //rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = (et == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), Integer.toString(type.ordinal()), Double.toString(constant), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + type.ordinal() + " " + constant + " " + expectedDir();
        //generate actual dataset
        double[][] A = getRandomMatrix(rows, cols, -1, 1, sparsity, 7);
        writeInputMatrixWithMTD("A", A, true);
        //run tests
        runTest(true, false, null, -1);
        runRScript(true);
        //compare matrices 
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
    }
}

Example 7 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class AlgorithmPNMF method runPNMFTest.

//TODO requires proper handling of blocksize constraints
//@Test
//public void testPNMFDenseSP() {
//	runPNMFTest(TEST_NAME1, false, false, ExecType.SPARK);
//}
//@Test
//public void testPNMFSparseSP() {
//	runPNMFTest(TEST_NAME1, false, true, ExecType.SPARK);
//}
private void runPNMFTest(String testname, boolean rewrites, boolean sparse, ExecType instType) {
    boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String TEST_NAME = testname;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        fullDMLScriptName = "scripts/staging/PNMF.dml";
        programArgs = new String[] { "-explain", "-stats", "-args", input("X"), input("W"), input("H"), String.valueOf(rank), String.valueOf(epsilon), String.valueOf(maxiter), output("W"), output("H") };
        rCmd = getRCmd(inputDir(), String.valueOf(rank), String.valueOf(epsilon), String.valueOf(maxiter), expectedDir());
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
        //generate actual datasets
        double[][] X = getRandomMatrix(rows, cols, 0, 1, sparse ? sparsity2 : sparsity1, 234);
        writeInputMatrixWithMTD("X", X, true);
        double[][] W = getRandomMatrix(rows, rank, 0, 0.025, 1.0, 3);
        writeInputMatrixWithMTD("W", W, true);
        double[][] H = getRandomMatrix(rank, cols, 0, 0.025, 1.0, 7);
        writeInputMatrixWithMTD("H", H, true);
        runTest(true, false, null, -1);
        runRScript(true);
        //compare matrices 
        HashMap<CellIndex, Double> dmlW = readDMLMatrixFromHDFS("W");
        HashMap<CellIndex, Double> dmlH = readDMLMatrixFromHDFS("H");
        HashMap<CellIndex, Double> rW = readRMatrixFromFS("W");
        HashMap<CellIndex, Double> rH = readRMatrixFromFS("H");
        TestUtils.compareMatrices(dmlW, rW, eps, "Stat-DML", "Stat-R");
        TestUtils.compareMatrices(dmlH, rH, eps, "Stat-DML", "Stat-R");
        Assert.assertTrue(heavyHittersContainsSubString("spoof") || heavyHittersContainsSubString("sp_spoof"));
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
        OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true;
        OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
    }
}

Example 8 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class TestUtils method convertHashMapToDoubleArray.

/**
 * Method to convert a hashmap of matrix entries into a double array
 * @param matrix
 * @return
 */
public static double[][] convertHashMapToDoubleArray(HashMap<CellIndex, Double> matrix) {
    int max_rows = -1, max_cols = -1;
    for (CellIndex ci : matrix.keySet()) {
        if (ci.row > max_rows) {
            max_rows = ci.row;
        }
        if (ci.column > max_cols) {
            max_cols = ci.column;
        }
    }
    double[][] ret_arr = new double[max_rows][max_cols];
    for (CellIndex ci : matrix.keySet()) {
        int i = ci.row - 1;
        int j = ci.column - 1;
        ret_arr[i][j] = matrix.get(ci);
    }
    return ret_arr;
}

Example 9 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class TestUtils method compareMMMatrixWithJavaMatrix.

/**
 * <p>
 * Compares the expected values calculated in Java by testcase and which are
 * in the normal filesystem, with those calculated by SystemML located in
 * HDFS with Matrix Market format
 * </p>
 *
 * @param expectedFile
 *            file with expected values, which is located in OS filesystem
 * @param actualDir
 *            file with actual values, which is located in HDFS
 * @param epsilon
 *            tolerance for value comparison
 */
public static void compareMMMatrixWithJavaMatrix(String expectedFile, String actualDir, double epsilon) {
    try {
        Path outDirectory = new Path(actualDir);
        Path compareFile = new Path(expectedFile);
        FileSystem fs = IOUtilFunctions.getFileSystem(outDirectory, conf);
        FSDataInputStream fsin = fs.open(compareFile);
        HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>();
        String[] expRcn = null;
        try (BufferedReader compareIn = new BufferedReader(new InputStreamReader(fsin))) {
            // skip the header of Matrix Market file
            String line = compareIn.readLine();
            // rows, cols and nnz
            line = compareIn.readLine();
            expRcn = line.split(" ");
            while ((line = compareIn.readLine()) != null) {
                StringTokenizer st = new StringTokenizer(line, " ");
                int i = Integer.parseInt(st.nextToken());
                int j = Integer.parseInt(st.nextToken());
                double v = Double.parseDouble(st.nextToken());
                expectedValues.put(new CellIndex(i, j), v);
            }
        }
        HashMap<CellIndex, Double> actualValues = new HashMap<CellIndex, Double>();
        FSDataInputStream fsout = fs.open(outDirectory);
        try (BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout))) {
            // skip MM header
            String line = outIn.readLine();
            // rows, cols and nnz
            line = outIn.readLine();
            String[] rcn = line.split(" ");
            if (Integer.parseInt(expRcn[0]) != Integer.parseInt(rcn[0])) {
                System.out.println(" Rows mismatch: expected " + Integer.parseInt(expRcn[0]) + ", actual " + Integer.parseInt(rcn[0]));
            } else if (Integer.parseInt(expRcn[1]) != Integer.parseInt(rcn[1])) {
                System.out.println(" Cols mismatch: expected " + Integer.parseInt(expRcn[1]) + ", actual " + Integer.parseInt(rcn[1]));
            } else if (Integer.parseInt(expRcn[2]) != Integer.parseInt(rcn[2])) {
                System.out.println(" Nnz mismatch: expected " + Integer.parseInt(expRcn[2]) + ", actual " + Integer.parseInt(rcn[2]));
            }
            while ((line = outIn.readLine()) != null) {
                StringTokenizer st = new StringTokenizer(line, " ");
                int i = Integer.parseInt(st.nextToken());
                int j = Integer.parseInt(st.nextToken());
                double v = Double.parseDouble(st.nextToken());
                actualValues.put(new CellIndex(i, j), v);
            }
        }
        int countErrors = 0;
        for (CellIndex index : expectedValues.keySet()) {
            Double expectedValue = expectedValues.get(index);
            Double actualValue = actualValues.get(index);
            if (expectedValue == null)
                expectedValue = 0.0;
            if (actualValue == null)
                actualValue = 0.0;
            if (!compareCellValue(expectedValue, actualValue, epsilon, false)) {
                System.out.println(expectedFile + ": " + index + " mismatch: expected " + expectedValue + ", actual " + actualValue);
                countErrors++;
            }
        }
        assertTrue("for file " + actualDir + " " + countErrors + " values are not equal", countErrors == 0);
    } catch (IOException e) {
        fail("unable to read file: " + e.getMessage());
    }
}

Example 10 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class TestUtils method readRMatrixFromFS.

/**
 * Reads values from a matrix file in OS's FS in R format
 *
 * @deprecated You should not use this method, it is recommended to use the
 *             corresponding method in AutomatedTestBase
 *
 * @param filePath
 * @return
 */
public static HashMap<CellIndex, Double> readRMatrixFromFS(String filePath) {
    HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>();
    BufferedReader reader = null;
    try {
        reader = new BufferedReader(new FileReader(filePath));
        // skip both R header lines
        String line = reader.readLine();
        int matrixType = -1;
        if (line.endsWith(" general"))
            matrixType = 1;
        if (line.endsWith(" symmetric"))
            matrixType = 2;
        if (matrixType == -1)
            throw new RuntimeException("unknown matrix type while reading R matrix: " + line);
        // header line with dimension and nnz information
        line = reader.readLine();
        while ((line = reader.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, " ");
            int i = Integer.parseInt(st.nextToken());
            int j = Integer.parseInt(st.nextToken());
            if (st.hasMoreTokens()) {
                double v = Double.parseDouble(st.nextToken());
                if (v == 0.0)
                    continue;
                expectedValues.put(new CellIndex(i, j), v);
                if (matrixType == 2)
                    expectedValues.put(new CellIndex(j, i), v);
            } else
                expectedValues.put(new CellIndex(i, j), 1.0);
        }
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    } finally {
        IOUtilFunctions.closeSilently(reader);
    }
    return expectedValues;
}