Examples with CellIndex org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex used on opensource projects

Example 16 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class BivariateOrdinalOrdinalTest method testOrdinalOrdinal.

@Test
public void testOrdinalOrdinal() {
    TestConfiguration config = getTestConfiguration(TEST_ORDINAL_ORDINAL);
    config.addVariable("rows", rows);
    loadTestConfiguration(config);
    /* This is for running the junit test the new way, i.e., construct the arguments directly */
    String OO_HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = OO_HOME + TEST_ORDINAL_ORDINAL + ".dml";
    programArgs = new String[] { "-args", input("A"), Integer.toString(rows), input("B"), output("Spearman") };
    fullRScriptName = OO_HOME + TEST_ORDINAL_ORDINAL + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
    double[][] A = getRandomMatrix(rows, 1, 1, ncatA, 1, System.currentTimeMillis());
    double[][] B = getRandomMatrix(rows, 1, 1, ncatB, 1, System.currentTimeMillis() + 1);
    TestUtils.floor(A);
    TestUtils.floor(B);
    writeInputMatrix("A", A, true);
    writeInputMatrix("B", B, true);
    runTest(true, false, null, -1);
    runRScript(true);
    for (String file : config.getOutputFiles()) {
        /* NOte that some files do not contain matrix, but just a single scalar value inside */
        HashMap<CellIndex, Double> dmlfile;
        HashMap<CellIndex, Double> rfile;
        if (file.endsWith(".scalar")) {
            file = file.replace(".scalar", "");
            dmlfile = readDMLScalarFromHDFS(file);
            rfile = readRScalarFromFS(file);
        } else {
            dmlfile = readDMLMatrixFromHDFS(file);
            rfile = readRMatrixFromFS(file);
        }
        TestUtils.compareMatrices(dmlfile, rfile, eps, file + "-DML", file + "-R");
    }
}

Example 17 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class BivariateScaleCategoricalTest method testScaleCategorical.

@Test
public void testScaleCategorical() {
    TestConfiguration config = getTestConfiguration(TEST_SCALE_NOMINAL);
    config.addVariable("rows", rows);
    loadTestConfiguration(config);
    /* This is for running the junit test the new way, i.e., construct the arguments directly */
    String SC_HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = SC_HOME + TEST_SCALE_NOMINAL + ".dml";
    programArgs = new String[] { "-args", input("A"), Integer.toString(rows), input("Y"), output("VarY"), output("MeanY"), output("CFreqs"), output("CMeans"), output("CVars"), output("Eta"), output("AnovaF") };
    fullRScriptName = SC_HOME + TEST_SCALE_NOMINAL + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
    double[][] A = getRandomMatrix(rows, 1, 1, ncatA, 1, System.currentTimeMillis());
    TestUtils.floor(A);
    double[][] Y = getRandomMatrix(rows, 1, minVal, maxVal, 0.1, System.currentTimeMillis());
    writeInputMatrix("A", A, true);
    writeInputMatrix("Y", Y, true);
    boolean exceptionExpected = false;
    runTest(true, exceptionExpected, null, -1);
    runRScript(true);
    for (String file : config.getOutputFiles()) {
        /* NOte that some files do not contain matrix, but just a single scalar value inside */
        HashMap<CellIndex, Double> dmlfile;
        HashMap<CellIndex, Double> rfile;
        if (file.endsWith(".scalar")) {
            file = file.replace(".scalar", "");
            dmlfile = readDMLScalarFromHDFS(file);
            rfile = readRScalarFromFS(file);
        } else {
            dmlfile = readDMLMatrixFromHDFS(file);
            rfile = readRMatrixFromFS(file);
        }
        TestUtils.compareMatrices(dmlfile, rfile, eps, file + "-DML", file + "-R");
    }
}

Example 18 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class UnivariateCategoricalTest method testWeightedCategoricalWithR.

@Test
public void testWeightedCategoricalWithR() {
    TestConfiguration config = getTestConfiguration("WeightedCategoricalTest");
    config.addVariable("rows1", rows1);
    loadTestConfiguration(config);
    // This is for running the junit test the new way, i.e., construct the arguments directly
    String C_HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = C_HOME + "WeightedCategoricalTest" + ".dml";
    programArgs = new String[] { "-args", input("vector"), Integer.toString(rows1), input("weight"), output("Nc"), output("R"), output("Pc"), output("C"), output("Mode") };
    fullRScriptName = C_HOME + "WeightedCategoricalTest" + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
    createHelperMatrix();
    double[][] vector = getRandomMatrix(rows1, 1, 1, 10, 1, System.currentTimeMillis());
    OrderStatisticsTest.round(vector);
    double[][] weight = getRandomMatrix(rows1, 1, 1, 10, 1, System.currentTimeMillis());
    OrderStatisticsTest.round(weight);
    writeInputMatrix("vector", vector, true);
    writeInputMatrix("weight", weight, true);
    boolean exceptionExpected = false;
    runTest(true, exceptionExpected, null, -1);
    runRScript(true);
    for (String file : config.getOutputFiles()) {
        // NOte that some files do not contain matrix, but just a single scalar value inside
        HashMap<CellIndex, Double> dmlfile;
        HashMap<CellIndex, Double> rfile;
        if (file.endsWith(".scalar")) {
            file = file.replace(".scalar", "");
            dmlfile = readDMLScalarFromHDFS(file);
            rfile = readRScalarFromFS(file);
        } else {
            dmlfile = readDMLMatrixFromHDFS(file);
            rfile = readRMatrixFromFS(file);
        }
        TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
    }
}

Example 19 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class UnivariateStatsBase method testScaleWithR.

/**
 * Shared test driver for tests of univariate statistics over continuous,
 * scaled, but not weighted, data
 *
 * @param sz
 *            size of primary input vector
 * @param rng
 *            range of randomly-generated values to use
 * @param sp
 *            sparsity of generated data
 * @param rt
 *            backend platform to test
 */
protected void testScaleWithR(SIZE sz, RANGE rng, SPARSITY sp, RUNTIME_PLATFORM rt) {
    RUNTIME_PLATFORM oldrt = rtplatform;
    rtplatform = rt;
    try {
        TestConfiguration config = getTestConfiguration("Scale");
        config.addVariable("rows1", sz.size);
        config.addVariable("rows2", rows2);
        loadTestConfiguration(config);
        // This is for running the junit test the new way, i.e., construct
        // the arguments directly
        String S_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = S_HOME + "Scale" + ".dml";
        programArgs = new String[] { "-args", input("vector"), Integer.toString(sz.size), input("prob"), Integer.toString(rows2), output("mean"), output("std"), output("se"), output("var"), output("cv"), output("min"), output("max"), output("rng"), output("g1"), output("se_g1"), output("g2"), output("se_g2"), output("median"), output("iqm"), output("out_minus"), output("out_plus"), output("quantile") };
        fullRScriptName = S_HOME + "Scale" + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        double[][] vector = getRandomMatrix(sz.size, 1, rng.min, rng.max, sp.sparsity, 20160124165501L);
        double[][] prob = getRandomMatrix(rows2, 1, 0, 1, 1, 20160124165502L);
        writeInputMatrix("vector", vector, true);
        writeInputMatrix("prob", prob, true);
        runTest(true, false, null, -1);
        runRScript(true);
        for (String file : config.getOutputFiles()) {
            // NOte that some files do not contain matrix, but just a single
            // scalar value inside
            HashMap<CellIndex, Double> dmlfile;
            HashMap<CellIndex, Double> rfile;
            if (file.endsWith(".scalar")) {
                file = file.replace(".scalar", "");
                dmlfile = readDMLScalarFromHDFS(file);
                rfile = readRScalarFromFS(file);
            } else {
                dmlfile = readDMLMatrixFromHDFS(file);
                rfile = readRMatrixFromFS(file);
            }
            TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
        }
    } finally {
        rtplatform = oldrt;
    }
}

Example 20 with CellIndex

use of org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex in project incubator-systemml by apache.

the class ParForBivariateStatsTest method runParForBivariateStatsTest.

/**
 * @param outer execution mode of outer parfor loop
 * @param inner execution mode of inner parfor loop
 * @param instType execution mode of instructions
 */
private void runParForBivariateStatsTest(boolean parallel, PExecMode outer, PExecMode inner, ExecType instType) {
    // inst exec type, influenced via rows
    int rows = -1;
    if (instType == ExecType.CP)
        rows = rows1;
    else
        // if type MR
        rows = rows2;
    // script
    int scriptNum = -1;
    if (parallel) {
        if (inner == PExecMode.REMOTE_MR)
            scriptNum = 2;
        else if (outer == PExecMode.REMOTE_MR)
            scriptNum = 3;
        else if (outer == PExecMode.LOCAL)
            scriptNum = 1;
        else
            // optimized
            scriptNum = 4;
    } else {
        scriptNum = 0;
    }
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    // config.addVariable("rows", rows);
    // config.addVariable("cols", cols);
    loadTestConfiguration(config);
    /* This is for running the junit test the new way, i.e., construct the arguments directly */
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml";
    programArgs = new String[] { "-args", input("D"), input("S1"), input("S2"), input("K1"), input("K2"), output("bivarstats"), Integer.toString(rows), Integer.toString(cols), Integer.toString(cols2), Integer.toString(cols2 * cols2), Integer.toString((int) maxVal) };
    fullRScriptName = HOME + TEST_NAME + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + Integer.toString((int) maxVal) + " " + expectedDir();
    // generate actual dataset
    double[][] D = getRandomMatrix(rows, cols, minVal, maxVal, 1, 7777);
    double[] Dkind = new double[cols];
    for (int i = 0; i < cols; i++) {
        // kind 1,2,3
        Dkind[i] = (i % 3) + 1;
        if (Dkind[i] != 1)
            // for ordinal and categorical vars
            TestUtils.floor(D, i);
    }
    writeInputMatrix("D", D, true);
    // generate attribute sets
    double[][] S1 = getRandomMatrix(1, cols2, 1, cols + 1 - eps, 1, 1112);
    double[][] S2 = getRandomMatrix(1, cols2, 1, cols + 1 - eps, 1, 1113);
    TestUtils.floor(S1);
    TestUtils.floor(S2);
    writeInputMatrix("S1", S1, true);
    writeInputMatrix("S2", S2, true);
    // generate kind for attributes (1,2,3)
    double[][] K1 = new double[1][cols2];
    double[][] K2 = new double[1][cols2];
    for (int i = 0; i < cols2; i++) {
        K1[0][i] = Dkind[(int) S1[0][i] - 1];
        K2[0][i] = Dkind[(int) S2[0][i] - 1];
    }
    writeInputMatrix("K1", K1, true);
    writeInputMatrix("K2", K2, true);
    boolean exceptionExpected = false;
    runTest(true, exceptionExpected, null, 92);
    runRScript(true);
    // compare matrices
    for (String out : new String[] { "bivar.stats", "category.counts", "category.means", "category.variances" }) {
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("bivarstats/" + out);
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS(out);
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    }
}